From 7380808b26a5fb30440fcc72a06fc58e0ca2ebf3 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Mon, 2 Jun 2025 16:39:21 +0300 Subject: [PATCH 001/312] tests: Faster batches:: IT tests Use shared server + unique indices where possible Related-to: https://github.com/meilisearch/meilisearch/issues/4840 Signed-off-by: Martin Tzvetanov Grigorov --- crates/meilisearch/tests/batches/mod.rs | 190 ++++++++++++++---------- 1 file changed, 108 insertions(+), 82 deletions(-) diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index e775d1ea4..4613f71fc 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -8,8 +8,8 @@ use crate::json; #[actix_rt::test] async fn error_get_unexisting_batch_status() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (task, _coder) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); let (response, code) = index.get_batch(1).await; @@ -27,8 +27,8 @@ async fn error_get_unexisting_batch_status() { #[actix_rt::test] async fn get_batch_status() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); let (_response, code) = index.get_batch(0).await; @@ -37,8 +37,8 @@ async fn get_batch_status() { #[actix_rt::test] async fn list_batches() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = index.create(None).await; @@ -62,7 +62,7 @@ async fn list_batches_pagination_and_reverse() { let index = server.index(format!("test-{i}")); last_batch = Some(index.create(None).await.0.uid()); } - server.wait_task(last_batch.unwrap()).await; + server.wait_task(last_batch.unwrap()).await.succeeded(); let (response, code) = server.batches_filter("limit=3").await; assert_eq!(code, 200); @@ -139,8 +139,8 @@ async fn list_batches_with_star_filters() { #[actix_rt::test] async fn list_batches_status_filtered() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = index.create(None).await; @@ -161,8 +161,8 @@ async fn list_batches_status_filtered() { #[actix_rt::test] async fn list_batches_type_filtered() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (task, _) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); let (task, _) = index.delete().await; @@ -183,8 +183,8 @@ async fn list_batches_type_filtered() { #[actix_rt::test] async fn list_batches_invalid_canceled_by_filter() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); @@ -195,8 +195,8 @@ async fn list_batches_invalid_canceled_by_filter() { #[actix_rt::test] async fn list_batches_status_and_type_filtered() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = index.update(Some("id")).await; @@ -219,7 +219,7 @@ async fn list_batches_status_and_type_filtered() { #[actix_rt::test] async fn list_batch_filter_error() { - let server = Server::new().await; + let server = Server::new_shared(); let (response, code) = server.batches_filter("lol=pied").await; assert_eq!(code, 400, "{}", response); @@ -268,14 +268,15 @@ async fn list_batch_filter_error() { #[actix_web::test] async fn test_summarized_document_addition_or_update() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (task, _status_code) = index.add_documents(json!({ "id": 42, "content": "doggos & fluff" }), None).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(0).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", @@ -286,7 +287,7 @@ async fn test_summarized_document_addition_or_update() { }, @r###" { - "uid": 0, + "uid": "[uid]", "progress": null, "details": { "receivedDocuments": 1, @@ -320,6 +321,7 @@ async fn test_summarized_document_addition_or_update() { let (batch, _) = index.get_batch(1).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", @@ -330,7 +332,7 @@ async fn test_summarized_document_addition_or_update() { }, @r###" { - "uid": 1, + "uid": "[uid]", "progress": null, "details": { "receivedDocuments": 1, @@ -360,23 +362,25 @@ async fn test_summarized_document_addition_or_update() { #[actix_web::test] async fn test_summarized_delete_documents_by_batch() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (task, _status_code) = index.delete_batch(vec![1, 2, 3]).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(0).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".stats.indexUids" => "{\n\t\"test\": 1}" }, @r###" { - "uid": 0, + "uid": "[uid]", "progress": null, "details": { "providedIds": 3, @@ -447,25 +451,27 @@ async fn test_summarized_delete_documents_by_batch() { #[actix_web::test] async fn test_summarized_delete_documents_by_filter() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (task, _status_code) = index.delete_document_by_filter(json!({ "filter": "doggo = bernese" })).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(0).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".stats.indexUids" => "{\n\t\"test\": 1}" }, @r###" { - "uid": 0, + "uid": "[uid]", "progress": null, "details": { "providedIds": 0, @@ -583,11 +589,11 @@ async fn test_summarized_delete_documents_by_filter() { #[actix_web::test] async fn test_summarized_delete_document_by_id() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (task, _status_code) = index.delete_document(1).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(0).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -596,7 +602,8 @@ async fn test_summarized_delete_document_by_id() { ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".stats.indexUids" => "{\n\t\"test\": 1}" }, @r###" { @@ -671,8 +678,8 @@ async fn test_summarized_delete_document_by_id() { #[actix_web::test] async fn test_summarized_settings_update() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); // here we should find my payload even in the failed batch. let (response, code) = index.update_settings(json!({ "rankingRules": ["custom"] })).await; meili_snap::snapshot!(code, @"400 Bad Request"); @@ -687,20 +694,24 @@ async fn test_summarized_settings_update() { let (task,_status_code) = index.update_settings(json!({ "displayedAttributes": ["doggos", "name"], "filterableAttributes": ["age", "nb_paw_pads"], "sortableAttributes": ["iq"] })).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(0).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]" + ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", + ".stats.indexUids" => "{\n\t\"test\": 1}", + ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" + }, @r###" { - "uid": 0, + "uid": "[uid]", "progress": null, "details": { "displayedAttributes": [ @@ -731,30 +742,33 @@ async fn test_summarized_settings_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" } "###); } #[actix_web::test] async fn test_summarized_index_creation() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(0).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".stats.indexUids" => "{\n\t\"test\": 1}", + ".batchCreationComplete" => "task with id X of type `indexCreation` cannot be batched" }, @r###" { - "uid": 0, + "uid": "[uid]", "progress": null, "details": {}, "stats": { @@ -773,7 +787,7 @@ async fn test_summarized_index_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id 0 of type `indexCreation` cannot be batched" + "batchCreationComplete": "task with id X of type `indexCreation` cannot be batched" } "###); @@ -819,8 +833,8 @@ async fn test_summarized_index_creation() { #[actix_web::test] async fn test_summarized_index_deletion() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); let (ret, _code) = index.delete().await; let batch = index.wait_task(ret.uid()).await.failed(); snapshot!(batch, @@ -828,7 +842,7 @@ async fn test_summarized_index_deletion() { { "uid": "[uid]", "batchUid": "[batch_uid]", - "indexUid": "test", + "indexUid": "[uuid]", "status": "failed", "type": "indexDeletion", "canceledBy": null, @@ -836,7 +850,7 @@ async fn test_summarized_index_deletion() { "deletedDocuments": 0 }, "error": { - "message": "Index `test` not found.", + "message": "Index `[uuid]` not found.", "code": "index_not_found", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#index_not_found" @@ -859,7 +873,7 @@ async fn test_summarized_index_deletion() { { "uid": "[uid]", "batchUid": "[batch_uid]", - "indexUid": "test", + "indexUid": "[uuid]", "status": "succeeded", "type": "documentAdditionOrUpdate", "canceledBy": null, @@ -928,24 +942,27 @@ async fn test_summarized_index_deletion() { #[actix_web::test] async fn test_summarized_index_update() { - let server = Server::new().await; - let index = server.index("test"); + let server = Server::new_shared(); + let index = server.unique_index(); // If the index doesn't exist yet, we should get errors with or without the primary key. let (task, _status_code) = index.update(None).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(0).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".stats.indexUids" => "{\n\t\"test\": 1}", + ".batchCreationComplete" => "task with id X of type `indexUpdate` cannot be batched" }, @r###" { - "uid": 0, + "uid": "[uid]", "progress": null, "details": {}, "stats": { @@ -964,7 +981,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id 0 of type `indexUpdate` cannot be batched" + "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" } "###); @@ -1089,26 +1106,28 @@ async fn test_summarized_index_update() { #[actix_web::test] async fn test_summarized_index_swap() { - let server = Server::new().await; + let server = Server::new_shared(); let (task, _status_code) = server .index_swap(json!([ { "indexes": ["doggos", "cattos"] } ])) .await; server.wait_task(task.uid()).await.failed(); - let (batch, _) = server.get_batch(0).await; + let (batch, _) = server.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".batchCreationComplete" => "task with id X of type `indexSwap` cannot be batched" }, @r###" { - "uid": 0, + "uid": "[uid]", "progress": null, "details": { "swaps": [ @@ -1134,31 +1153,35 @@ async fn test_summarized_index_swap() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id 0 of type `indexSwap` cannot be batched" + "batchCreationComplete": "task with id X of type `indexSwap` cannot be batched" } "###); - server.index("doggos").create(None).await; - let (task, _status_code) = server.index("cattos").create(None).await; + let doggos_index = server.unique_index(); + doggos_index.create(None).await; + let cattos_index = server.unique_index(); + let (task, _status_code) = cattos_index.create(None).await; server .index_swap(json!([ - { "indexes": ["doggos", "cattos"] } + { "indexes": [doggos_index.uid, cattos_index.uid] } ])) .await; server.wait_task(task.uid()).await.succeeded(); let (batch, _) = server.get_batch(1).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".stats.indexUids" => "{\n\t\"doggos\": 1}" }, @r###" { - "uid": 1, + "uid": "[uid]", "progress": null, "details": {}, "stats": { @@ -1184,12 +1207,12 @@ async fn test_summarized_index_swap() { #[actix_web::test] async fn test_summarized_batch_cancelation() { - let server = Server::new().await; - let index = server.index("doggos"); + let server = Server::new_shared(); + let index = server.unique_index(); // to avoid being flaky we're only going to cancel an already finished batch :( let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); - let (task, _status_code) = server.cancel_tasks("uids=0").await; + let (task, _status_code) = server.cancel_tasks(format!("uids={}", task.uid()).as_str()).await; index.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(1).await; assert_json_snapshot!(batch, @@ -1231,31 +1254,33 @@ async fn test_summarized_batch_cancelation() { #[actix_web::test] async fn test_summarized_batch_deletion() { - let server = Server::new().await; - let index = server.index("doggos"); + let server = Server::new_shared(); + let index = server.unique_index(); // to avoid being flaky we're only going to delete an already finished batch :( let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); - let (task, _status_code) = server.delete_tasks("uids=0").await; + let (task, _status_code) = server.delete_tasks(format!("uids={}", task.uid()).as_str()).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(1).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".details.originalFilter" => "?uids=X" }, @r###" { - "uid": 1, + "uid": "[uid]", "progress": null, "details": { "matchedTasks": 1, "deletedTasks": 1, - "originalFilter": "?uids=0" + "originalFilter": "?uids=X" }, "stats": { "totalNbTasks": 1, @@ -1278,12 +1303,13 @@ async fn test_summarized_batch_deletion() { #[actix_web::test] async fn test_summarized_dump_creation() { - let server = Server::new().await; + let server = Server::new_shared(); let (task, _status_code) = server.create_dump().await; - server.wait_task(task.uid()).await; - let (batch, _) = server.get_batch(0).await; + server.wait_task(task.uid()).await.succeeded(); + let (batch, _) = server.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".details.dumpUid" => "[dumpUid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", @@ -1294,7 +1320,7 @@ async fn test_summarized_dump_creation() { }, @r###" { - "uid": 0, + "uid": "[uid]", "progress": null, "details": { "dumpUid": "[dumpUid]" From cb15e5c67e1c6fed85f3d6c85875b9bb34e75093 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 3 Jun 2025 09:13:56 +0300 Subject: [PATCH 002/312] WIP: More snapshot updates Signed-off-by: Martin Tzvetanov Grigorov --- crates/meilisearch/tests/batches/mod.rs | 107 +++++++++++------------- 1 file changed, 48 insertions(+), 59 deletions(-) diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index 4613f71fc..bb926af70 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -12,10 +12,10 @@ async fn error_get_unexisting_batch_status() { let index = server.unique_index(); let (task, _coder) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); - let (response, code) = index.get_batch(1).await; + let (response, code) = index.get_batch(task.uid() as u32).await; let expected_response = json!({ - "message": "Batch `1` not found.", + "message": format!("Batch `{}` not found.", task.uid()), "code": "batch_not_found", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#batch_not_found" @@ -147,15 +147,15 @@ async fn list_batches_status_filtered() { index.wait_task(task.uid()).await.failed(); let (response, code) = index.filtered_batches(&[], &["succeeded"], &[]).await; - assert_eq!(code, 200, "{}", response); + assert_eq!(code, 200, "{response}"); assert_eq!(response["results"].as_array().unwrap().len(), 1); let (response, code) = index.filtered_batches(&[], &["succeeded"], &[]).await; - assert_eq!(code, 200, "{}", response); + assert_eq!(code, 200, "{response}"); assert_eq!(response["results"].as_array().unwrap().len(), 1); let (response, code) = index.filtered_batches(&[], &["succeeded", "failed"], &[]).await; - assert_eq!(code, 200, "{}", response); + assert_eq!(code, 200, "{response}"); assert_eq!(response["results"].as_array().unwrap().len(), 2); } @@ -168,16 +168,16 @@ async fn list_batches_type_filtered() { let (task, _) = index.delete().await; index.wait_task(task.uid()).await.succeeded(); let (response, code) = index.filtered_batches(&["indexCreation"], &[], &[]).await; - assert_eq!(code, 200, "{}", response); + assert_eq!(code, 200, "{response}"); assert_eq!(response["results"].as_array().unwrap().len(), 1); let (response, code) = - index.filtered_batches(&["indexCreation", "IndexDeletion"], &[], &[]).await; - assert_eq!(code, 200, "{}", response); + index.filtered_batches(&["indexCreation", "indexDeletion"], &[], &[]).await; + assert_eq!(code, 200, "{response}"); assert_eq!(response["results"].as_array().unwrap().len(), 2); let (response, code) = index.filtered_batches(&["indexCreation"], &[], &[]).await; - assert_eq!(code, 200, "{}", response); + assert_eq!(code, 200, "{response}"); assert_eq!(response["results"].as_array().unwrap().len(), 1); } @@ -189,7 +189,7 @@ async fn list_batches_invalid_canceled_by_filter() { index.wait_task(task.uid()).await.succeeded(); let (response, code) = index.filtered_batches(&[], &[], &["0"]).await; - assert_eq!(code, 200, "{}", response); + assert_eq!(code, 200, "{response}"); assert_eq!(response["results"].as_array().unwrap().len(), 0); } @@ -203,7 +203,7 @@ async fn list_batches_status_and_type_filtered() { index.wait_task(task.uid()).await.succeeded(); let (response, code) = index.filtered_batches(&["indexCreation"], &["failed"], &[]).await; - assert_eq!(code, 200, "{}", response); + assert_eq!(code, 200, "{response}"); assert_eq!(response["results"].as_array().unwrap().len(), 0); let (response, code) = index @@ -213,7 +213,7 @@ async fn list_batches_status_and_type_filtered() { &[], ) .await; - assert_eq!(code, 200, "{}", response); + assert_eq!(code, 200, "{response}"); assert_eq!(response["results"].as_array().unwrap().len(), 2); } @@ -222,7 +222,7 @@ async fn list_batch_filter_error() { let server = Server::new_shared(); let (response, code) = server.batches_filter("lol=pied").await; - assert_eq!(code, 400, "{}", response); + assert_eq!(code, 400, "{response}"); meili_snap::snapshot!(meili_snap::json_string!(response), @r#" { "message": "Unknown parameter `lol`: expected one of `limit`, `from`, `reverse`, `batchUids`, `uids`, `canceledBy`, `types`, `statuses`, `indexUids`, `afterEnqueuedAt`, `beforeEnqueuedAt`, `afterStartedAt`, `beforeStartedAt`, `afterFinishedAt`, `beforeFinishedAt`", @@ -233,7 +233,7 @@ async fn list_batch_filter_error() { "#); let (response, code) = server.batches_filter("uids=pied").await; - assert_eq!(code, 400, "{}", response); + assert_eq!(code, 400, "{response}"); meili_snap::snapshot!(meili_snap::json_string!(response), @r#" { "message": "Invalid value in parameter `uids`: could not parse `pied` as a positive integer", @@ -244,7 +244,7 @@ async fn list_batch_filter_error() { "#); let (response, code) = server.batches_filter("from=pied").await; - assert_eq!(code, 400, "{}", response); + assert_eq!(code, 400, "{response}"); meili_snap::snapshot!(meili_snap::json_string!(response), @r#" { "message": "Invalid value in parameter `from`: could not parse `pied` as a positive integer", @@ -255,7 +255,7 @@ async fn list_batch_filter_error() { "#); let (response, code) = server.batches_filter("beforeStartedAt=pied").await; - assert_eq!(code, 400, "{}", response); + assert_eq!(code, 400, "{response}"); meili_snap::snapshot!(meili_snap::json_string!(response), @r#" { "message": "Invalid value in parameter `beforeStartedAt`: `pied` is an invalid date-time. It should follow the YYYY-MM-DD or RFC 3339 date-time format.", @@ -283,7 +283,8 @@ async fn test_summarized_document_addition_or_update() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]" + ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", + ".stats.indexUids" => r#"""{"test": 1}"""# }, @r###" { @@ -301,9 +302,7 @@ async fn test_summarized_document_addition_or_update() { "types": { "documentAdditionOrUpdate": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": {"test": 1}, "progressTrace": "[progressTrace]", "writeChannelCongestion": "[writeChannelCongestion]", "internalDatabaseSizes": "[internalDatabaseSizes]" @@ -376,7 +375,7 @@ async fn test_summarized_delete_documents_by_batch() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.indexUids" => "{\n\t\"test\": 1}" + ".stats.indexUids" => r#"""{"test": 1}"""# }, @r###" { @@ -394,9 +393,7 @@ async fn test_summarized_delete_documents_by_batch() { "types": { "documentDeletion": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": {"test": 1}, "progressTrace": "[progressTrace]" }, "duration": "[duration]", @@ -467,7 +464,8 @@ async fn test_summarized_delete_documents_by_filter() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.indexUids" => "{\n\t\"test\": 1}" + ".stats.indexUids" => r#"""{"test": 1}"""#, + ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid`" }, @r###" { @@ -486,15 +484,13 @@ async fn test_summarized_delete_documents_by_filter() { "types": { "documentDeletion": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": {"test": 1}, "progressTrace": "[progressTrace]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks for index `[uuid`" } "###); @@ -603,7 +599,7 @@ async fn test_summarized_delete_document_by_id() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.indexUids" => "{\n\t\"test\": 1}" + ".stats.indexUids" => r#"""{"test": 1}"""# }, @r###" { @@ -621,9 +617,7 @@ async fn test_summarized_delete_document_by_id() { "types": { "documentDeletion": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": {"test": 1}, "progressTrace": "[progressTrace]" }, "duration": "[duration]", @@ -705,7 +699,7 @@ async fn test_summarized_settings_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", - ".stats.indexUids" => "{\n\t\"test\": 1}", + ".stats.indexUids" => r#"""{"test": 1}"""#, ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @@ -734,9 +728,7 @@ async fn test_summarized_settings_update() { "types": { "settingsUpdate": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": {"test": 1}, "progressTrace": "[progressTrace]" }, "duration": "[duration]", @@ -763,7 +755,7 @@ async fn test_summarized_index_creation() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.indexUids" => "{\n\t\"test\": 1}", + ".stats.indexUids" => r#"""{"test": 1}"""#, ".batchCreationComplete" => "task with id X of type `indexCreation` cannot be batched" }, @r###" @@ -779,9 +771,7 @@ async fn test_summarized_index_creation() { "types": { "indexCreation": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": {"test": 1}, "progressTrace": "[progressTrace]" }, "duration": "[duration]", @@ -896,7 +886,7 @@ async fn test_summarized_index_deletion() { { "uid": "[uid]", "batchUid": "[batch_uid]", - "indexUid": "test", + "indexUid": "[uuid]", "status": "succeeded", "type": "indexDeletion", "canceledBy": null, @@ -919,7 +909,7 @@ async fn test_summarized_index_deletion() { { "uid": "[uid]", "batchUid": "[batch_uid]", - "indexUid": "test", + "indexUid": "[uuid]", "status": "failed", "type": "indexDeletion", "canceledBy": null, @@ -927,7 +917,7 @@ async fn test_summarized_index_deletion() { "deletedDocuments": 0 }, "error": { - "message": "Index `test` not found.", + "message": "Index `[uuid]` not found.", "code": "index_not_found", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#index_not_found" @@ -947,7 +937,7 @@ async fn test_summarized_index_update() { // If the index doesn't exist yet, we should get errors with or without the primary key. let (task, _status_code) = index.update(None).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -957,7 +947,7 @@ async fn test_summarized_index_update() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.indexUids" => "{\n\t\"test\": 1}", + ".stats.indexUids" => r#"{"test": 1}"#, ".batchCreationComplete" => "task with id X of type `indexUpdate` cannot be batched" }, @r###" @@ -973,9 +963,7 @@ async fn test_summarized_index_update() { "types": { "indexUpdate": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": {"test": 1}, "progressTrace": "[progressTrace]" }, "duration": "[duration]", @@ -1177,7 +1165,7 @@ async fn test_summarized_index_swap() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.indexUids" => "{\n\t\"doggos\": 1}" + ".stats.indexUids" => r#"""{"doggos": 1}"""# }, @r###" { @@ -1192,9 +1180,7 @@ async fn test_summarized_index_swap() { "types": { "indexCreation": 1 }, - "indexUids": { - "doggos": 1 - }, + "indexUids": {"doggos": 1}, "progressTrace": "[progressTrace]" }, "duration": "[duration]", @@ -1214,19 +1200,21 @@ async fn test_summarized_batch_cancelation() { index.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = server.cancel_tasks(format!("uids={}", task.uid()).as_str()).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(1).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".batchCreationComplete" => "task with id X of type `taskCancelation` cannot be batched" }, @r###" { - "uid": 1, + "uid": "[uid]", "progress": null, "details": { "matchedTasks": 1, @@ -1247,7 +1235,7 @@ async fn test_summarized_batch_cancelation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id 1 of type `taskCancelation` cannot be batched" + "batchCreationComplete": "task with id X of type `taskCancelation` cannot be batched" } "###); } @@ -1316,7 +1304,8 @@ async fn test_summarized_dump_creation() { ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".batchCreationComplete" => "task with id X of type `dumpCreation` cannot be batched" }, @r###" { @@ -1339,7 +1328,7 @@ async fn test_summarized_dump_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id 0 of type `dumpCreation` cannot be batched" + "batchCreationComplete": "task with id X of type `dumpCreation` cannot be batched" } "###); } From 48460678dfa2a62b4fe8676a0dc0474da90f1d74 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 3 Jun 2025 10:50:22 +0300 Subject: [PATCH 003/312] More assertion fixes Signed-off-by: Martin Tzvetanov Grigorov --- crates/meilisearch/tests/batches/mod.rs | 195 +++++++++++++----------- 1 file changed, 105 insertions(+), 90 deletions(-) diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index bb926af70..ce5ad41b6 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -12,10 +12,10 @@ async fn error_get_unexisting_batch_status() { let index = server.unique_index(); let (task, _coder) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); - let (response, code) = index.get_batch(task.uid() as u32).await; + let (response, code) = index.get_batch(u32::MAX).await; let expected_response = json!({ - "message": format!("Batch `{}` not found.", task.uid()), + "message": format!("Batch `{}` not found.", u32::MAX), "code": "batch_not_found", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#batch_not_found" @@ -31,7 +31,7 @@ async fn get_batch_status() { let index = server.unique_index(); let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); - let (_response, code) = index.get_batch(0).await; + let (_response, code) = index.get_batch(task.uid() as u32).await; assert_eq!(code, 200); } @@ -284,7 +284,8 @@ async fn test_summarized_document_addition_or_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", - ".stats.indexUids" => r#"""{"test": 1}"""# + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { @@ -302,7 +303,7 @@ async fn test_summarized_document_addition_or_update() { "types": { "documentAdditionOrUpdate": 1 }, - "indexUids": {"test": 1}, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]", "writeChannelCongestion": "[writeChannelCongestion]", "internalDatabaseSizes": "[internalDatabaseSizes]" @@ -310,14 +311,14 @@ async fn test_summarized_document_addition_or_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" } "###); let (task, _status_code) = index.add_documents(json!({ "id": 42, "content": "doggos & fluff" }), Some("id")).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(1).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -327,7 +328,9 @@ async fn test_summarized_document_addition_or_update() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]" + ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { @@ -345,16 +348,14 @@ async fn test_summarized_document_addition_or_update() { "types": { "documentAdditionOrUpdate": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]", "writeChannelCongestion": "[writeChannelCongestion]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" } "###); } @@ -363,7 +364,10 @@ async fn test_summarized_document_addition_or_update() { async fn test_summarized_delete_documents_by_batch() { let server = Server::new_shared(); let index = server.unique_index(); - let (task, _status_code) = index.delete_batch(vec![1, 2, 3]).await; + let task_uid_1 = (u32::MAX - 1) as u64; + let task_uid_2 = (u32::MAX - 2) as u64; + let task_uid_3 = (u32::MAX - 3) as u64; + let (task, _status_code) = index.delete_batch(vec![task_uid_1, task_uid_2, task_uid_3]).await; index.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, @@ -375,7 +379,8 @@ async fn test_summarized_delete_documents_by_batch() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.indexUids" => r#"""{"test": 1}"""# + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { @@ -393,33 +398,35 @@ async fn test_summarized_delete_documents_by_batch() { "types": { "documentDeletion": 1 }, - "indexUids": {"test": 1}, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" } "###); index.create(None).await; let (task, _status_code) = index.delete_batch(vec![42]).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(2).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]" + ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", + ".stats.indexUids" => r#"{"[uuid]": 1}"#, }, @r###" { - "uid": 2, + "uid": "[uid]", "progress": null, "details": { "providedIds": 1, @@ -433,9 +440,7 @@ async fn test_summarized_delete_documents_by_batch() { "types": { "documentDeletion": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", @@ -464,8 +469,8 @@ async fn test_summarized_delete_documents_by_filter() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.indexUids" => r#"""{"test": 1}"""#, - ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid`" + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { @@ -484,13 +489,13 @@ async fn test_summarized_delete_documents_by_filter() { "types": { "documentDeletion": 1 }, - "indexUids": {"test": 1}, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks for index `[uuid`" + "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" } "###); @@ -498,20 +503,23 @@ async fn test_summarized_delete_documents_by_filter() { let (task, _status_code) = index.delete_document_by_filter(json!({ "filter": "doggo = bernese" })).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(2).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]" + ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { - "uid": 2, + "uid": "[uid]", "progress": null, "details": { "providedIds": 0, @@ -526,15 +534,13 @@ async fn test_summarized_delete_documents_by_filter() { "types": { "documentDeletion": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" } "###); @@ -542,20 +548,23 @@ async fn test_summarized_delete_documents_by_filter() { let (task, _status_code) = index.delete_document_by_filter(json!({ "filter": "doggo = bernese" })).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(4).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]" + ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { - "uid": 4, + "uid": "[uid]", "progress": null, "details": { "providedIds": 0, @@ -570,15 +579,13 @@ async fn test_summarized_delete_documents_by_filter() { "types": { "documentDeletion": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" } "###); } @@ -599,7 +606,8 @@ async fn test_summarized_delete_document_by_id() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.indexUids" => r#"""{"test": 1}"""# + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { @@ -617,33 +625,36 @@ async fn test_summarized_delete_document_by_id() { "types": { "documentDeletion": 1 }, - "indexUids": {"test": 1}, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" } "###); index.create(None).await; let (task, _status_code) = index.delete_document(42).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(2).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]" + ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { - "uid": 2, + "uid": "[uid]", "progress": null, "details": { "providedIds": 1, @@ -657,15 +668,13 @@ async fn test_summarized_delete_document_by_id() { "types": { "documentDeletion": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" } "###); } @@ -699,7 +708,7 @@ async fn test_summarized_settings_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", - ".stats.indexUids" => r#"""{"test": 1}"""#, + ".stats.indexUids" => r#"{"[uuid]": 1}"#, ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @@ -728,7 +737,7 @@ async fn test_summarized_settings_update() { "types": { "settingsUpdate": 1 }, - "indexUids": {"test": 1}, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", @@ -755,7 +764,7 @@ async fn test_summarized_index_creation() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.indexUids" => r#"""{"test": 1}"""#, + ".stats.indexUids" => r#"{"[uuid]": 1}"#, ".batchCreationComplete" => "task with id X of type `indexCreation` cannot be batched" }, @r###" @@ -771,7 +780,7 @@ async fn test_summarized_index_creation() { "types": { "indexCreation": 1 }, - "indexUids": {"test": 1}, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", @@ -783,19 +792,22 @@ async fn test_summarized_index_creation() { let (task, _status_code) = index.create(Some("doggos")).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(1).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "task with id X of type `indexCreation` cannot be batched" }, @r###" { - "uid": 1, + "uid": "[uid]", "progress": null, "details": { "primaryKey": "doggos" @@ -808,15 +820,13 @@ async fn test_summarized_index_creation() { "types": { "indexCreation": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id 1 of type `indexCreation` cannot be batched" + "batchCreationComplete": "task with id X of type `indexCreation` cannot be batched" } "###); } @@ -947,7 +957,7 @@ async fn test_summarized_index_update() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.indexUids" => r#"{"test": 1}"#, + ".stats.indexUids" => r#"{"[uuid]": 1}"#, ".batchCreationComplete" => "task with id X of type `indexUpdate` cannot be batched" }, @r###" @@ -963,7 +973,7 @@ async fn test_summarized_index_update() { "types": { "indexUpdate": 1 }, - "indexUids": {"test": 1}, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", @@ -975,19 +985,22 @@ async fn test_summarized_index_update() { let (task, _status_code) = index.update(Some("bones")).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(1).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "task with id X of type `indexUpdate` cannot be batched" }, @r###" { - "uid": 1, + "uid": "[uid]", "progress": null, "details": { "primaryKey": "bones" @@ -1000,15 +1013,13 @@ async fn test_summarized_index_update() { "types": { "indexUpdate": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id 1 of type `indexUpdate` cannot be batched" + "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" } "###); @@ -1017,19 +1028,22 @@ async fn test_summarized_index_update() { let (task, _status_code) = index.update(None).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(3).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "task with id X of type `indexUpdate` cannot be batched" }, @r###" { - "uid": 3, + "uid": "[uid]", "progress": null, "details": {}, "stats": { @@ -1040,33 +1054,34 @@ async fn test_summarized_index_update() { "types": { "indexUpdate": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id 3 of type `indexUpdate` cannot be batched" + "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" } "###); let (task, _status_code) = index.update(Some("bones")).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(4).await; + let (batch, _) = index.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { + ".uid" => "[uid]", ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", - ".stats.writeChannelCongestion" => "[writeChannelCongestion]" + ".stats.writeChannelCongestion" => "[writeChannelCongestion]", + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "task with id X of type `indexUpdate` cannot be batched" }, @r###" { - "uid": 4, + "uid": "[uid]", "progress": null, "details": { "primaryKey": "bones" @@ -1079,15 +1094,13 @@ async fn test_summarized_index_update() { "types": { "indexUpdate": 1 }, - "indexUids": { - "test": 1 - }, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id 4 of type `indexUpdate` cannot be batched" + "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" } "###); } @@ -1155,7 +1168,7 @@ async fn test_summarized_index_swap() { ])) .await; server.wait_task(task.uid()).await.succeeded(); - let (batch, _) = server.get_batch(1).await; + let (batch, _) = server.get_batch(task.uid() as u32).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -1165,7 +1178,8 @@ async fn test_summarized_index_swap() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".stats.indexUids" => r#"""{"doggos": 1}"""# + ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "task with id X of type `indexCreation` cannot be batched" }, @r###" { @@ -1180,13 +1194,13 @@ async fn test_summarized_index_swap() { "types": { "indexCreation": 1 }, - "indexUids": {"doggos": 1}, + "indexUids": "{\"[uuid]\": 1}", "progressTrace": "[progressTrace]" }, "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id 1 of type `indexCreation` cannot be batched" + "batchCreationComplete": "task with id X of type `indexCreation` cannot be batched" } "###); } @@ -1210,7 +1224,8 @@ async fn test_summarized_batch_cancelation() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".batchCreationComplete" => "task with id X of type `taskCancelation` cannot be batched" + ".batchCreationComplete" => "task with id X of type `taskCancelation` cannot be batched", + ".details.originalFilter" => "?uids=X", }, @r###" { @@ -1219,7 +1234,7 @@ async fn test_summarized_batch_cancelation() { "details": { "matchedTasks": 1, "canceledTasks": 0, - "originalFilter": "?uids=0" + "originalFilter": "?uids=X" }, "stats": { "totalNbTasks": 1, From 2691999bd3d0f1929108ad6db769f1dc7e7b7e63 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 3 Jun 2025 11:15:27 +0300 Subject: [PATCH 004/312] Add a helper method for getting the latest batch Signed-off-by: Martin Tzvetanov Grigorov --- crates/meilisearch/tests/batches/mod.rs | 2 +- crates/meilisearch/tests/common/server.rs | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index ce5ad41b6..82403fe3b 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -1309,7 +1309,7 @@ async fn test_summarized_dump_creation() { let server = Server::new_shared(); let (task, _status_code) = server.create_dump().await; server.wait_task(task.uid()).await.succeeded(); - let (batch, _) = server.get_batch(task.uid() as u32).await; + let (batch, _) = server.get_latest_batch().await; assert_json_snapshot!(batch, { ".uid" => "[uid]", diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 431972983..787cafc9f 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -429,6 +429,19 @@ impl Server { self.service.get(url).await } + // https://www.meilisearch.com/docs/reference/api/batches#get-batches states: + // "Batches are always returned in descending order of uid. This means that by default, + // the most recently created batch objects appear first." + pub async fn get_latest_batch(&self) -> (Option, StatusCode) { + let url = "/batches?limit=1&offset=0"; + let (value, code) = self.service.get(url).await; + value + .get("results") + .and_then(|results| results.as_array()) + .and_then(|array| array.first()) + .map_or((None, code), |latest| (Some(Value(latest.clone())), code)) + } + pub async fn get_features(&self) -> (Value, StatusCode) { self.service.get("/experimental-features").await } From 139ec8c7827c3694be24cb9090edc4b4056cd15c Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Tue, 3 Jun 2025 15:23:14 +0300 Subject: [PATCH 005/312] Add task.batch_uid() helper method Signed-off-by: Martin Tzvetanov Grigorov --- crates/meilisearch/tests/batches/mod.rs | 38 ++++++++++++------------- crates/meilisearch/tests/common/mod.rs | 9 ++++++ 2 files changed, 28 insertions(+), 19 deletions(-) diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index 82403fe3b..e6801f269 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -31,7 +31,7 @@ async fn get_batch_status() { let index = server.unique_index(); let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); - let (_response, code) = index.get_batch(task.uid() as u32).await; + let (_response, code) = index.get_batch(task.batch_uid()).await; assert_eq!(code, 200); } @@ -273,7 +273,7 @@ async fn test_summarized_document_addition_or_update() { let (task, _status_code) = index.add_documents(json!({ "id": 42, "content": "doggos & fluff" }), None).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -318,7 +318,7 @@ async fn test_summarized_document_addition_or_update() { let (task, _status_code) = index.add_documents(json!({ "id": 42, "content": "doggos & fluff" }), Some("id")).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -369,7 +369,7 @@ async fn test_summarized_delete_documents_by_batch() { let task_uid_3 = (u32::MAX - 3) as u64; let (task, _status_code) = index.delete_batch(vec![task_uid_1, task_uid_2, task_uid_3]).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -411,7 +411,7 @@ async fn test_summarized_delete_documents_by_batch() { index.create(None).await; let (task, _status_code) = index.delete_batch(vec![42]).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -459,7 +459,7 @@ async fn test_summarized_delete_documents_by_filter() { let (task, _status_code) = index.delete_document_by_filter(json!({ "filter": "doggo = bernese" })).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -503,7 +503,7 @@ async fn test_summarized_delete_documents_by_filter() { let (task, _status_code) = index.delete_document_by_filter(json!({ "filter": "doggo = bernese" })).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -548,7 +548,7 @@ async fn test_summarized_delete_documents_by_filter() { let (task, _status_code) = index.delete_document_by_filter(json!({ "filter": "doggo = bernese" })).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -596,7 +596,7 @@ async fn test_summarized_delete_document_by_id() { let index = server.unique_index(); let (task, _status_code) = index.delete_document(1).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -638,7 +638,7 @@ async fn test_summarized_delete_document_by_id() { index.create(None).await; let (task, _status_code) = index.delete_document(42).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -697,7 +697,7 @@ async fn test_summarized_settings_update() { let (task,_status_code) = index.update_settings(json!({ "displayedAttributes": ["doggos", "name"], "filterableAttributes": ["age", "nb_paw_pads"], "sortableAttributes": ["iq"] })).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -754,7 +754,7 @@ async fn test_summarized_index_creation() { let index = server.unique_index(); let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -792,7 +792,7 @@ async fn test_summarized_index_creation() { let (task, _status_code) = index.create(Some("doggos")).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -947,7 +947,7 @@ async fn test_summarized_index_update() { // If the index doesn't exist yet, we should get errors with or without the primary key. let (task, _status_code) = index.update(None).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -985,7 +985,7 @@ async fn test_summarized_index_update() { let (task, _status_code) = index.update(Some("bones")).await; index.wait_task(task.uid()).await.failed(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -1028,7 +1028,7 @@ async fn test_summarized_index_update() { let (task, _status_code) = index.update(None).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -1066,7 +1066,7 @@ async fn test_summarized_index_update() { let (task, _status_code) = index.update(Some("bones")).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -1214,7 +1214,7 @@ async fn test_summarized_batch_cancelation() { index.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = server.cancel_tasks(format!("uids={}", task.uid()).as_str()).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -1264,7 +1264,7 @@ async fn test_summarized_batch_deletion() { index.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = server.delete_tasks(format!("uids={}", task.uid()).as_str()).await; index.wait_task(task.uid()).await.succeeded(); - let (batch, _) = index.get_batch(task.uid() as u32).await; + let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", diff --git a/crates/meilisearch/tests/common/mod.rs b/crates/meilisearch/tests/common/mod.rs index 373f89f78..d1da616ad 100644 --- a/crates/meilisearch/tests/common/mod.rs +++ b/crates/meilisearch/tests/common/mod.rs @@ -38,6 +38,15 @@ impl Value { self["uid"].as_u64().is_some() || self["taskUid"].as_u64().is_some() } + #[track_caller] + pub fn batch_uid(&self) -> u32 { + if let Some(batch_uid) = self["batchUid"].as_u64() { + batch_uid as u32 + } else { + panic!("Didn't find `batchUid` in: {self}"); + } + } + /// Return `true` if the `status` field is set to `succeeded`. /// Panic if the `status` field doesn't exists. #[track_caller] From 9e31d6ceff910fd8d6eba731fd665e69de4544c2 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 5 Jun 2025 11:11:54 +0300 Subject: [PATCH 006/312] Add batch_uid to all successful and failed tasks too Signed-off-by: Martin Tzvetanov Grigorov --- crates/index-scheduler/src/queue/tasks.rs | 7 ++++++- crates/meilisearch/tests/batches/mod.rs | 20 ++++++++++---------- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/crates/index-scheduler/src/queue/tasks.rs b/crates/index-scheduler/src/queue/tasks.rs index 74192232e..92789b93f 100644 --- a/crates/index-scheduler/src/queue/tasks.rs +++ b/crates/index-scheduler/src/queue/tasks.rs @@ -530,7 +530,12 @@ impl Queue { ..task } } else { - task + dbg!(&task); + if task.status == Status::Succeeded || task.status == Status::Failed { + Task { batch_uid: Some(batch.uid), ..task } + } else { + task + } } }) .collect(), diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index e6801f269..d5374a144 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -1,7 +1,7 @@ mod errors; use meili_snap::insta::assert_json_snapshot; -use meili_snap::snapshot; +use meili_snap::{json_string, snapshot}; use crate::common::Server; use crate::json; @@ -119,21 +119,21 @@ async fn list_batches_with_star_filters() { let (response, code) = index.service.get("/batches?types=*,documentAdditionOrUpdate&statuses=*").await; - assert_eq!(code, 200, "{:?}", response); + assert_eq!(code, 200, "{response:?}"); assert_eq!(response["results"].as_array().unwrap().len(), 2); let (response, code) = index .service .get("/batches?types=*,documentAdditionOrUpdate&statuses=*,failed&indexUids=test") .await; - assert_eq!(code, 200, "{:?}", response); + assert_eq!(code, 200, "{response:?}"); assert_eq!(response["results"].as_array().unwrap().len(), 2); let (response, code) = index .service .get("/batches?types=*,documentAdditionOrUpdate&statuses=*,failed&indexUids=test,*") .await; - assert_eq!(code, 200, "{:?}", response); + assert_eq!(code, 200, "{response:?}"); assert_eq!(response["results"].as_array().unwrap().len(), 2); } @@ -223,7 +223,7 @@ async fn list_batch_filter_error() { let (response, code) = server.batches_filter("lol=pied").await; assert_eq!(code, 400, "{response}"); - meili_snap::snapshot!(meili_snap::json_string!(response), @r#" + snapshot!(json_string!(response), @r#" { "message": "Unknown parameter `lol`: expected one of `limit`, `from`, `reverse`, `batchUids`, `uids`, `canceledBy`, `types`, `statuses`, `indexUids`, `afterEnqueuedAt`, `beforeEnqueuedAt`, `afterStartedAt`, `beforeStartedAt`, `afterFinishedAt`, `beforeFinishedAt`", "code": "bad_request", @@ -234,7 +234,7 @@ async fn list_batch_filter_error() { let (response, code) = server.batches_filter("uids=pied").await; assert_eq!(code, 400, "{response}"); - meili_snap::snapshot!(meili_snap::json_string!(response), @r#" + snapshot!(json_string!(response), @r#" { "message": "Invalid value in parameter `uids`: could not parse `pied` as a positive integer", "code": "invalid_task_uids", @@ -245,7 +245,7 @@ async fn list_batch_filter_error() { let (response, code) = server.batches_filter("from=pied").await; assert_eq!(code, 400, "{response}"); - meili_snap::snapshot!(meili_snap::json_string!(response), @r#" + snapshot!(json_string!(response), @r#" { "message": "Invalid value in parameter `from`: could not parse `pied` as a positive integer", "code": "invalid_task_from", @@ -256,7 +256,7 @@ async fn list_batch_filter_error() { let (response, code) = server.batches_filter("beforeStartedAt=pied").await; assert_eq!(code, 400, "{response}"); - meili_snap::snapshot!(meili_snap::json_string!(response), @r#" + snapshot!(json_string!(response), @r#" { "message": "Invalid value in parameter `beforeStartedAt`: `pied` is an invalid date-time. It should follow the YYYY-MM-DD or RFC 3339 date-time format.", "code": "invalid_task_before_started_at", @@ -685,8 +685,8 @@ async fn test_summarized_settings_update() { let index = server.unique_index(); // here we should find my payload even in the failed batch. let (response, code) = index.update_settings(json!({ "rankingRules": ["custom"] })).await; - meili_snap::snapshot!(code, @"400 Bad Request"); - meili_snap::snapshot!(meili_snap::json_string!(response), @r###" + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(response), @r###" { "message": "Invalid value at `.rankingRules[0]`: `custom` ranking rule is invalid. Valid ranking rules are words, typo, sort, proximity, attribute, exactness and custom ranking rules.", "code": "invalid_settings_ranking_rules", From 00eb258a538ed39220c3dbb469f5c193f160cade Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 11:16:07 +0200 Subject: [PATCH 007/312] Fix comment --- crates/meilisearch-auth/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch-auth/src/lib.rs b/crates/meilisearch-auth/src/lib.rs index 27d163192..02d9201c5 100644 --- a/crates/meilisearch-auth/src/lib.rs +++ b/crates/meilisearch-auth/src/lib.rs @@ -158,7 +158,7 @@ impl AuthController { self.store.delete_all_keys() } - /// Delete all the keys in the DB. + /// Insert a key directly into the store. pub fn raw_insert_key(&mut self, key: Key) -> Result<()> { self.store.put_api_key(key)?; Ok(()) @@ -353,6 +353,7 @@ fn generate_default_keys(store: &HeedAuthStore) -> Result<()> { store.put_api_key(Key::default_chat())?; store.put_api_key(Key::default_admin())?; store.put_api_key(Key::default_search())?; + store.put_api_key(Key::default_management())?; Ok(()) } From b421c8e7deb13ca5bcfbf03ef33f958ffb8dbf32 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 11:29:16 +0200 Subject: [PATCH 008/312] Add an AllRead key --- crates/meilisearch-auth/src/store.rs | 1 + crates/meilisearch-types/src/keys.rs | 48 ++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/crates/meilisearch-auth/src/store.rs b/crates/meilisearch-auth/src/store.rs index bae27afe4..6e4ff8389 100644 --- a/crates/meilisearch-auth/src/store.rs +++ b/crates/meilisearch-auth/src/store.rs @@ -89,6 +89,7 @@ impl HeedAuthStore { for action in &key.actions { match action { Action::All => actions.extend(enum_iterator::all::()), + Action::AllRead => actions.extend(enum_iterator::all::().filter(|a| a.is_read())), Action::DocumentsAll => { actions.extend( [Action::DocumentsGet, Action::DocumentsDelete, Action::DocumentsAdd] diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index df2810727..023e7e786 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -218,6 +218,9 @@ pub enum Action { #[serde(rename = "*")] #[deserr(rename = "*")] All = 0, + #[serde(rename = "*.read")] + #[deserr(rename = "*.read")] + AllRead, #[serde(rename = "search")] #[deserr(rename = "search")] Search, @@ -396,6 +399,51 @@ impl Action { } } + /// Whether the action should be included in [Action::AllRead]. + pub fn is_read(&self) -> bool { + use Action::*; + + // It's using an exhaustive match to force the addition of new actions. + match self { + // Any action that expands to others must return false, as it wouldn't be able to expand recursively. + All | AllRead | DocumentsAll | IndexesAll | ChatsAll | TasksAll | SettingsAll + | StatsAll | MetricsAll | DumpsAll | SnapshotsAll | ChatsSettingsAll => false, + + Search => true, + DocumentsAdd => false, + DocumentsGet => true, + DocumentsDelete => false, + IndexesAdd => false, + IndexesGet => true, + IndexesUpdate => false, + IndexesDelete => false, + IndexesSwap => false, + TasksCancel => false, + TasksDelete => false, + TasksGet => true, + SettingsGet => true, + SettingsUpdate => false, + StatsGet => true, + MetricsGet => true, + DumpsCreate => false, + SnapshotsCreate => false, + Version => true, + KeysAdd => false, + KeysGet => false, // Prevent privilege escalation by not allowing reading other keys. + KeysUpdate => false, + KeysDelete => false, + ExperimentalFeaturesGet => true, + ExperimentalFeaturesUpdate => false, + NetworkGet => true, + NetworkUpdate => false, + ChatCompletions => false, // Disabled because it might trigger generation of new chats. + ChatsGet => true, + ChatsDelete => false, + ChatsSettingsGet => true, + ChatsSettingsUpdate => false, + } + } + pub const fn repr(&self) -> u8 { *self as u8 } From 032b34c37716e6fd11981c7046d8a824a5e826a7 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 11:29:32 +0200 Subject: [PATCH 009/312] Add a default management key --- crates/meilisearch-types/src/keys.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 023e7e786..e8db4014d 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -144,6 +144,21 @@ impl Key { } } + pub fn default_management() -> Self { + let now = OffsetDateTime::now_utc(); + let uid = Uuid::new_v4(); + Self { + name: Some("Default Management API Key".to_string()), + description: Some("Use it to peek into the instance in a read-only mode. Caution! Do not expose it on a public frontend".to_string()), + uid, + actions: vec![Action::AllRead], + indexes: vec![IndexUidPattern::all()], + expires_at: None, + created_at: now, + updated_at: now, + } + } + pub fn default_search() -> Self { let now = OffsetDateTime::now_utc(); let uid = Uuid::new_v4(); @@ -453,6 +468,7 @@ pub mod actions { use super::Action::*; pub(crate) const ALL: u8 = All.repr(); + pub const ALL_READ: u8 = AllRead.repr(); pub const SEARCH: u8 = Search.repr(); pub const DOCUMENTS_ALL: u8 = DocumentsAll.repr(); pub const DOCUMENTS_ADD: u8 = DocumentsAdd.repr(); From 11fedea788115448f6c6f7a854b340d65a1fd641 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 11:42:45 +0200 Subject: [PATCH 010/312] Set static uuids to keys --- crates/meilisearch-types/src/keys.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index e8db4014d..4a3b58c20 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -131,7 +131,7 @@ pub struct Key { impl Key { pub fn default_admin() -> Self { let now = OffsetDateTime::now_utc(); - let uid = Uuid::new_v4(); + let uid = Uuid::from_u128(0); Self { name: Some("Default Admin API Key".to_string()), description: Some("Use it for anything that is not a search operation. Caution! Do not expose it on a public frontend".to_string()), @@ -146,9 +146,9 @@ impl Key { pub fn default_management() -> Self { let now = OffsetDateTime::now_utc(); - let uid = Uuid::new_v4(); + let uid = Uuid::from_u128(1); Self { - name: Some("Default Management API Key".to_string()), + name: Some("Read-only Admin key".to_string()), description: Some("Use it to peek into the instance in a read-only mode. Caution! Do not expose it on a public frontend".to_string()), uid, actions: vec![Action::AllRead], @@ -161,7 +161,7 @@ impl Key { pub fn default_search() -> Self { let now = OffsetDateTime::now_utc(); - let uid = Uuid::new_v4(); + let uid = Uuid::from_u128(2); Self { name: Some("Default Search API Key".to_string()), description: Some("Use it to search from the frontend".to_string()), @@ -176,7 +176,7 @@ impl Key { pub fn default_chat() -> Self { let now = OffsetDateTime::now_utc(); - let uid = Uuid::new_v4(); + let uid = Uuid::from_u128(3); Self { name: Some("Default Chat API Key".to_string()), description: Some("Use it to chat and search from the frontend".to_string()), From f50e586a4f265a06295d3cbd364fa9c8f353002b Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 11:52:58 +0200 Subject: [PATCH 011/312] Allow management key to read other keys --- crates/meilisearch-types/src/keys.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 4a3b58c20..bfedf1e99 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -148,8 +148,8 @@ impl Key { let now = OffsetDateTime::now_utc(); let uid = Uuid::from_u128(1); Self { - name: Some("Read-only Admin key".to_string()), - description: Some("Use it to peek into the instance in a read-only mode. Caution! Do not expose it on a public frontend".to_string()), + name: Some("Default Read-Only Admin API Key".to_string()), + description: Some("Use it to peek into the instance in a read-only mode. Caution! Do not expose it on a public frontend. It would give access to all other keys".to_string()), uid, actions: vec![Action::AllRead], indexes: vec![IndexUidPattern::all()], @@ -444,7 +444,7 @@ impl Action { SnapshotsCreate => false, Version => true, KeysAdd => false, - KeysGet => false, // Prevent privilege escalation by not allowing reading other keys. + KeysGet => true, KeysUpdate => false, KeysDelete => false, ExperimentalFeaturesGet => true, From b6b7ede266af7221d9271c86ba08d4b9e5d2c0da Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 11:53:42 +0200 Subject: [PATCH 012/312] Rename Action `*.read` to `*.get` --- crates/meilisearch-types/src/keys.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index bfedf1e99..c3269fb44 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -233,8 +233,8 @@ pub enum Action { #[serde(rename = "*")] #[deserr(rename = "*")] All = 0, - #[serde(rename = "*.read")] - #[deserr(rename = "*.read")] + #[serde(rename = "*.get")] + #[deserr(rename = "*.get")] AllRead, #[serde(rename = "search")] #[deserr(rename = "search")] From 9e1cb792f4940ee6e5897192c8925dc0dab2c344 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 11:55:25 +0200 Subject: [PATCH 013/312] Rename Action::AllRead to AllGet --- crates/meilisearch-auth/src/store.rs | 2 +- crates/meilisearch-types/src/keys.rs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch-auth/src/store.rs b/crates/meilisearch-auth/src/store.rs index 6e4ff8389..9c0ac4b00 100644 --- a/crates/meilisearch-auth/src/store.rs +++ b/crates/meilisearch-auth/src/store.rs @@ -89,7 +89,7 @@ impl HeedAuthStore { for action in &key.actions { match action { Action::All => actions.extend(enum_iterator::all::()), - Action::AllRead => actions.extend(enum_iterator::all::().filter(|a| a.is_read())), + Action::AllGet => actions.extend(enum_iterator::all::().filter(|a| a.is_read())), Action::DocumentsAll => { actions.extend( [Action::DocumentsGet, Action::DocumentsDelete, Action::DocumentsAdd] diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index c3269fb44..b9a9ae21c 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -151,7 +151,7 @@ impl Key { name: Some("Default Read-Only Admin API Key".to_string()), description: Some("Use it to peek into the instance in a read-only mode. Caution! Do not expose it on a public frontend. It would give access to all other keys".to_string()), uid, - actions: vec![Action::AllRead], + actions: vec![Action::AllGet], indexes: vec![IndexUidPattern::all()], expires_at: None, created_at: now, @@ -235,7 +235,7 @@ pub enum Action { All = 0, #[serde(rename = "*.get")] #[deserr(rename = "*.get")] - AllRead, + AllGet, #[serde(rename = "search")] #[deserr(rename = "search")] Search, @@ -421,7 +421,7 @@ impl Action { // It's using an exhaustive match to force the addition of new actions. match self { // Any action that expands to others must return false, as it wouldn't be able to expand recursively. - All | AllRead | DocumentsAll | IndexesAll | ChatsAll | TasksAll | SettingsAll + All | AllGet | DocumentsAll | IndexesAll | ChatsAll | TasksAll | SettingsAll | StatsAll | MetricsAll | DumpsAll | SnapshotsAll | ChatsSettingsAll => false, Search => true, @@ -468,7 +468,7 @@ pub mod actions { use super::Action::*; pub(crate) const ALL: u8 = All.repr(); - pub const ALL_READ: u8 = AllRead.repr(); + pub const ALL_READ: u8 = AllGet.repr(); pub const SEARCH: u8 = Search.repr(); pub const DOCUMENTS_ALL: u8 = DocumentsAll.repr(); pub const DOCUMENTS_ADD: u8 = DocumentsAdd.repr(); From 5081d837ea54dcba76038bb96afaa73f7278c189 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 12:12:30 +0200 Subject: [PATCH 014/312] Fix AllGet action being included in All --- crates/meilisearch-auth/src/store.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch-auth/src/store.rs b/crates/meilisearch-auth/src/store.rs index 9c0ac4b00..bec5d3561 100644 --- a/crates/meilisearch-auth/src/store.rs +++ b/crates/meilisearch-auth/src/store.rs @@ -88,7 +88,10 @@ impl HeedAuthStore { let mut actions = HashSet::new(); for action in &key.actions { match action { - Action::All => actions.extend(enum_iterator::all::()), + Action::All => { + actions.extend(enum_iterator::all::()); + actions.remove(&Action::AllGet); + }, Action::AllGet => actions.extend(enum_iterator::all::().filter(|a| a.is_read())), Action::DocumentsAll => { actions.extend( From 99732f4084475b4955173c3f3c2b96e405b8327a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 13:04:55 +0200 Subject: [PATCH 015/312] Fix some tests --- crates/meilisearch-auth/src/lib.rs | 2 +- crates/meilisearch/tests/auth/api_keys.rs | 6 +++--- crates/meilisearch/tests/auth/errors.rs | 6 +++--- crates/meilisearch/tests/common/server.rs | 1 + 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch-auth/src/lib.rs b/crates/meilisearch-auth/src/lib.rs index 02d9201c5..000e574ac 100644 --- a/crates/meilisearch-auth/src/lib.rs +++ b/crates/meilisearch-auth/src/lib.rs @@ -351,9 +351,9 @@ pub struct IndexSearchRules { fn generate_default_keys(store: &HeedAuthStore) -> Result<()> { store.put_api_key(Key::default_chat())?; + store.put_api_key(Key::default_management())?; store.put_api_key(Key::default_admin())?; store.put_api_key(Key::default_search())?; - store.put_api_key(Key::default_management())?; Ok(()) } diff --git a/crates/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs index 5a18b4dbf..63eb0d21c 100644 --- a/crates/meilisearch/tests/auth/api_keys.rs +++ b/crates/meilisearch/tests/auth/api_keys.rs @@ -419,14 +419,14 @@ async fn error_add_api_key_invalid_parameters_actions() { let (response, code) = server.add_api_key(content).await; meili_snap::snapshot!(code, @"400 Bad Request"); - meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r###" + meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r#" { - "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", + "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `*.get`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" } - "###); + "#); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/auth/errors.rs b/crates/meilisearch/tests/auth/errors.rs index ebe2e53fa..c9fa2ee9c 100644 --- a/crates/meilisearch/tests/auth/errors.rs +++ b/crates/meilisearch/tests/auth/errors.rs @@ -91,14 +91,14 @@ async fn create_api_key_bad_actions() { // can't parse let (response, code) = server.add_api_key(json!({ "actions": ["doggo"] })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", + "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `*.get`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" } - "###); + "#); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 1f5688a02..19a082cf3 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -97,6 +97,7 @@ impl Server { self.use_api_key(master_key); let (response, code) = self.list_api_keys("").await; assert_eq!(200, code, "{:?}", response); + // TODO: relying on the order of keys is not ideal, we should use the static uuid let admin_key = &response["results"][1]["key"]; self.use_api_key(admin_key.as_str().unwrap()); } From 67f2a30d7c9b0b8331912b1ec7471a744e04fb64 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 13:10:08 +0200 Subject: [PATCH 016/312] Fix test --- crates/meilisearch/tests/auth/api_keys.rs | 27 ++++++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs index 63eb0d21c..15edd8f3a 100644 --- a/crates/meilisearch/tests/auth/api_keys.rs +++ b/crates/meilisearch/tests/auth/api_keys.rs @@ -790,7 +790,7 @@ async fn list_api_keys() { meili_snap::snapshot!(code, @"201 Created"); let (response, code) = server.list_api_keys("").await; - meili_snap::snapshot!(meili_snap::json_string!(response, { ".results[].createdAt" => "[ignored]", ".results[].updatedAt" => "[ignored]", ".results[].uid" => "[ignored]", ".results[].key" => "[ignored]" }), @r###" + meili_snap::snapshot!(meili_snap::json_string!(response, { ".results[].createdAt" => "[ignored]", ".results[].updatedAt" => "[ignored]", ".results[0].uid" => "[ignored]", ".results[].key" => "[ignored]" }), @r#" { "results": [ { @@ -824,7 +824,7 @@ async fn list_api_keys() { "name": "Default Search API Key", "description": "Use it to search from the frontend", "key": "[ignored]", - "uid": "[ignored]", + "uid": "00000000-0000-0000-0000-000000000002", "actions": [ "search" ], @@ -839,7 +839,7 @@ async fn list_api_keys() { "name": "Default Admin API Key", "description": "Use it for anything that is not a search operation. Caution! Do not expose it on a public frontend", "key": "[ignored]", - "uid": "[ignored]", + "uid": "00000000-0000-0000-0000-000000000000", "actions": [ "*" ], @@ -850,11 +850,26 @@ async fn list_api_keys() { "createdAt": "[ignored]", "updatedAt": "[ignored]" }, + { + "name": "Default Read-Only Admin API Key", + "description": "Use it to peek into the instance in a read-only mode. Caution! Do not expose it on a public frontend. It would give access to all other keys", + "key": "[ignored]", + "uid": "00000000-0000-0000-0000-000000000001", + "actions": [ + "*.get" + ], + "indexes": [ + "*" + ], + "expiresAt": null, + "createdAt": "[ignored]", + "updatedAt": "[ignored]" + }, { "name": "Default Chat API Key", "description": "Use it to chat and search from the frontend", "key": "[ignored]", - "uid": "[ignored]", + "uid": "00000000-0000-0000-0000-000000000003", "actions": [ "chatCompletions", "search" @@ -869,9 +884,9 @@ async fn list_api_keys() { ], "offset": 0, "limit": 20, - "total": 4 + "total": 5 } - "###); + "#); meili_snap::snapshot!(code, @"200 OK"); } From 705e9a9e5e2464cf5f33758e22272f6ff8984de4 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 15:45:09 +0200 Subject: [PATCH 017/312] Make the uuids random again to prevent abuse using rainbow tables --- crates/meilisearch-types/src/keys.rs | 8 ++++---- crates/meilisearch/tests/auth/api_keys.rs | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index b9a9ae21c..3d30c7a0e 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -131,7 +131,7 @@ pub struct Key { impl Key { pub fn default_admin() -> Self { let now = OffsetDateTime::now_utc(); - let uid = Uuid::from_u128(0); + let uid = Uuid::new_v4(); Self { name: Some("Default Admin API Key".to_string()), description: Some("Use it for anything that is not a search operation. Caution! Do not expose it on a public frontend".to_string()), @@ -146,7 +146,7 @@ impl Key { pub fn default_management() -> Self { let now = OffsetDateTime::now_utc(); - let uid = Uuid::from_u128(1); + let uid = Uuid::new_v4(); Self { name: Some("Default Read-Only Admin API Key".to_string()), description: Some("Use it to peek into the instance in a read-only mode. Caution! Do not expose it on a public frontend. It would give access to all other keys".to_string()), @@ -161,7 +161,7 @@ impl Key { pub fn default_search() -> Self { let now = OffsetDateTime::now_utc(); - let uid = Uuid::from_u128(2); + let uid = Uuid::new_v4(); Self { name: Some("Default Search API Key".to_string()), description: Some("Use it to search from the frontend".to_string()), @@ -176,7 +176,7 @@ impl Key { pub fn default_chat() -> Self { let now = OffsetDateTime::now_utc(); - let uid = Uuid::from_u128(3); + let uid = Uuid::new_v4(); Self { name: Some("Default Chat API Key".to_string()), description: Some("Use it to chat and search from the frontend".to_string()), diff --git a/crates/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs index 15edd8f3a..fa09f17cb 100644 --- a/crates/meilisearch/tests/auth/api_keys.rs +++ b/crates/meilisearch/tests/auth/api_keys.rs @@ -790,7 +790,7 @@ async fn list_api_keys() { meili_snap::snapshot!(code, @"201 Created"); let (response, code) = server.list_api_keys("").await; - meili_snap::snapshot!(meili_snap::json_string!(response, { ".results[].createdAt" => "[ignored]", ".results[].updatedAt" => "[ignored]", ".results[0].uid" => "[ignored]", ".results[].key" => "[ignored]" }), @r#" + meili_snap::snapshot!(meili_snap::json_string!(response, { ".results[].createdAt" => "[ignored]", ".results[].updatedAt" => "[ignored]", ".results[].uid" => "[ignored]", ".results[].key" => "[ignored]" }), @r#" { "results": [ { @@ -824,7 +824,7 @@ async fn list_api_keys() { "name": "Default Search API Key", "description": "Use it to search from the frontend", "key": "[ignored]", - "uid": "00000000-0000-0000-0000-000000000002", + "uid": "[ignored]", "actions": [ "search" ], @@ -839,7 +839,7 @@ async fn list_api_keys() { "name": "Default Admin API Key", "description": "Use it for anything that is not a search operation. Caution! Do not expose it on a public frontend", "key": "[ignored]", - "uid": "00000000-0000-0000-0000-000000000000", + "uid": "[ignored]", "actions": [ "*" ], @@ -854,7 +854,7 @@ async fn list_api_keys() { "name": "Default Read-Only Admin API Key", "description": "Use it to peek into the instance in a read-only mode. Caution! Do not expose it on a public frontend. It would give access to all other keys", "key": "[ignored]", - "uid": "00000000-0000-0000-0000-000000000001", + "uid": "[ignored]", "actions": [ "*.get" ], @@ -869,7 +869,7 @@ async fn list_api_keys() { "name": "Default Chat API Key", "description": "Use it to chat and search from the frontend", "key": "[ignored]", - "uid": "00000000-0000-0000-0000-000000000003", + "uid": "[ignored]", "actions": [ "chatCompletions", "search" From ab768f379ff54ee66f9f3bdf4f643e92a04e8e92 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 15:49:34 +0200 Subject: [PATCH 018/312] Fix comment --- crates/meilisearch/tests/common/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 19a082cf3..671ed1ab6 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -97,7 +97,7 @@ impl Server { self.use_api_key(master_key); let (response, code) = self.list_api_keys("").await; assert_eq!(200, code, "{:?}", response); - // TODO: relying on the order of keys is not ideal, we should use the static uuid + // TODO: relying on the order of keys is not ideal, we should use the name instead let admin_key = &response["results"][1]["key"]; self.use_api_key(admin_key.as_str().unwrap()); } From 2d6dc83940fe394389e91a6e53e63f53e36b3a22 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 19 Jun 2025 15:55:12 +0200 Subject: [PATCH 019/312] Format the code --- crates/meilisearch-auth/src/store.rs | 6 ++++-- crates/meilisearch-types/src/keys.rs | 2 +- crates/meilisearch/tests/index/stats.rs | 1 - 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch-auth/src/store.rs b/crates/meilisearch-auth/src/store.rs index bec5d3561..eb2170f08 100644 --- a/crates/meilisearch-auth/src/store.rs +++ b/crates/meilisearch-auth/src/store.rs @@ -91,8 +91,10 @@ impl HeedAuthStore { Action::All => { actions.extend(enum_iterator::all::()); actions.remove(&Action::AllGet); - }, - Action::AllGet => actions.extend(enum_iterator::all::().filter(|a| a.is_read())), + } + Action::AllGet => { + actions.extend(enum_iterator::all::().filter(|a| a.is_read())) + } Action::DocumentsAll => { actions.extend( [Action::DocumentsGet, Action::DocumentsDelete, Action::DocumentsAdd] diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 3d30c7a0e..48f908a81 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -422,7 +422,7 @@ impl Action { match self { // Any action that expands to others must return false, as it wouldn't be able to expand recursively. All | AllGet | DocumentsAll | IndexesAll | ChatsAll | TasksAll | SettingsAll - | StatsAll | MetricsAll | DumpsAll | SnapshotsAll | ChatsSettingsAll => false, + | StatsAll | MetricsAll | DumpsAll | SnapshotsAll | ChatsSettingsAll => false, Search => true, DocumentsAdd => false, diff --git a/crates/meilisearch/tests/index/stats.rs b/crates/meilisearch/tests/index/stats.rs index 90c77cec8..6b2ba16ac 100644 --- a/crates/meilisearch/tests/index/stats.rs +++ b/crates/meilisearch/tests/index/stats.rs @@ -1,5 +1,4 @@ use crate::common::{shared_does_not_exists_index, Server}; - use crate::json; #[actix_rt::test] From c4a96b40eb050d85406d04ec4c7402e98140cde6 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 24 Jun 2025 17:40:06 +0200 Subject: [PATCH 020/312] Remove KeysGet from AllGet --- crates/meilisearch-types/src/keys.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 48f908a81..e4a0dd5d8 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -151,7 +151,7 @@ impl Key { name: Some("Default Read-Only Admin API Key".to_string()), description: Some("Use it to peek into the instance in a read-only mode. Caution! Do not expose it on a public frontend. It would give access to all other keys".to_string()), uid, - actions: vec![Action::AllGet], + actions: vec![Action::AllGet, Action::KeysGet], indexes: vec![IndexUidPattern::all()], expires_at: None, created_at: now, @@ -444,14 +444,14 @@ impl Action { SnapshotsCreate => false, Version => true, KeysAdd => false, - KeysGet => true, + KeysGet => false, // Disabled in order to prevent privilege escalation KeysUpdate => false, KeysDelete => false, ExperimentalFeaturesGet => true, ExperimentalFeaturesUpdate => false, NetworkGet => true, NetworkUpdate => false, - ChatCompletions => false, // Disabled because it might trigger generation of new chats. + ChatCompletions => false, // Disabled because it might trigger generation of new chats ChatsGet => true, ChatsDelete => false, ChatsSettingsGet => true, From 1c8f1c18f4fd1969613e75b80ca39c171f65cf3d Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 25 Jun 2025 09:59:34 +0200 Subject: [PATCH 021/312] Fix constant name and key description --- crates/meilisearch-types/src/keys.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index e4a0dd5d8..96b2e8ae1 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -149,7 +149,7 @@ impl Key { let uid = Uuid::new_v4(); Self { name: Some("Default Read-Only Admin API Key".to_string()), - description: Some("Use it to peek into the instance in a read-only mode. Caution! Do not expose it on a public frontend. It would give access to all other keys".to_string()), + description: Some("Use it to peek into the instance in a read-only mode.".to_string()), uid, actions: vec![Action::AllGet, Action::KeysGet], indexes: vec![IndexUidPattern::all()], @@ -468,7 +468,7 @@ pub mod actions { use super::Action::*; pub(crate) const ALL: u8 = All.repr(); - pub const ALL_READ: u8 = AllGet.repr(); + pub const ALL_GET: u8 = AllGet.repr(); pub const SEARCH: u8 = Search.repr(); pub const DOCUMENTS_ALL: u8 = DocumentsAll.repr(); pub const DOCUMENTS_ADD: u8 = DocumentsAdd.repr(); From 2090e9ea316b2dedfa272ba4a00d3b20109d8867 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 25 Jun 2025 10:08:25 +0200 Subject: [PATCH 022/312] Update test --- crates/meilisearch/tests/auth/api_keys.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs index fa09f17cb..60cb2ff46 100644 --- a/crates/meilisearch/tests/auth/api_keys.rs +++ b/crates/meilisearch/tests/auth/api_keys.rs @@ -852,11 +852,12 @@ async fn list_api_keys() { }, { "name": "Default Read-Only Admin API Key", - "description": "Use it to peek into the instance in a read-only mode. Caution! Do not expose it on a public frontend. It would give access to all other keys", + "description": "Use it to peek into the instance in a read-only mode.", "key": "[ignored]", "uid": "[ignored]", "actions": [ - "*.get" + "*.get", + "keys.get" ], "indexes": [ "*" From 6e0526090aa827ffd302d57d4a78f0fc25010589 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 25 Jun 2025 15:36:12 +0200 Subject: [PATCH 023/312] Implement sorting documents --- .../src/routes/indexes/documents.rs | 13 ++++ .../milli/src/facet/facet_sort_recursive.rs | 68 +++++++++++++++++++ crates/milli/src/facet/mod.rs | 1 + 3 files changed, 82 insertions(+) create mode 100644 crates/milli/src/facet/facet_sort_recursive.rs diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 50eec46fe..99ca2b7df 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -17,6 +17,10 @@ use meilisearch_types::error::deserr_codes::*; use meilisearch_types::error::{Code, ResponseError}; use meilisearch_types::heed::RoTxn; use meilisearch_types::index_uid::IndexUid; +use meilisearch_types::milli::facet::facet_sort_recursive::recursive_facet_sort; +use meilisearch_types::milli::facet::{ascending_facet_sort, descending_facet_sort}; +use meilisearch_types::milli::heed_codec::facet::FacetGroupKeyCodec; +use meilisearch_types::milli::heed_codec::BytesRefCodec; use meilisearch_types::milli::update::IndexDocumentsMethod; use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors; use meilisearch_types::milli::DocumentId; @@ -1533,6 +1537,15 @@ fn retrieve_documents>( })? } + let fields = vec![(0, true)]; + let number_db = index + .facet_id_f64_docids + .remap_key_type::>(); + let string_db = index + .facet_id_string_docids + .remap_key_type::>(); + candidates = recursive_facet_sort(&rtxn, number_db, string_db, &fields, candidates)?; + let (it, number_of_documents) = { let number_of_documents = candidates.len(); ( diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/facet/facet_sort_recursive.rs new file mode 100644 index 000000000..a6bbad906 --- /dev/null +++ b/crates/milli/src/facet/facet_sort_recursive.rs @@ -0,0 +1,68 @@ +use roaring::RoaringBitmap; +use heed::Database; +use crate::{facet::{ascending_facet_sort, descending_facet_sort}, heed_codec::{facet::{FacetGroupKeyCodec, FacetGroupValueCodec}, BytesRefCodec}}; + +pub fn recursive_facet_sort<'t>( + rtxn: &'t heed::RoTxn<'t>, + number_db: Database, FacetGroupValueCodec>, + string_db: Database, FacetGroupValueCodec>, + fields: &[(u16, bool)], + candidates: RoaringBitmap, +) -> heed::Result { + let (field_id, ascending) = match fields.first() { + Some(first) => *first, + None => return Ok(candidates), + }; + + let (number_iter, string_iter) = if ascending { + let number_iter = ascending_facet_sort( + rtxn, + number_db, + field_id, + candidates.clone(), + )?; + let string_iter = ascending_facet_sort( + rtxn, + string_db, + field_id, + candidates, + )?; + + (itertools::Either::Left(number_iter), itertools::Either::Left(string_iter)) + } else { + let number_iter = descending_facet_sort( + rtxn, + number_db, + field_id, + candidates.clone(), + )?; + let string_iter = descending_facet_sort( + rtxn, + string_db, + field_id, + candidates, + )?; + + (itertools::Either::Right(number_iter), itertools::Either::Right(string_iter)) + }; + + let chained_iter = number_iter.chain(string_iter); + let mut result = RoaringBitmap::new(); + for part in chained_iter { + let (inner_candidates, _) = part?; + if inner_candidates.len() <= 1 || fields.len() <= 1 { + result |= inner_candidates; + } else { + let inner_candidates = recursive_facet_sort( + rtxn, + number_db, + string_db, + &fields[1..], + inner_candidates, + )?; + result |= inner_candidates; + } + } + + Ok(result) +} diff --git a/crates/milli/src/facet/mod.rs b/crates/milli/src/facet/mod.rs index 274d2588d..a6351b42c 100644 --- a/crates/milli/src/facet/mod.rs +++ b/crates/milli/src/facet/mod.rs @@ -1,6 +1,7 @@ mod facet_type; mod facet_value; pub mod value_encoding; +pub mod facet_sort_recursive; pub use self::facet_type::FacetType; pub use self::facet_value::FacetValue; From b05cb80803873d4a17829ba15296b2ad33f3e856 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 25 Jun 2025 16:41:08 +0200 Subject: [PATCH 024/312] Take sort criteria from the request --- .../src/routes/indexes/documents.rs | 50 ++++++++++++------- .../milli/src/facet/facet_sort_recursive.rs | 39 +++++++++++++-- crates/milli/src/search/new/mod.rs | 19 +++---- 3 files changed, 79 insertions(+), 29 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 99ca2b7df..d91f43d21 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -1,6 +1,7 @@ use std::collections::HashSet; use std::io::{ErrorKind, Seek as _}; use std::marker::PhantomData; +use std::str::FromStr; use actix_web::http::header::CONTENT_TYPE; use actix_web::web::Data; @@ -18,12 +19,9 @@ use meilisearch_types::error::{Code, ResponseError}; use meilisearch_types::heed::RoTxn; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::milli::facet::facet_sort_recursive::recursive_facet_sort; -use meilisearch_types::milli::facet::{ascending_facet_sort, descending_facet_sort}; -use meilisearch_types::milli::heed_codec::facet::FacetGroupKeyCodec; -use meilisearch_types::milli::heed_codec::BytesRefCodec; use meilisearch_types::milli::update::IndexDocumentsMethod; use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors; -use meilisearch_types::milli::DocumentId; +use meilisearch_types::milli::{AscDesc, DocumentId}; use meilisearch_types::serde_cs::vec::CS; use meilisearch_types::star_or::OptionStarOrList; use meilisearch_types::tasks::KindWithContent; @@ -46,6 +44,7 @@ use crate::extractors::authentication::policies::*; use crate::extractors::authentication::GuardedData; use crate::extractors::payload::Payload; use crate::extractors::sequential_extractor::SeqHandler; +use crate::routes::indexes::search::fix_sort_query_parameters; use crate::routes::{ get_task_id, is_dry_run, PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT, }; @@ -410,6 +409,8 @@ pub struct BrowseQueryGet { #[param(default, value_type = Option, example = "popularity > 1000")] #[deserr(default, error = DeserrQueryParamError)] filter: Option, + #[deserr(default, error = DeserrQueryParamError)] + sort: Option, // TODO: change deser error } #[derive(Debug, Deserr, ToSchema)] @@ -434,6 +435,9 @@ pub struct BrowseQuery { #[schema(default, value_type = Option, example = "popularity > 1000")] #[deserr(default, error = DeserrJsonError)] filter: Option, + #[schema(default, value_type = Option>, example = json!(["title:asc", "rating:desc"]))] + #[deserr(default, error = DeserrJsonError)] // TODO: Change error + pub sort: Option>, } /// Get documents with POST @@ -575,7 +579,7 @@ pub async fn get_documents( ) -> Result { debug!(parameters = ?params, "Get documents GET"); - let BrowseQueryGet { limit, offset, fields, retrieve_vectors, filter, ids } = + let BrowseQueryGet { limit, offset, fields, retrieve_vectors, filter, ids, sort } = params.into_inner(); let filter = match filter { @@ -586,15 +590,14 @@ pub async fn get_documents( None => None, }; - let ids = ids.map(|ids| ids.into_iter().map(Into::into).collect()); - let query = BrowseQuery { offset: offset.0, limit: limit.0, fields: fields.merge_star_and_none(), retrieve_vectors: retrieve_vectors.0, filter, - ids, + ids: ids.map(|ids| ids.into_iter().map(Into::into).collect()), + sort: sort.map(|attr| fix_sort_query_parameters(&attr)), }; analytics.publish( @@ -619,7 +622,7 @@ fn documents_by_query( query: BrowseQuery, ) -> Result { let index_uid = IndexUid::try_from(index_uid.into_inner())?; - let BrowseQuery { offset, limit, fields, retrieve_vectors, filter, ids } = query; + let BrowseQuery { offset, limit, fields, retrieve_vectors, filter, ids, sort } = query; let retrieve_vectors = RetrieveVectors::new(retrieve_vectors); @@ -637,6 +640,22 @@ fn documents_by_query( None }; + let sort_criteria = if let Some(sort) = &sort { + let sorts: Vec<_> = + match sort.iter().map(|s| milli::AscDesc::from_str(s)).collect() { + Ok(sorts) => sorts, + Err(asc_desc_error) => { + return Err(milli::Error::from(milli::SortError::from( + asc_desc_error, + )) + .into()) + } + }; + Some(sorts) + } else { + None + }; + let index = index_scheduler.index(&index_uid)?; let (total, documents) = retrieve_documents( &index, @@ -647,6 +666,7 @@ fn documents_by_query( fields, retrieve_vectors, index_scheduler.features(), + sort_criteria, )?; let ret = PaginationView::new(offset, limit, total as usize, documents); @@ -1505,6 +1525,7 @@ fn retrieve_documents>( attributes_to_retrieve: Option>, retrieve_vectors: RetrieveVectors, features: RoFeatures, + sort_criteria: Option>, ) -> Result<(u64, Vec), ResponseError> { let rtxn = index.read_txn()?; let filter = &filter; @@ -1537,14 +1558,9 @@ fn retrieve_documents>( })? } - let fields = vec![(0, true)]; - let number_db = index - .facet_id_f64_docids - .remap_key_type::>(); - let string_db = index - .facet_id_string_docids - .remap_key_type::>(); - candidates = recursive_facet_sort(&rtxn, number_db, string_db, &fields, candidates)?; + if let Some(sort) = sort_criteria { + candidates = recursive_facet_sort(index, &rtxn, &sort, candidates)?; + } let (it, number_of_documents) = { let number_of_documents = candidates.len(); diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/facet/facet_sort_recursive.rs index a6bbad906..c0fd6ca6f 100644 --- a/crates/milli/src/facet/facet_sort_recursive.rs +++ b/crates/milli/src/facet/facet_sort_recursive.rs @@ -1,8 +1,8 @@ use roaring::RoaringBitmap; use heed::Database; -use crate::{facet::{ascending_facet_sort, descending_facet_sort}, heed_codec::{facet::{FacetGroupKeyCodec, FacetGroupValueCodec}, BytesRefCodec}}; +use crate::{heed_codec::{facet::{FacetGroupKeyCodec, FacetGroupValueCodec}, BytesRefCodec}, search::{facet::{ascending_facet_sort, descending_facet_sort}, new::check_sort_criteria}, AscDesc, Member}; -pub fn recursive_facet_sort<'t>( +fn recursive_facet_sort_inner<'t>( rtxn: &'t heed::RoTxn<'t>, number_db: Database, FacetGroupValueCodec>, string_db: Database, FacetGroupValueCodec>, @@ -53,7 +53,7 @@ pub fn recursive_facet_sort<'t>( if inner_candidates.len() <= 1 || fields.len() <= 1 { result |= inner_candidates; } else { - let inner_candidates = recursive_facet_sort( + let inner_candidates = recursive_facet_sort_inner( rtxn, number_db, string_db, @@ -66,3 +66,36 @@ pub fn recursive_facet_sort<'t>( Ok(result) } + +pub fn recursive_facet_sort<'t>( + index: &crate::Index, + rtxn: &'t heed::RoTxn<'t>, + sort: &[AscDesc], + candidates: RoaringBitmap, +) -> crate::Result { + check_sort_criteria(index, rtxn, Some(sort))?; + + let mut fields = Vec::new(); + let fields_ids_map = index.fields_ids_map(rtxn)?; + for sort in sort { + let (field_id, ascending) = match sort { + AscDesc::Asc(Member::Field(field)) => (fields_ids_map.id(field), true), + AscDesc::Desc(Member::Field(field)) => (fields_ids_map.id(field), false), + AscDesc::Asc(Member::Geo(_)) => todo!(), + AscDesc::Desc(Member::Geo(_)) => todo!(), + }; + if let Some(field_id) = field_id { + fields.push((field_id, ascending)); // FIXME: Should this return an error if the field is not found? + } + } + + let number_db = index + .facet_id_f64_docids + .remap_key_type::>(); + let string_db = index + .facet_id_string_docids + .remap_key_type::>(); + + let candidates = recursive_facet_sort_inner(rtxn, number_db, string_db, &fields, candidates)?; + Ok(candidates) +} diff --git a/crates/milli/src/search/new/mod.rs b/crates/milli/src/search/new/mod.rs index a65b4076b..5cb4c9fd5 100644 --- a/crates/milli/src/search/new/mod.rs +++ b/crates/milli/src/search/new/mod.rs @@ -638,7 +638,7 @@ pub fn execute_vector_search( time_budget: TimeBudget, ranking_score_threshold: Option, ) -> Result { - check_sort_criteria(ctx, sort_criteria.as_ref())?; + check_sort_criteria(ctx.index, ctx.txn, sort_criteria.as_deref())?; // FIXME: input universe = universe & documents_with_vectors // for now if we're computing embeddings for ALL documents, we can assume that this is just universe @@ -702,7 +702,7 @@ pub fn execute_search( ranking_score_threshold: Option, locales: Option<&Vec>, ) -> Result { - check_sort_criteria(ctx, sort_criteria.as_ref())?; + check_sort_criteria(ctx.index, ctx.txn, sort_criteria.as_deref())?; let mut used_negative_operator = false; let mut located_query_terms = None; @@ -872,9 +872,10 @@ pub fn execute_search( }) } -fn check_sort_criteria( - ctx: &SearchContext<'_>, - sort_criteria: Option<&Vec>, +pub(crate) fn check_sort_criteria( + index: &Index, + rtxn: &RoTxn<'_>, + sort_criteria: Option<&[AscDesc]>, ) -> Result<()> { let sort_criteria = if let Some(sort_criteria) = sort_criteria { sort_criteria @@ -888,19 +889,19 @@ fn check_sort_criteria( // We check that the sort ranking rule exists and throw an // error if we try to use it and that it doesn't. - let sort_ranking_rule_missing = !ctx.index.criteria(ctx.txn)?.contains(&crate::Criterion::Sort); + let sort_ranking_rule_missing = !index.criteria(rtxn)?.contains(&crate::Criterion::Sort); if sort_ranking_rule_missing { return Err(UserError::SortRankingRuleMissing.into()); } // We check that we are allowed to use the sort criteria, we check // that they are declared in the sortable fields. - let sortable_fields = ctx.index.sortable_fields(ctx.txn)?; + let sortable_fields = index.sortable_fields(rtxn)?; for asc_desc in sort_criteria { match asc_desc.member() { Member::Field(ref field) if !crate::is_faceted(field, &sortable_fields) => { let (valid_fields, hidden_fields) = - ctx.index.remove_hidden_fields(ctx.txn, sortable_fields)?; + index.remove_hidden_fields(rtxn, sortable_fields)?; return Err(UserError::InvalidSortableAttribute { field: field.to_string(), @@ -911,7 +912,7 @@ fn check_sort_criteria( } Member::Geo(_) if !sortable_fields.contains(RESERVED_GEO_FIELD_NAME) => { let (valid_fields, hidden_fields) = - ctx.index.remove_hidden_fields(ctx.txn, sortable_fields)?; + index.remove_hidden_fields(rtxn, sortable_fields)?; return Err(UserError::InvalidSortableAttribute { field: RESERVED_GEO_FIELD_NAME.to_string(), From 4534dc2cab1ede94527dc3c58690a3d0798b21c6 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 25 Jun 2025 16:45:32 +0200 Subject: [PATCH 025/312] Create another deserr error --- crates/meilisearch-types/src/error.rs | 1 + crates/meilisearch/src/routes/indexes/documents.rs | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index d2500b7e1..2eb22035e 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -237,6 +237,7 @@ InvalidDocumentRetrieveVectors , InvalidRequest , BAD_REQU MissingDocumentFilter , InvalidRequest , BAD_REQUEST ; MissingDocumentEditionFunction , InvalidRequest , BAD_REQUEST ; InvalidDocumentFilter , InvalidRequest , BAD_REQUEST ; +InvalidDocumentSort , InvalidRequest , BAD_REQUEST ; InvalidDocumentGeoField , InvalidRequest , BAD_REQUEST ; InvalidVectorDimensions , InvalidRequest , BAD_REQUEST ; InvalidVectorsType , InvalidRequest , BAD_REQUEST ; diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index d91f43d21..425930ced 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -409,8 +409,8 @@ pub struct BrowseQueryGet { #[param(default, value_type = Option, example = "popularity > 1000")] #[deserr(default, error = DeserrQueryParamError)] filter: Option, - #[deserr(default, error = DeserrQueryParamError)] - sort: Option, // TODO: change deser error + #[deserr(default, error = DeserrQueryParamError)] + sort: Option, } #[derive(Debug, Deserr, ToSchema)] @@ -436,8 +436,8 @@ pub struct BrowseQuery { #[deserr(default, error = DeserrJsonError)] filter: Option, #[schema(default, value_type = Option>, example = json!(["title:asc", "rating:desc"]))] - #[deserr(default, error = DeserrJsonError)] // TODO: Change error - pub sort: Option>, + #[deserr(default, error = DeserrJsonError)] + sort: Option>, } /// Get documents with POST From c15763f9104d809cad0e9f1889de016addd27da8 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 27 Jun 2025 09:39:24 +0200 Subject: [PATCH 026/312] Improve key description Co-authored-by: Tamo --- crates/meilisearch-auth/src/lib.rs | 2 +- crates/meilisearch-types/src/keys.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch-auth/src/lib.rs b/crates/meilisearch-auth/src/lib.rs index 000e574ac..582606651 100644 --- a/crates/meilisearch-auth/src/lib.rs +++ b/crates/meilisearch-auth/src/lib.rs @@ -351,7 +351,7 @@ pub struct IndexSearchRules { fn generate_default_keys(store: &HeedAuthStore) -> Result<()> { store.put_api_key(Key::default_chat())?; - store.put_api_key(Key::default_management())?; + store.put_api_key(Key::default_read_only_admin_key())?; store.put_api_key(Key::default_admin())?; store.put_api_key(Key::default_search())?; diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 96b2e8ae1..4a4bc40a8 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -144,14 +144,14 @@ impl Key { } } - pub fn default_management() -> Self { + pub fn default_read_only_admin_key() -> Self { let now = OffsetDateTime::now_utc(); let uid = Uuid::new_v4(); Self { name: Some("Default Read-Only Admin API Key".to_string()), - description: Some("Use it to peek into the instance in a read-only mode.".to_string()), + description: Some("Use it to peek into the instance in a read-only mode. Caution: This key gives you access to all the other api keys. Do not expose it on a public frontend".to_string()), uid, - actions: vec![Action::AllGet, Action::KeysGet], + actions: vec![Action::AllGet, Action::KeysGedt], indexes: vec![IndexUidPattern::all()], expires_at: None, created_at: now, From fb9170b8e3120f535a0a1949b8227ac5f862dd94 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 27 Jun 2025 09:40:30 +0200 Subject: [PATCH 027/312] Keep name consistent with others --- crates/meilisearch-auth/src/lib.rs | 2 +- crates/meilisearch-types/src/keys.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch-auth/src/lib.rs b/crates/meilisearch-auth/src/lib.rs index 582606651..6f5a5c2a2 100644 --- a/crates/meilisearch-auth/src/lib.rs +++ b/crates/meilisearch-auth/src/lib.rs @@ -351,7 +351,7 @@ pub struct IndexSearchRules { fn generate_default_keys(store: &HeedAuthStore) -> Result<()> { store.put_api_key(Key::default_chat())?; - store.put_api_key(Key::default_read_only_admin_key())?; + store.put_api_key(Key::default_read_only_admin())?; store.put_api_key(Key::default_admin())?; store.put_api_key(Key::default_search())?; diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 4a4bc40a8..7f10e9265 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -144,7 +144,7 @@ impl Key { } } - pub fn default_read_only_admin_key() -> Self { + pub fn default_read_only_admin() -> Self { let now = OffsetDateTime::now_utc(); let uid = Uuid::new_v4(); Self { From e3fba62e13ebfc0615fdaac06dc27fd2a9085407 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 27 Jun 2025 09:40:59 +0200 Subject: [PATCH 028/312] Fix typo --- crates/meilisearch-types/src/keys.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 7f10e9265..2911f22a2 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -151,7 +151,7 @@ impl Key { name: Some("Default Read-Only Admin API Key".to_string()), description: Some("Use it to peek into the instance in a read-only mode. Caution: This key gives you access to all the other api keys. Do not expose it on a public frontend".to_string()), uid, - actions: vec![Action::AllGet, Action::KeysGedt], + actions: vec![Action::AllGet, Action::KeysGet], indexes: vec![IndexUidPattern::all()], expires_at: None, created_at: now, From 28adbc0d1878c365e295aabdf41a7089a5c4c01e Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 27 Jun 2025 09:47:46 +0200 Subject: [PATCH 029/312] Update tests --- crates/meilisearch/tests/auth/api_keys.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs index 60cb2ff46..f717fd53e 100644 --- a/crates/meilisearch/tests/auth/api_keys.rs +++ b/crates/meilisearch/tests/auth/api_keys.rs @@ -852,7 +852,7 @@ async fn list_api_keys() { }, { "name": "Default Read-Only Admin API Key", - "description": "Use it to peek into the instance in a read-only mode.", + "description": "Use it to peek into the instance in a read-only mode. Caution: This key gives you access to all the other api keys. Do not expose it on a public frontend", "key": "[ignored]", "uid": "[ignored]", "actions": [ From 340d9e6edc71621b3c4dba7e95bbd7cf101453ff Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 27 Jun 2025 14:40:55 +0200 Subject: [PATCH 030/312] Optimize facet sort 5 to 10x speedup --- .../src/routes/indexes/documents.rs | 21 +- .../milli/src/facet/facet_sort_recursive.rs | 295 ++++++++++++++---- 2 files changed, 256 insertions(+), 60 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 425930ced..bcd227300 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -1558,19 +1558,32 @@ fn retrieve_documents>( })? } + let mut facet_sort = None; if let Some(sort) = sort_criteria { - candidates = recursive_facet_sort(index, &rtxn, &sort, candidates)?; + facet_sort = Some(recursive_facet_sort(index, &rtxn, &sort, &candidates)?) } - let (it, number_of_documents) = { + let (it, number_of_documents) = if let Some(facet_sort) = &facet_sort { + let number_of_documents = candidates.len(); + let iter = facet_sort.iter()?; + ( + itertools::Either::Left(some_documents( + index, + &rtxn, + iter.map(|d| d.unwrap()).skip(offset).take(limit), + retrieve_vectors, + )?), + number_of_documents, + ) + } else { let number_of_documents = candidates.len(); ( - some_documents( + itertools::Either::Right(some_documents( index, &rtxn, candidates.into_iter().skip(offset).take(limit), retrieve_vectors, - )?, + )?), number_of_documents, ) }; diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/facet/facet_sort_recursive.rs index c0fd6ca6f..47c3696f3 100644 --- a/crates/milli/src/facet/facet_sort_recursive.rs +++ b/crates/milli/src/facet/facet_sort_recursive.rs @@ -1,78 +1,256 @@ use roaring::RoaringBitmap; use heed::Database; -use crate::{heed_codec::{facet::{FacetGroupKeyCodec, FacetGroupValueCodec}, BytesRefCodec}, search::{facet::{ascending_facet_sort, descending_facet_sort}, new::check_sort_criteria}, AscDesc, Member}; +use crate::{heed_codec::{facet::{FacetGroupKeyCodec, FacetGroupValueCodec}, BytesRefCodec}, search::{facet::{ascending_facet_sort, descending_facet_sort}, new::check_sort_criteria}, AscDesc, DocumentId, Member}; -fn recursive_facet_sort_inner<'t>( - rtxn: &'t heed::RoTxn<'t>, +/// Builder for a [`SortedDocumentsIterator`]. +/// Most builders won't ever be built, because pagination will skip them. +pub struct SortedDocumentsIteratorBuilder<'ctx> { + rtxn: &'ctx heed::RoTxn<'ctx>, number_db: Database, FacetGroupValueCodec>, string_db: Database, FacetGroupValueCodec>, - fields: &[(u16, bool)], + fields: &'ctx [(u16, bool)], candidates: RoaringBitmap, -) -> heed::Result { - let (field_id, ascending) = match fields.first() { - Some(first) => *first, - None => return Ok(candidates), - }; +} - let (number_iter, string_iter) = if ascending { - let number_iter = ascending_facet_sort( +impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { + /// Performs the sort and builds a [`SortedDocumentsIterator`]. + fn build(self) -> heed::Result> { + let SortedDocumentsIteratorBuilder { rtxn, number_db, - field_id, - candidates.clone(), - )?; - let string_iter = ascending_facet_sort( - rtxn, string_db, - field_id, + fields, candidates, - )?; + } = self; + let size = candidates.len() as usize; - (itertools::Either::Left(number_iter), itertools::Either::Left(string_iter)) - } else { - let number_iter = descending_facet_sort( - rtxn, - number_db, - field_id, - candidates.clone(), - )?; - let string_iter = descending_facet_sort( - rtxn, - string_db, - field_id, - candidates, - )?; + // There is no point sorting a 1-element array + if size <= 1 { + return Ok(SortedDocumentsIterator::Leaf { + size, + values: Box::new(candidates.into_iter()), + }); + } - (itertools::Either::Right(number_iter), itertools::Either::Right(string_iter)) - }; + // There is no variable to sort on + let Some((field_id, ascending)) = fields.first().copied() else { + return Ok(SortedDocumentsIterator::Leaf { + size, + values: Box::new(candidates.into_iter()), + }); + }; - let chained_iter = number_iter.chain(string_iter); - let mut result = RoaringBitmap::new(); - for part in chained_iter { - let (inner_candidates, _) = part?; - if inner_candidates.len() <= 1 || fields.len() <= 1 { - result |= inner_candidates; + // Perform the sort on the first field + let (number_iter, string_iter) = if ascending { + let number_iter = ascending_facet_sort( + rtxn, + number_db, + field_id, + candidates.clone(), + )?; + let string_iter = ascending_facet_sort( + rtxn, + string_db, + field_id, + candidates, + )?; + + (itertools::Either::Left(number_iter), itertools::Either::Left(string_iter)) } else { - let inner_candidates = recursive_facet_sort_inner( + let number_iter = descending_facet_sort( + rtxn, + number_db, + field_id, + candidates.clone(), + )?; + let string_iter = descending_facet_sort( + rtxn, + string_db, + field_id, + candidates, + )?; + + (itertools::Either::Right(number_iter), itertools::Either::Right(string_iter)) + }; + + // Create builders for the next level of the tree + let number_db2 = number_db; + let string_db2 = string_db; + let number_iter = number_iter.map(move |r| -> heed::Result { + let (docids, _bytes) = r?; + Ok(SortedDocumentsIteratorBuilder { rtxn, number_db, string_db, - &fields[1..], - inner_candidates, - )?; - result |= inner_candidates; - } - } + fields: &fields[1..], + candidates: docids, + }) + }); + let string_iter = string_iter.map(move |r| -> heed::Result { + let (docids, _bytes) = r?; + Ok(SortedDocumentsIteratorBuilder { + rtxn, + number_db: number_db2, + string_db: string_db2, + fields: &fields[1..], + candidates: docids, + }) + }); - Ok(result) + Ok(SortedDocumentsIterator::Branch { + current_child: None, + next_children_size: size, + next_children: Box::new(number_iter.chain(string_iter)), + }) + } } -pub fn recursive_facet_sort<'t>( - index: &crate::Index, - rtxn: &'t heed::RoTxn<'t>, +/// A [`SortedDocumentsIterator`] allows efficient access to a continuous range of sorted documents. +/// This is ideal in the context of paginated queries in which only a small number of documents are needed at a time. +/// Search operations will only be performed upon access. +pub enum SortedDocumentsIterator<'ctx> { + Leaf { + /// The exact number of documents remaining + size: usize, + values: Box + 'ctx> + }, + Branch { + /// The current child, got from the children iterator + current_child: Option>>, + /// The exact number of documents remaining, excluding documents in the current child + next_children_size: usize, + /// Iterators to become the current child once it is exhausted + next_children: Box>> + 'ctx>, + } +} + +impl SortedDocumentsIterator<'_> { + /// Takes care of updating the current child if it is `None`, and also updates the size + fn update_current<'ctx>(current_child: &mut Option>>, next_children_size: &mut usize, next_children: &mut Box>> + 'ctx>) -> heed::Result<()> { + if current_child.is_none() { + *current_child = match next_children.next() { + Some(Ok(builder)) => { + let next_child = Box::new(builder.build()?); + *next_children_size -= next_child.size_hint().0; + Some(next_child) + }, + Some(Err(e)) => return Err(e), + None => return Ok(()), + }; + } + Ok(()) + } +} + +impl Iterator for SortedDocumentsIterator<'_> { + type Item = heed::Result; + + fn nth(&mut self, n: usize) -> Option { + // If it's at the leaf level, just forward the call to the values iterator + let (current_child, next_children, next_children_size) = match self { + SortedDocumentsIterator::Leaf { values, size } => { + *size = size.saturating_sub(n); + return values.nth(n).map(Ok) + }, + SortedDocumentsIterator::Branch { current_child, next_children, next_children_size } => (current_child, next_children, next_children_size), + }; + + // Otherwise don't directly iterate over children, skip them if we know we will go further + let mut to_skip = n - 1; + while to_skip > 0 { + if let Err(e) = SortedDocumentsIterator::update_current(current_child, next_children_size, next_children) { + return Some(Err(e)); + } + let Some(inner) = current_child else { + return None; // No more inner iterators, everything has been consumed. + }; + + if to_skip >= inner.size_hint().0 { + // The current child isn't large enough to contain the nth element. + // Skip it and continue with the next one. + to_skip -= inner.size_hint().0; + *current_child = None; + continue; + } else { + // The current iterator is large enough, so we can forward the call to it. + return inner.nth(to_skip + 1); + } + } + + self.next() + } + + fn size_hint(&self) -> (usize, Option) { + let size = match self { + SortedDocumentsIterator::Leaf { size, .. } => *size, + SortedDocumentsIterator::Branch { next_children_size, current_child: Some(current_child), .. } => current_child.size_hint().0 + next_children_size, + SortedDocumentsIterator::Branch { next_children_size, current_child: None, .. } => *next_children_size, + }; + + (size, Some(size)) + } + + fn next(&mut self) -> Option { + match self { + SortedDocumentsIterator::Leaf { values, size } => { + let result = values.next().map(Ok); + if result.is_some() { + *size -= 1; + } + result + }, + SortedDocumentsIterator::Branch { current_child, next_children_size, next_children } => { + let mut result = None; + while result.is_none() { + // Ensure we have selected an iterator to work with + if let Err(e) = SortedDocumentsIterator::update_current(current_child, next_children_size, next_children) { + return Some(Err(e)); + } + let Some(inner) = current_child else { + return None; + }; + + result = inner.next(); + + // If the current iterator is exhausted, we need to try the next one + if result.is_none() { + *current_child = None; + } + } + result + } + } + } +} + +/// A structure owning the data needed during the lifetime of a [`SortedDocumentsIterator`]. +pub struct SortedDocuments<'ctx> { + rtxn: &'ctx heed::RoTxn<'ctx>, + fields: Vec<(u16, bool)>, + number_db: Database, FacetGroupValueCodec>, + string_db: Database, FacetGroupValueCodec>, + candidates: &'ctx RoaringBitmap, +} + +impl <'ctx> SortedDocuments<'ctx> { + pub fn iter(&'ctx self) -> heed::Result> { + let builder = SortedDocumentsIteratorBuilder { + rtxn: self.rtxn, + number_db: self.number_db, + string_db: self.string_db, + fields: &self.fields, + candidates: self.candidates.clone(), + }; + builder.build() + } +} + +pub fn recursive_facet_sort<'ctx>( + index: &'ctx crate::Index, + rtxn: &'ctx heed::RoTxn<'ctx>, sort: &[AscDesc], - candidates: RoaringBitmap, -) -> crate::Result { + candidates: &'ctx RoaringBitmap, +) -> crate::Result> { check_sort_criteria(index, rtxn, Some(sort))?; let mut fields = Vec::new(); @@ -88,7 +266,7 @@ pub fn recursive_facet_sort<'t>( fields.push((field_id, ascending)); // FIXME: Should this return an error if the field is not found? } } - + let number_db = index .facet_id_f64_docids .remap_key_type::>(); @@ -96,6 +274,11 @@ pub fn recursive_facet_sort<'t>( .facet_id_string_docids .remap_key_type::>(); - let candidates = recursive_facet_sort_inner(rtxn, number_db, string_db, &fields, candidates)?; - Ok(candidates) + Ok(SortedDocuments { + rtxn, + fields, + number_db, + string_db, + candidates, + }) } From 63827bbee04e7a98e444901d2d0b2e83e46f7fe3 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 30 Jun 2025 11:59:59 +0200 Subject: [PATCH 031/312] Move sorting code out of search --- .gitignore | 8 +- crates/milli/src/documents/geo_sort.rs | 182 ++++++++++++++++++++ crates/milli/src/documents/mod.rs | 1 + crates/milli/src/search/new/distinct.rs | 2 +- crates/milli/src/search/new/geo_sort.rs | 210 ++++-------------------- crates/milli/src/search/new/mod.rs | 2 +- 6 files changed, 227 insertions(+), 178 deletions(-) create mode 100644 crates/milli/src/documents/geo_sort.rs diff --git a/.gitignore b/.gitignore index 07453a58f..d28baee77 100644 --- a/.gitignore +++ b/.gitignore @@ -11,12 +11,18 @@ /bench /_xtask_benchmark.ms /benchmarks +.DS_Store # Snapshots ## ... large *.full.snap -## ... unreviewed +## ... unreviewed *.snap.new +## ... pending +*.pending-snap + +# Tmp files +.tmp* # Fuzzcheck data for the facet indexing fuzz test crates/milli/fuzz/update::facet::incremental::fuzz::fuzz/ diff --git a/crates/milli/src/documents/geo_sort.rs b/crates/milli/src/documents/geo_sort.rs new file mode 100644 index 000000000..5b3968b39 --- /dev/null +++ b/crates/milli/src/documents/geo_sort.rs @@ -0,0 +1,182 @@ +use std::collections::VecDeque; + +use heed::RoTxn; +use roaring::RoaringBitmap; +use rstar::RTree; + +use crate::{ + distance_between_two_points, lat_lng_to_xyz, + search::new::geo_sort::{geo_value, opposite_of}, + GeoPoint, GeoSortStrategy, Index, +}; + +// TODO: Make it take a mut reference to cache +#[allow(clippy::too_many_arguments)] +pub fn fill_cache( + index: &Index, + txn: &RoTxn, + strategy: GeoSortStrategy, + ascending: bool, + target_point: [f64; 2], + field_ids: &Option<[u16; 2]>, + rtree: &mut Option>, + geo_candidates: &RoaringBitmap, + cached_sorted_docids: &mut VecDeque<(u32, [f64; 2])>, +) -> crate::Result<()> { + debug_assert!(cached_sorted_docids.is_empty()); + + // lazily initialize the rtree if needed by the strategy, and cache it in `self.rtree` + let rtree = if strategy.use_rtree(geo_candidates.len() as usize) { + if let Some(rtree) = rtree.as_ref() { + // get rtree from cache + Some(rtree) + } else { + let rtree2 = index.geo_rtree(txn)?.expect("geo candidates but no rtree"); + // insert rtree in cache and returns it. + // Can't use `get_or_insert_with` because getting the rtree from the DB is a fallible operation. + Some(&*rtree.insert(rtree2)) + } + } else { + None + }; + + let cache_size = strategy.cache_size(); + if let Some(rtree) = rtree { + if ascending { + let point = lat_lng_to_xyz(&target_point); + for point in rtree.nearest_neighbor_iter(&point) { + if geo_candidates.contains(point.data.0) { + cached_sorted_docids.push_back(point.data); + if cached_sorted_docids.len() >= cache_size { + break; + } + } + } + } else { + // in the case of the desc geo sort we look for the closest point to the opposite of the queried point + // and we insert the points in reverse order they get reversed when emptying the cache later on + let point = lat_lng_to_xyz(&opposite_of(target_point)); + for point in rtree.nearest_neighbor_iter(&point) { + if geo_candidates.contains(point.data.0) { + cached_sorted_docids.push_front(point.data); + if cached_sorted_docids.len() >= cache_size { + break; + } + } + } + } + } else { + // the iterative version + let [lat, lng] = field_ids.expect("fill_buffer can't be called without the lat&lng"); + + let mut documents = geo_candidates + .iter() + .map(|id| -> crate::Result<_> { Ok((id, geo_value(id, lat, lng, index, txn)?)) }) + .collect::>>()?; + // computing the distance between two points is expensive thus we cache the result + documents.sort_by_cached_key(|(_, p)| distance_between_two_points(&target_point, p) as usize); + cached_sorted_docids.extend(documents); + }; + + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub fn next_bucket( + index: &Index, + txn: &RoTxn, + universe: &RoaringBitmap, + strategy: GeoSortStrategy, + ascending: bool, + target_point: [f64; 2], + field_ids: &Option<[u16; 2]>, + rtree: &mut Option>, + + cached_sorted_docids: &mut VecDeque<(u32, [f64; 2])>, + geo_candidates: &RoaringBitmap, + + // Limit the number of docs in a single bucket to avoid unexpectedly large overhead + max_bucket_size: u64, + // Considering the errors of GPS and geographical calculations, distances less than distance_error_margin will be treated as equal + distance_error_margin: f64, +) -> crate::Result)>> { + let mut geo_candidates = geo_candidates & universe; + + if geo_candidates.is_empty() { + return Ok(Some((universe.clone(), None))); + } + + let next = |cache: &mut VecDeque<_>| { + if ascending { + cache.pop_front() + } else { + cache.pop_back() + } + }; + let put_back = |cache: &mut VecDeque<_>, x: _| { + if ascending { + cache.push_front(x) + } else { + cache.push_back(x) + } + }; + + let mut current_bucket = RoaringBitmap::new(); + // current_distance stores the first point and distance in current bucket + let mut current_distance: Option<([f64; 2], f64)> = None; + loop { + // The loop will only exit when we have found all points with equal distance or have exhausted the candidates. + if let Some((id, point)) = next(cached_sorted_docids) { + if geo_candidates.contains(id) { + let distance = distance_between_two_points(&target_point, &point); + if let Some((point0, bucket_distance)) = current_distance.as_ref() { + if (bucket_distance - distance).abs() > distance_error_margin { + // different distance, point belongs to next bucket + put_back(cached_sorted_docids, (id, point)); + return Ok(Some((current_bucket, Some(point0.to_owned())))); + } else { + // same distance, point belongs to current bucket + current_bucket.insert(id); + // remove from candidates to prevent it from being added to the cache again + geo_candidates.remove(id); + // current bucket size reaches limit, force return + if current_bucket.len() == max_bucket_size { + return Ok(Some((current_bucket, Some(point0.to_owned())))); + } + } + } else { + // first doc in current bucket + current_distance = Some((point, distance)); + current_bucket.insert(id); + geo_candidates.remove(id); + // current bucket size reaches limit, force return + if current_bucket.len() == max_bucket_size { + return Ok(Some((current_bucket, Some(point.to_owned())))); + } + } + } + } else { + // cache exhausted, we need to refill it + fill_cache( + index, + txn, + strategy, + ascending, + target_point, + field_ids, + rtree, + &geo_candidates, + cached_sorted_docids, + )?; + + if cached_sorted_docids.is_empty() { + // candidates exhausted, exit + if let Some((point0, _)) = current_distance.as_ref() { + return Ok(Some((current_bucket, Some(point0.to_owned())))); + } else { + return Ok(Some((universe.clone(), None))); + } + } + } + } +} diff --git a/crates/milli/src/documents/mod.rs b/crates/milli/src/documents/mod.rs index f43f7e842..6a05f61a5 100644 --- a/crates/milli/src/documents/mod.rs +++ b/crates/milli/src/documents/mod.rs @@ -3,6 +3,7 @@ mod enriched; mod primary_key; mod reader; mod serde_impl; +pub mod geo_sort; use std::fmt::Debug; use std::io; diff --git a/crates/milli/src/search/new/distinct.rs b/crates/milli/src/search/new/distinct.rs index 36172302a..48ad152ee 100644 --- a/crates/milli/src/search/new/distinct.rs +++ b/crates/milli/src/search/new/distinct.rs @@ -82,7 +82,7 @@ fn facet_value_docids( } /// Return an iterator over each number value in the given field of the given document. -fn facet_number_values<'a>( +pub(crate) fn facet_number_values<'a>( docid: u32, field_id: u16, index: &Index, diff --git a/crates/milli/src/search/new/geo_sort.rs b/crates/milli/src/search/new/geo_sort.rs index 3e7fe3458..a52a84575 100644 --- a/crates/milli/src/search/new/geo_sort.rs +++ b/crates/milli/src/search/new/geo_sort.rs @@ -7,12 +7,10 @@ use rstar::RTree; use super::facet_string_values; use super::ranking_rules::{RankingRule, RankingRuleOutput, RankingRuleQueryTrait}; +use crate::documents::geo_sort::{fill_cache, next_bucket}; use crate::heed_codec::facet::{FieldDocIdFacetCodec, OrderedF64Codec}; use crate::score_details::{self, ScoreDetails}; -use crate::{ - distance_between_two_points, lat_lng_to_xyz, GeoPoint, Index, Result, SearchContext, - SearchLogger, -}; +use crate::{GeoPoint, Index, Result, SearchContext, SearchLogger}; const FID_SIZE: usize = 2; const DOCID_SIZE: usize = 4; @@ -134,62 +132,17 @@ impl GeoSort { ctx: &mut SearchContext<'_>, geo_candidates: &RoaringBitmap, ) -> Result<()> { - debug_assert!(self.field_ids.is_some(), "fill_buffer can't be called without the lat&lng"); - debug_assert!(self.cached_sorted_docids.is_empty()); - - // lazily initialize the rtree if needed by the strategy, and cache it in `self.rtree` - let rtree = if self.strategy.use_rtree(geo_candidates.len() as usize) { - if let Some(rtree) = self.rtree.as_ref() { - // get rtree from cache - Some(rtree) - } else { - let rtree = ctx.index.geo_rtree(ctx.txn)?.expect("geo candidates but no rtree"); - // insert rtree in cache and returns it. - // Can't use `get_or_insert_with` because getting the rtree from the DB is a fallible operation. - Some(&*self.rtree.insert(rtree)) - } - } else { - None - }; - - let cache_size = self.strategy.cache_size(); - if let Some(rtree) = rtree { - if self.ascending { - let point = lat_lng_to_xyz(&self.point); - for point in rtree.nearest_neighbor_iter(&point) { - if geo_candidates.contains(point.data.0) { - self.cached_sorted_docids.push_back(point.data); - if self.cached_sorted_docids.len() >= cache_size { - break; - } - } - } - } else { - // in the case of the desc geo sort we look for the closest point to the opposite of the queried point - // and we insert the points in reverse order they get reversed when emptying the cache later on - let point = lat_lng_to_xyz(&opposite_of(self.point)); - for point in rtree.nearest_neighbor_iter(&point) { - if geo_candidates.contains(point.data.0) { - self.cached_sorted_docids.push_front(point.data); - if self.cached_sorted_docids.len() >= cache_size { - break; - } - } - } - } - } else { - // the iterative version - let [lat, lng] = self.field_ids.unwrap(); - - let mut documents = geo_candidates - .iter() - .map(|id| -> Result<_> { Ok((id, geo_value(id, lat, lng, ctx.index, ctx.txn)?)) }) - .collect::>>()?; - // computing the distance between two points is expensive thus we cache the result - documents - .sort_by_cached_key(|(_, p)| distance_between_two_points(&self.point, p) as usize); - self.cached_sorted_docids.extend(documents); - }; + fill_cache( + ctx.index, + ctx.txn, + self.strategy, + self.ascending, + self.point, + &self.field_ids, + &mut self.rtree, + geo_candidates, + &mut self.cached_sorted_docids, + )?; Ok(()) } @@ -199,7 +152,7 @@ impl GeoSort { /// /// If it is not able to find it in the facet number index it will extract it /// from the facet string index and parse it as f64 (as the geo extraction behaves). -fn geo_value( +pub(crate) fn geo_value( docid: u32, field_lat: u16, field_lng: u16, @@ -267,124 +220,31 @@ impl<'ctx, Q: RankingRuleQueryTrait> RankingRule<'ctx, Q> for GeoSort { ) -> Result>> { let query = self.query.as_ref().unwrap().clone(); - let mut geo_candidates = &self.geo_candidates & universe; - - if geo_candidates.is_empty() { - return Ok(Some(RankingRuleOutput { + next_bucket( + ctx.index, + ctx.txn, + universe, + self.strategy, + self.ascending, + self.point, + &self.field_ids, + &mut self.rtree, + &mut self.cached_sorted_docids, + &self.geo_candidates, + self.max_bucket_size, + self.distance_error_margin, + ) + .map(|o| { + o.map(|(candidates, point)| RankingRuleOutput { query, - candidates: universe.clone(), + candidates, score: ScoreDetails::GeoSort(score_details::GeoSort { target_point: self.point, ascending: self.ascending, - value: None, + value: point, }), - })); - } - - let ascending = self.ascending; - let next = |cache: &mut VecDeque<_>| { - if ascending { - cache.pop_front() - } else { - cache.pop_back() - } - }; - let put_back = |cache: &mut VecDeque<_>, x: _| { - if ascending { - cache.push_front(x) - } else { - cache.push_back(x) - } - }; - - let mut current_bucket = RoaringBitmap::new(); - // current_distance stores the first point and distance in current bucket - let mut current_distance: Option<([f64; 2], f64)> = None; - loop { - // The loop will only exit when we have found all points with equal distance or have exhausted the candidates. - if let Some((id, point)) = next(&mut self.cached_sorted_docids) { - if geo_candidates.contains(id) { - let distance = distance_between_two_points(&self.point, &point); - if let Some((point0, bucket_distance)) = current_distance.as_ref() { - if (bucket_distance - distance).abs() > self.distance_error_margin { - // different distance, point belongs to next bucket - put_back(&mut self.cached_sorted_docids, (id, point)); - return Ok(Some(RankingRuleOutput { - query, - candidates: current_bucket, - score: ScoreDetails::GeoSort(score_details::GeoSort { - target_point: self.point, - ascending: self.ascending, - value: Some(point0.to_owned()), - }), - })); - } else { - // same distance, point belongs to current bucket - current_bucket.insert(id); - // remove from cadidates to prevent it from being added to the cache again - geo_candidates.remove(id); - // current bucket size reaches limit, force return - if current_bucket.len() == self.max_bucket_size { - return Ok(Some(RankingRuleOutput { - query, - candidates: current_bucket, - score: ScoreDetails::GeoSort(score_details::GeoSort { - target_point: self.point, - ascending: self.ascending, - value: Some(point0.to_owned()), - }), - })); - } - } - } else { - // first doc in current bucket - current_distance = Some((point, distance)); - current_bucket.insert(id); - geo_candidates.remove(id); - // current bucket size reaches limit, force return - if current_bucket.len() == self.max_bucket_size { - return Ok(Some(RankingRuleOutput { - query, - candidates: current_bucket, - score: ScoreDetails::GeoSort(score_details::GeoSort { - target_point: self.point, - ascending: self.ascending, - value: Some(point.to_owned()), - }), - })); - } - } - } - } else { - // cache exhausted, we need to refill it - self.fill_buffer(ctx, &geo_candidates)?; - - if self.cached_sorted_docids.is_empty() { - // candidates exhausted, exit - if let Some((point0, _)) = current_distance.as_ref() { - return Ok(Some(RankingRuleOutput { - query, - candidates: current_bucket, - score: ScoreDetails::GeoSort(score_details::GeoSort { - target_point: self.point, - ascending: self.ascending, - value: Some(point0.to_owned()), - }), - })); - } else { - return Ok(Some(RankingRuleOutput { - query, - candidates: universe.clone(), - score: ScoreDetails::GeoSort(score_details::GeoSort { - target_point: self.point, - ascending: self.ascending, - value: None, - }), - })); - } - } - } - } + }) + }) } #[tracing::instrument(level = "trace", skip_all, target = "search::geo_sort")] @@ -396,7 +256,7 @@ impl<'ctx, Q: RankingRuleQueryTrait> RankingRule<'ctx, Q> for GeoSort { } /// Compute the antipodal coordinate of `coord` -fn opposite_of(mut coord: [f64; 2]) -> [f64; 2] { +pub(crate) fn opposite_of(mut coord: [f64; 2]) -> [f64; 2] { coord[0] *= -1.; // in the case of x,0 we want to return x,180 if coord[1] > 0. { diff --git a/crates/milli/src/search/new/mod.rs b/crates/milli/src/search/new/mod.rs index 5cb4c9fd5..da5e971af 100644 --- a/crates/milli/src/search/new/mod.rs +++ b/crates/milli/src/search/new/mod.rs @@ -1,7 +1,7 @@ mod bucket_sort; mod db_cache; mod distinct; -mod geo_sort; +pub(crate) mod geo_sort; mod graph_based_ranking_rule; mod interner; mod limits; From e35d58b531d5753aa47371bc056831b6edf4b2c9 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 30 Jun 2025 13:12:00 +0200 Subject: [PATCH 032/312] Move geosort code out of search --- .../src/routes/indexes/documents.rs | 16 +- crates/milli/src/documents/geo_sort.rs | 153 ++++++++++++++-- crates/milli/src/documents/mod.rs | 3 +- .../milli/src/facet/facet_sort_recursive.rs | 171 +++++++++--------- crates/milli/src/facet/mod.rs | 2 +- crates/milli/src/lib.rs | 5 +- crates/milli/src/search/mod.rs | 8 +- crates/milli/src/search/new/distinct.rs | 2 +- crates/milli/src/search/new/geo_sort.rs | 124 +------------ crates/milli/src/search/new/mod.rs | 16 +- 10 files changed, 257 insertions(+), 243 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index bcd227300..5545c870e 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -641,16 +641,12 @@ fn documents_by_query( }; let sort_criteria = if let Some(sort) = &sort { - let sorts: Vec<_> = - match sort.iter().map(|s| milli::AscDesc::from_str(s)).collect() { - Ok(sorts) => sorts, - Err(asc_desc_error) => { - return Err(milli::Error::from(milli::SortError::from( - asc_desc_error, - )) - .into()) - } - }; + let sorts: Vec<_> = match sort.iter().map(|s| milli::AscDesc::from_str(s)).collect() { + Ok(sorts) => sorts, + Err(asc_desc_error) => { + return Err(milli::Error::from(milli::SortError::from(asc_desc_error)).into()) + } + }; Some(sorts) } else { None diff --git a/crates/milli/src/documents/geo_sort.rs b/crates/milli/src/documents/geo_sort.rs index 5b3968b39..5b899e6d5 100644 --- a/crates/milli/src/documents/geo_sort.rs +++ b/crates/milli/src/documents/geo_sort.rs @@ -1,14 +1,70 @@ -use std::collections::VecDeque; - -use heed::RoTxn; +use crate::{ + distance_between_two_points, + heed_codec::facet::{FieldDocIdFacetCodec, OrderedF64Codec}, + lat_lng_to_xyz, + search::new::{facet_string_values, facet_values_prefix_key}, + GeoPoint, Index, +}; +use heed::{ + types::{Bytes, Unit}, + RoPrefix, RoTxn, +}; use roaring::RoaringBitmap; use rstar::RTree; +use std::collections::VecDeque; -use crate::{ - distance_between_two_points, lat_lng_to_xyz, - search::new::geo_sort::{geo_value, opposite_of}, - GeoPoint, GeoSortStrategy, Index, -}; +#[derive(Debug, Clone, Copy)] +pub struct GeoSortParameter { + // Define the strategy used by the geo sort + pub strategy: GeoSortStrategy, + // Limit the number of docs in a single bucket to avoid unexpectedly large overhead + pub max_bucket_size: u64, + // Considering the errors of GPS and geographical calculations, distances less than distance_error_margin will be treated as equal + pub distance_error_margin: f64, +} + +impl Default for GeoSortParameter { + fn default() -> Self { + Self { + strategy: GeoSortStrategy::default(), + max_bucket_size: 1000, + distance_error_margin: 1.0, + } + } +} +/// Define the strategy used by the geo sort. +/// The parameter represents the cache size, and, in the case of the Dynamic strategy, +/// the point where we move from using the iterative strategy to the rtree. +#[derive(Debug, Clone, Copy)] +pub enum GeoSortStrategy { + AlwaysIterative(usize), + AlwaysRtree(usize), + Dynamic(usize), +} + +impl Default for GeoSortStrategy { + fn default() -> Self { + GeoSortStrategy::Dynamic(1000) + } +} + +impl GeoSortStrategy { + pub fn use_rtree(&self, candidates: usize) -> bool { + match self { + GeoSortStrategy::AlwaysIterative(_) => false, + GeoSortStrategy::AlwaysRtree(_) => true, + GeoSortStrategy::Dynamic(i) => candidates >= *i, + } + } + + pub fn cache_size(&self) -> usize { + match self { + GeoSortStrategy::AlwaysIterative(i) + | GeoSortStrategy::AlwaysRtree(i) + | GeoSortStrategy::Dynamic(i) => *i, + } + } +} // TODO: Make it take a mut reference to cache #[allow(clippy::too_many_arguments)] @@ -74,7 +130,8 @@ pub fn fill_cache( .map(|id| -> crate::Result<_> { Ok((id, geo_value(id, lat, lng, index, txn)?)) }) .collect::>>()?; // computing the distance between two points is expensive thus we cache the result - documents.sort_by_cached_key(|(_, p)| distance_between_two_points(&target_point, p) as usize); + documents + .sort_by_cached_key(|(_, p)| distance_between_two_points(&target_point, p) as usize); cached_sorted_docids.extend(documents); }; @@ -86,19 +143,13 @@ pub fn next_bucket( index: &Index, txn: &RoTxn, universe: &RoaringBitmap, - strategy: GeoSortStrategy, ascending: bool, target_point: [f64; 2], field_ids: &Option<[u16; 2]>, rtree: &mut Option>, - cached_sorted_docids: &mut VecDeque<(u32, [f64; 2])>, geo_candidates: &RoaringBitmap, - - // Limit the number of docs in a single bucket to avoid unexpectedly large overhead - max_bucket_size: u64, - // Considering the errors of GPS and geographical calculations, distances less than distance_error_margin will be treated as equal - distance_error_margin: f64, + parameter: GeoSortParameter, ) -> crate::Result)>> { let mut geo_candidates = geo_candidates & universe; @@ -130,7 +181,7 @@ pub fn next_bucket( if geo_candidates.contains(id) { let distance = distance_between_two_points(&target_point, &point); if let Some((point0, bucket_distance)) = current_distance.as_ref() { - if (bucket_distance - distance).abs() > distance_error_margin { + if (bucket_distance - distance).abs() > parameter.distance_error_margin { // different distance, point belongs to next bucket put_back(cached_sorted_docids, (id, point)); return Ok(Some((current_bucket, Some(point0.to_owned())))); @@ -140,7 +191,7 @@ pub fn next_bucket( // remove from candidates to prevent it from being added to the cache again geo_candidates.remove(id); // current bucket size reaches limit, force return - if current_bucket.len() == max_bucket_size { + if current_bucket.len() == parameter.max_bucket_size { return Ok(Some((current_bucket, Some(point0.to_owned())))); } } @@ -150,7 +201,7 @@ pub fn next_bucket( current_bucket.insert(id); geo_candidates.remove(id); // current bucket size reaches limit, force return - if current_bucket.len() == max_bucket_size { + if current_bucket.len() == parameter.max_bucket_size { return Ok(Some((current_bucket, Some(point.to_owned())))); } } @@ -160,7 +211,7 @@ pub fn next_bucket( fill_cache( index, txn, - strategy, + parameter.strategy, ascending, target_point, field_ids, @@ -180,3 +231,65 @@ pub fn next_bucket( } } } + +/// Return an iterator over each number value in the given field of the given document. +fn facet_number_values<'a>( + docid: u32, + field_id: u16, + index: &Index, + txn: &'a RoTxn<'a>, +) -> crate::Result, Unit>> { + let key = facet_values_prefix_key(field_id, docid); + + let iter = index + .field_id_docid_facet_f64s + .remap_key_type::() + .prefix_iter(txn, &key)? + .remap_key_type(); + + Ok(iter) +} + +/// Extracts the lat and long values from a single document. +/// +/// If it is not able to find it in the facet number index it will extract it +/// from the facet string index and parse it as f64 (as the geo extraction behaves). +pub(crate) fn geo_value( + docid: u32, + field_lat: u16, + field_lng: u16, + index: &Index, + rtxn: &RoTxn<'_>, +) -> crate::Result<[f64; 2]> { + let extract_geo = |geo_field: u16| -> crate::Result { + match facet_number_values(docid, geo_field, index, rtxn)?.next() { + Some(Ok(((_, _, geo), ()))) => Ok(geo), + Some(Err(e)) => Err(e.into()), + None => match facet_string_values(docid, geo_field, index, rtxn)?.next() { + Some(Ok((_, geo))) => { + Ok(geo.parse::().expect("cannot parse geo field as f64")) + } + Some(Err(e)) => Err(e.into()), + None => panic!("A geo faceted document doesn't contain any lat or lng"), + }, + } + }; + + let lat = extract_geo(field_lat)?; + let lng = extract_geo(field_lng)?; + + Ok([lat, lng]) +} + +/// Compute the antipodal coordinate of `coord` +pub(crate) fn opposite_of(mut coord: [f64; 2]) -> [f64; 2] { + coord[0] *= -1.; + // in the case of x,0 we want to return x,180 + if coord[1] > 0. { + coord[1] -= 180.; + } else { + coord[1] += 180.; + } + + coord +} diff --git a/crates/milli/src/documents/mod.rs b/crates/milli/src/documents/mod.rs index 6a05f61a5..b515c4e98 100644 --- a/crates/milli/src/documents/mod.rs +++ b/crates/milli/src/documents/mod.rs @@ -1,9 +1,9 @@ mod builder; mod enriched; +pub mod geo_sort; mod primary_key; mod reader; mod serde_impl; -pub mod geo_sort; use std::fmt::Debug; use std::io; @@ -20,6 +20,7 @@ pub use primary_key::{ pub use reader::{DocumentsBatchCursor, DocumentsBatchCursorError, DocumentsBatchReader}; use serde::{Deserialize, Serialize}; +pub use self::geo_sort::{GeoSortParameter, GeoSortStrategy}; use crate::error::{FieldIdMapMissingEntry, InternalError}; use crate::{FieldId, Object, Result}; diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/facet/facet_sort_recursive.rs index 47c3696f3..7342114ef 100644 --- a/crates/milli/src/facet/facet_sort_recursive.rs +++ b/crates/milli/src/facet/facet_sort_recursive.rs @@ -1,6 +1,16 @@ -use roaring::RoaringBitmap; +use crate::{ + heed_codec::{ + facet::{FacetGroupKeyCodec, FacetGroupValueCodec}, + BytesRefCodec, + }, + search::{ + facet::{ascending_facet_sort, descending_facet_sort}, + new::check_sort_criteria, + }, + AscDesc, DocumentId, Member, +}; use heed::Database; -use crate::{heed_codec::{facet::{FacetGroupKeyCodec, FacetGroupValueCodec}, BytesRefCodec}, search::{facet::{ascending_facet_sort, descending_facet_sort}, new::check_sort_criteria}, AscDesc, DocumentId, Member}; +use roaring::RoaringBitmap; /// Builder for a [`SortedDocumentsIterator`]. /// Most builders won't ever be built, because pagination will skip them. @@ -15,13 +25,8 @@ pub struct SortedDocumentsIteratorBuilder<'ctx> { impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { /// Performs the sort and builds a [`SortedDocumentsIterator`]. fn build(self) -> heed::Result> { - let SortedDocumentsIteratorBuilder { - rtxn, - number_db, - string_db, - fields, - candidates, - } = self; + let SortedDocumentsIteratorBuilder { rtxn, number_db, string_db, fields, candidates } = + self; let size = candidates.len() as usize; // There is no point sorting a 1-element array @@ -42,33 +47,13 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { // Perform the sort on the first field let (number_iter, string_iter) = if ascending { - let number_iter = ascending_facet_sort( - rtxn, - number_db, - field_id, - candidates.clone(), - )?; - let string_iter = ascending_facet_sort( - rtxn, - string_db, - field_id, - candidates, - )?; + let number_iter = ascending_facet_sort(rtxn, number_db, field_id, candidates.clone())?; + let string_iter = ascending_facet_sort(rtxn, string_db, field_id, candidates)?; (itertools::Either::Left(number_iter), itertools::Either::Left(string_iter)) } else { - let number_iter = descending_facet_sort( - rtxn, - number_db, - field_id, - candidates.clone(), - )?; - let string_iter = descending_facet_sort( - rtxn, - string_db, - field_id, - candidates, - )?; + let number_iter = descending_facet_sort(rtxn, number_db, field_id, candidates.clone())?; + let string_iter = descending_facet_sort(rtxn, string_db, field_id, candidates)?; (itertools::Either::Right(number_iter), itertools::Either::Right(string_iter)) }; @@ -76,26 +61,28 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { // Create builders for the next level of the tree let number_db2 = number_db; let string_db2 = string_db; - let number_iter = number_iter.map(move |r| -> heed::Result { - let (docids, _bytes) = r?; - Ok(SortedDocumentsIteratorBuilder { - rtxn, - number_db, - string_db, - fields: &fields[1..], - candidates: docids, - }) - }); - let string_iter = string_iter.map(move |r| -> heed::Result { - let (docids, _bytes) = r?; - Ok(SortedDocumentsIteratorBuilder { - rtxn, - number_db: number_db2, - string_db: string_db2, - fields: &fields[1..], - candidates: docids, - }) - }); + let number_iter = + number_iter.map(move |r| -> heed::Result { + let (docids, _bytes) = r?; + Ok(SortedDocumentsIteratorBuilder { + rtxn, + number_db, + string_db, + fields: &fields[1..], + candidates: docids, + }) + }); + let string_iter = + string_iter.map(move |r| -> heed::Result { + let (docids, _bytes) = r?; + Ok(SortedDocumentsIteratorBuilder { + rtxn, + number_db: number_db2, + string_db: string_db2, + fields: &fields[1..], + candidates: docids, + }) + }); Ok(SortedDocumentsIterator::Branch { current_child: None, @@ -112,7 +99,7 @@ pub enum SortedDocumentsIterator<'ctx> { Leaf { /// The exact number of documents remaining size: usize, - values: Box + 'ctx> + values: Box + 'ctx>, }, Branch { /// The current child, got from the children iterator @@ -120,20 +107,27 @@ pub enum SortedDocumentsIterator<'ctx> { /// The exact number of documents remaining, excluding documents in the current child next_children_size: usize, /// Iterators to become the current child once it is exhausted - next_children: Box>> + 'ctx>, - } + next_children: + Box>> + 'ctx>, + }, } impl SortedDocumentsIterator<'_> { /// Takes care of updating the current child if it is `None`, and also updates the size - fn update_current<'ctx>(current_child: &mut Option>>, next_children_size: &mut usize, next_children: &mut Box>> + 'ctx>) -> heed::Result<()> { + fn update_current<'ctx>( + current_child: &mut Option>>, + next_children_size: &mut usize, + next_children: &mut Box< + dyn Iterator>> + 'ctx, + >, + ) -> heed::Result<()> { if current_child.is_none() { *current_child = match next_children.next() { Some(Ok(builder)) => { let next_child = Box::new(builder.build()?); *next_children_size -= next_child.size_hint().0; Some(next_child) - }, + } Some(Err(e)) => return Err(e), None => return Ok(()), }; @@ -150,15 +144,23 @@ impl Iterator for SortedDocumentsIterator<'_> { let (current_child, next_children, next_children_size) = match self { SortedDocumentsIterator::Leaf { values, size } => { *size = size.saturating_sub(n); - return values.nth(n).map(Ok) - }, - SortedDocumentsIterator::Branch { current_child, next_children, next_children_size } => (current_child, next_children, next_children_size), + return values.nth(n).map(Ok); + } + SortedDocumentsIterator::Branch { + current_child, + next_children, + next_children_size, + } => (current_child, next_children, next_children_size), }; // Otherwise don't directly iterate over children, skip them if we know we will go further let mut to_skip = n - 1; while to_skip > 0 { - if let Err(e) = SortedDocumentsIterator::update_current(current_child, next_children_size, next_children) { + if let Err(e) = SortedDocumentsIterator::update_current( + current_child, + next_children_size, + next_children, + ) { return Some(Err(e)); } let Some(inner) = current_child else { @@ -183,8 +185,14 @@ impl Iterator for SortedDocumentsIterator<'_> { fn size_hint(&self) -> (usize, Option) { let size = match self { SortedDocumentsIterator::Leaf { size, .. } => *size, - SortedDocumentsIterator::Branch { next_children_size, current_child: Some(current_child), .. } => current_child.size_hint().0 + next_children_size, - SortedDocumentsIterator::Branch { next_children_size, current_child: None, .. } => *next_children_size, + SortedDocumentsIterator::Branch { + next_children_size, + current_child: Some(current_child), + .. + } => current_child.size_hint().0 + next_children_size, + SortedDocumentsIterator::Branch { next_children_size, current_child: None, .. } => { + *next_children_size + } }; (size, Some(size)) @@ -198,12 +206,20 @@ impl Iterator for SortedDocumentsIterator<'_> { *size -= 1; } result - }, - SortedDocumentsIterator::Branch { current_child, next_children_size, next_children } => { + } + SortedDocumentsIterator::Branch { + current_child, + next_children_size, + next_children, + } => { let mut result = None; while result.is_none() { // Ensure we have selected an iterator to work with - if let Err(e) = SortedDocumentsIterator::update_current(current_child, next_children_size, next_children) { + if let Err(e) = SortedDocumentsIterator::update_current( + current_child, + next_children_size, + next_children, + ) { return Some(Err(e)); } let Some(inner) = current_child else { @@ -232,7 +248,7 @@ pub struct SortedDocuments<'ctx> { candidates: &'ctx RoaringBitmap, } -impl <'ctx> SortedDocuments<'ctx> { +impl<'ctx> SortedDocuments<'ctx> { pub fn iter(&'ctx self) -> heed::Result> { let builder = SortedDocumentsIteratorBuilder { rtxn: self.rtxn, @@ -266,19 +282,10 @@ pub fn recursive_facet_sort<'ctx>( fields.push((field_id, ascending)); // FIXME: Should this return an error if the field is not found? } } - - let number_db = index - .facet_id_f64_docids - .remap_key_type::>(); - let string_db = index - .facet_id_string_docids - .remap_key_type::>(); - Ok(SortedDocuments { - rtxn, - fields, - number_db, - string_db, - candidates, - }) + let number_db = index.facet_id_f64_docids.remap_key_type::>(); + let string_db = + index.facet_id_string_docids.remap_key_type::>(); + + Ok(SortedDocuments { rtxn, fields, number_db, string_db, candidates }) } diff --git a/crates/milli/src/facet/mod.rs b/crates/milli/src/facet/mod.rs index a6351b42c..8b0b9a25e 100644 --- a/crates/milli/src/facet/mod.rs +++ b/crates/milli/src/facet/mod.rs @@ -1,7 +1,7 @@ +pub mod facet_sort_recursive; mod facet_type; mod facet_value; pub mod value_encoding; -pub mod facet_sort_recursive; pub use self::facet_type::FacetType; pub use self::facet_value::FacetValue; diff --git a/crates/milli/src/lib.rs b/crates/milli/src/lib.rs index 504b4c68d..6fdae86b3 100644 --- a/crates/milli/src/lib.rs +++ b/crates/milli/src/lib.rs @@ -43,12 +43,13 @@ use std::fmt; use std::hash::BuildHasherDefault; use charabia::normalizer::{CharNormalizer, CompatibilityDecompositionNormalizer}; +pub use documents::GeoSortStrategy; pub use filter_parser::{Condition, FilterCondition, Span, Token}; use fxhash::{FxHasher32, FxHasher64}; pub use grenad::CompressionType; pub use search::new::{ - execute_search, filtered_universe, DefaultSearchLogger, GeoSortStrategy, SearchContext, - SearchLogger, VisualSearchLogger, + execute_search, filtered_universe, DefaultSearchLogger, SearchContext, SearchLogger, + VisualSearchLogger, }; use serde_json::Value; pub use thread_pool_no_abort::{PanicCatched, ThreadPoolNoAbort, ThreadPoolNoAbortBuilder}; diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index 62183afc3..48013b2ee 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -9,6 +9,8 @@ use roaring::bitmap::RoaringBitmap; pub use self::facet::{FacetDistribution, Filter, OrderBy, DEFAULT_VALUES_PER_FACET}; pub use self::new::matches::{FormatOptions, MatchBounds, MatcherBuilder, MatchingWords}; use self::new::{execute_vector_search, PartialSearchResult, VectorStoreStats}; +use crate::documents::GeoSortParameter; +use crate::documents::GeoSortStrategy; use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features}; use crate::index::MatchingStrategy; use crate::score_details::{ScoreDetails, ScoringStrategy}; @@ -46,7 +48,7 @@ pub struct Search<'a> { sort_criteria: Option>, distinct: Option, searchable_attributes: Option<&'a [String]>, - geo_param: new::GeoSortParameter, + geo_param: GeoSortParameter, terms_matching_strategy: TermsMatchingStrategy, scoring_strategy: ScoringStrategy, words_limit: usize, @@ -69,7 +71,7 @@ impl<'a> Search<'a> { sort_criteria: None, distinct: None, searchable_attributes: None, - geo_param: new::GeoSortParameter::default(), + geo_param: GeoSortParameter::default(), terms_matching_strategy: TermsMatchingStrategy::default(), scoring_strategy: Default::default(), exhaustive_number_hits: false, @@ -145,7 +147,7 @@ impl<'a> Search<'a> { } #[cfg(test)] - pub fn geo_sort_strategy(&mut self, strategy: new::GeoSortStrategy) -> &mut Search<'a> { + pub fn geo_sort_strategy(&mut self, strategy: GeoSortStrategy) -> &mut Search<'a> { self.geo_param.strategy = strategy; self } diff --git a/crates/milli/src/search/new/distinct.rs b/crates/milli/src/search/new/distinct.rs index 48ad152ee..455b495f5 100644 --- a/crates/milli/src/search/new/distinct.rs +++ b/crates/milli/src/search/new/distinct.rs @@ -118,7 +118,7 @@ pub fn facet_string_values<'a>( } #[allow(clippy::drop_non_drop)] -fn facet_values_prefix_key(distinct: u16, id: u32) -> [u8; FID_SIZE + DOCID_SIZE] { +pub(crate) fn facet_values_prefix_key(distinct: u16, id: u32) -> [u8; FID_SIZE + DOCID_SIZE] { concat_arrays::concat_arrays!(distinct.to_be_bytes(), id.to_be_bytes()) } diff --git a/crates/milli/src/search/new/geo_sort.rs b/crates/milli/src/search/new/geo_sort.rs index a52a84575..47001267d 100644 --- a/crates/milli/src/search/new/geo_sort.rs +++ b/crates/milli/src/search/new/geo_sort.rs @@ -8,6 +8,7 @@ use rstar::RTree; use super::facet_string_values; use super::ranking_rules::{RankingRule, RankingRuleOutput, RankingRuleQueryTrait}; use crate::documents::geo_sort::{fill_cache, next_bucket}; +use crate::documents::{GeoSortParameter, GeoSortStrategy}; use crate::heed_codec::facet::{FieldDocIdFacetCodec, OrderedF64Codec}; use crate::score_details::{self, ScoreDetails}; use crate::{GeoPoint, Index, Result, SearchContext, SearchLogger}; @@ -20,75 +21,10 @@ fn facet_values_prefix_key(distinct: u16, id: u32) -> [u8; FID_SIZE + DOCID_SIZE concat_arrays::concat_arrays!(distinct.to_be_bytes(), id.to_be_bytes()) } -/// Return an iterator over each number value in the given field of the given document. -fn facet_number_values<'a>( - docid: u32, - field_id: u16, - index: &Index, - txn: &'a RoTxn<'a>, -) -> Result, Unit>> { - let key = facet_values_prefix_key(field_id, docid); - - let iter = index - .field_id_docid_facet_f64s - .remap_key_type::() - .prefix_iter(txn, &key)? - .remap_key_type(); - - Ok(iter) -} - -#[derive(Debug, Clone, Copy)] -pub struct Parameter { - // Define the strategy used by the geo sort - pub strategy: Strategy, - // Limit the number of docs in a single bucket to avoid unexpectedly large overhead - pub max_bucket_size: u64, - // Considering the errors of GPS and geographical calculations, distances less than distance_error_margin will be treated as equal - pub distance_error_margin: f64, -} - -impl Default for Parameter { - fn default() -> Self { - Self { strategy: Strategy::default(), max_bucket_size: 1000, distance_error_margin: 1.0 } - } -} -/// Define the strategy used by the geo sort. -/// The parameter represents the cache size, and, in the case of the Dynamic strategy, -/// the point where we move from using the iterative strategy to the rtree. -#[derive(Debug, Clone, Copy)] -pub enum Strategy { - AlwaysIterative(usize), - AlwaysRtree(usize), - Dynamic(usize), -} - -impl Default for Strategy { - fn default() -> Self { - Strategy::Dynamic(1000) - } -} - -impl Strategy { - pub fn use_rtree(&self, candidates: usize) -> bool { - match self { - Strategy::AlwaysIterative(_) => false, - Strategy::AlwaysRtree(_) => true, - Strategy::Dynamic(i) => candidates >= *i, - } - } - - pub fn cache_size(&self) -> usize { - match self { - Strategy::AlwaysIterative(i) | Strategy::AlwaysRtree(i) | Strategy::Dynamic(i) => *i, - } - } -} - pub struct GeoSort { query: Option, - strategy: Strategy, + strategy: GeoSortStrategy, ascending: bool, point: [f64; 2], field_ids: Option<[u16; 2]>, @@ -105,12 +41,12 @@ pub struct GeoSort { impl GeoSort { pub fn new( - parameter: Parameter, + parameter: GeoSortParameter, geo_faceted_docids: RoaringBitmap, point: [f64; 2], ascending: bool, ) -> Result { - let Parameter { strategy, max_bucket_size, distance_error_margin } = parameter; + let GeoSortParameter { strategy, max_bucket_size, distance_error_margin } = parameter; Ok(Self { query: None, strategy, @@ -148,37 +84,6 @@ impl GeoSort { } } -/// Extracts the lat and long values from a single document. -/// -/// If it is not able to find it in the facet number index it will extract it -/// from the facet string index and parse it as f64 (as the geo extraction behaves). -pub(crate) fn geo_value( - docid: u32, - field_lat: u16, - field_lng: u16, - index: &Index, - rtxn: &RoTxn<'_>, -) -> Result<[f64; 2]> { - let extract_geo = |geo_field: u16| -> Result { - match facet_number_values(docid, geo_field, index, rtxn)?.next() { - Some(Ok(((_, _, geo), ()))) => Ok(geo), - Some(Err(e)) => Err(e.into()), - None => match facet_string_values(docid, geo_field, index, rtxn)?.next() { - Some(Ok((_, geo))) => { - Ok(geo.parse::().expect("cannot parse geo field as f64")) - } - Some(Err(e)) => Err(e.into()), - None => panic!("A geo faceted document doesn't contain any lat or lng"), - }, - } - }; - - let lat = extract_geo(field_lat)?; - let lng = extract_geo(field_lng)?; - - Ok([lat, lng]) -} - impl<'ctx, Q: RankingRuleQueryTrait> RankingRule<'ctx, Q> for GeoSort { fn id(&self) -> String { "geo_sort".to_owned() @@ -224,15 +129,17 @@ impl<'ctx, Q: RankingRuleQueryTrait> RankingRule<'ctx, Q> for GeoSort { ctx.index, ctx.txn, universe, - self.strategy, self.ascending, self.point, &self.field_ids, &mut self.rtree, &mut self.cached_sorted_docids, &self.geo_candidates, - self.max_bucket_size, - self.distance_error_margin, + GeoSortParameter { + strategy: self.strategy, + max_bucket_size: self.max_bucket_size, + distance_error_margin: self.distance_error_margin, + }, ) .map(|o| { o.map(|(candidates, point)| RankingRuleOutput { @@ -254,16 +161,3 @@ impl<'ctx, Q: RankingRuleQueryTrait> RankingRule<'ctx, Q> for GeoSort { self.cached_sorted_docids.clear(); } } - -/// Compute the antipodal coordinate of `coord` -pub(crate) fn opposite_of(mut coord: [f64; 2]) -> [f64; 2] { - coord[0] *= -1.; - // in the case of x,0 we want to return x,180 - if coord[1] > 0. { - coord[1] -= 180.; - } else { - coord[1] += 180.; - } - - coord -} diff --git a/crates/milli/src/search/new/mod.rs b/crates/milli/src/search/new/mod.rs index da5e971af..b5258413e 100644 --- a/crates/milli/src/search/new/mod.rs +++ b/crates/milli/src/search/new/mod.rs @@ -46,14 +46,14 @@ use resolve_query_graph::{compute_query_graph_docids, PhraseDocIdsCache}; use roaring::RoaringBitmap; use sort::Sort; -use self::distinct::facet_string_values; +pub(crate) use self::distinct::{facet_string_values, facet_values_prefix_key}; use self::geo_sort::GeoSort; -pub use self::geo_sort::{Parameter as GeoSortParameter, Strategy as GeoSortStrategy}; use self::graph_based_ranking_rule::Words; use self::interner::Interned; use self::vector_sort::VectorSort; use crate::attribute_patterns::{match_pattern, PatternMatch}; use crate::constants::RESERVED_GEO_FIELD_NAME; +use crate::documents::GeoSortParameter; use crate::index::PrefixSearch; use crate::localized_attributes_rules::LocalizedFieldIds; use crate::score_details::{ScoreDetails, ScoringStrategy}; @@ -319,7 +319,7 @@ fn resolve_negative_phrases( fn get_ranking_rules_for_placeholder_search<'ctx>( ctx: &SearchContext<'ctx>, sort_criteria: &Option>, - geo_param: geo_sort::Parameter, + geo_param: GeoSortParameter, ) -> Result>> { let mut sort = false; let mut sorted_fields = HashSet::new(); @@ -371,7 +371,7 @@ fn get_ranking_rules_for_placeholder_search<'ctx>( fn get_ranking_rules_for_vector<'ctx>( ctx: &SearchContext<'ctx>, sort_criteria: &Option>, - geo_param: geo_sort::Parameter, + geo_param: GeoSortParameter, limit_plus_offset: usize, target: &[f32], embedder_name: &str, @@ -448,7 +448,7 @@ fn get_ranking_rules_for_vector<'ctx>( fn get_ranking_rules_for_query_graph_search<'ctx>( ctx: &SearchContext<'ctx>, sort_criteria: &Option>, - geo_param: geo_sort::Parameter, + geo_param: GeoSortParameter, terms_matching_strategy: TermsMatchingStrategy, ) -> Result>> { // query graph search @@ -559,7 +559,7 @@ fn resolve_sort_criteria<'ctx, Query: RankingRuleQueryTrait>( ranking_rules: &mut Vec>, sorted_fields: &mut HashSet, geo_sorted: &mut bool, - geo_param: geo_sort::Parameter, + geo_param: GeoSortParameter, ) -> Result<()> { let sort_criteria = sort_criteria.clone().unwrap_or_default(); ranking_rules.reserve(sort_criteria.len()); @@ -629,7 +629,7 @@ pub fn execute_vector_search( universe: RoaringBitmap, sort_criteria: &Option>, distinct: &Option, - geo_param: geo_sort::Parameter, + geo_param: GeoSortParameter, from: usize, length: usize, embedder_name: &str, @@ -692,7 +692,7 @@ pub fn execute_search( mut universe: RoaringBitmap, sort_criteria: &Option>, distinct: &Option, - geo_param: geo_sort::Parameter, + geo_param: GeoSortParameter, from: usize, length: usize, words_limit: Option, From f86f4f619f2b69d408347b85c631f0e3942e888c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 30 Jun 2025 13:57:30 +0200 Subject: [PATCH 033/312] Implement geo sort on documents --- .../src/routes/indexes/documents.rs | 2 +- crates/milli/src/documents/geo_sort.rs | 1 - .../milli/src/facet/facet_sort_recursive.rs | 180 +++++++++++++++--- crates/milli/src/search/mod.rs | 3 +- crates/milli/src/search/new/geo_sort.rs | 14 +- 5 files changed, 152 insertions(+), 48 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 5545c870e..be6d647f7 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -1556,7 +1556,7 @@ fn retrieve_documents>( let mut facet_sort = None; if let Some(sort) = sort_criteria { - facet_sort = Some(recursive_facet_sort(index, &rtxn, &sort, &candidates)?) + facet_sort = Some(recursive_facet_sort(index, &rtxn, sort, &candidates)?) } let (it, number_of_documents) = if let Some(facet_sort) = &facet_sort { diff --git a/crates/milli/src/documents/geo_sort.rs b/crates/milli/src/documents/geo_sort.rs index 5b899e6d5..0750dfe5c 100644 --- a/crates/milli/src/documents/geo_sort.rs +++ b/crates/milli/src/documents/geo_sort.rs @@ -66,7 +66,6 @@ impl GeoSortStrategy { } } -// TODO: Make it take a mut reference to cache #[allow(clippy::too_many_arguments)] pub fn fill_cache( index: &Index, diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/facet/facet_sort_recursive.rs index 7342114ef..87da20391 100644 --- a/crates/milli/src/facet/facet_sort_recursive.rs +++ b/crates/milli/src/facet/facet_sort_recursive.rs @@ -1,4 +1,7 @@ +use std::collections::VecDeque; + use crate::{ + documents::{geo_sort::next_bucket, GeoSortParameter}, heed_codec::{ facet::{FacetGroupKeyCodec, FacetGroupValueCodec}, BytesRefCodec, @@ -12,38 +15,64 @@ use crate::{ use heed::Database; use roaring::RoaringBitmap; +#[derive(Debug, Clone, Copy)] +enum AscDescId { + Facet { field_id: u16, ascending: bool }, + Geo { field_ids: [u16; 2], target_point: [f64; 2], ascending: bool }, +} + /// Builder for a [`SortedDocumentsIterator`]. /// Most builders won't ever be built, because pagination will skip them. pub struct SortedDocumentsIteratorBuilder<'ctx> { + index: &'ctx crate::Index, rtxn: &'ctx heed::RoTxn<'ctx>, number_db: Database, FacetGroupValueCodec>, string_db: Database, FacetGroupValueCodec>, - fields: &'ctx [(u16, bool)], + fields: &'ctx [AscDescId], candidates: RoaringBitmap, + geo_candidates: &'ctx RoaringBitmap, } impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { /// Performs the sort and builds a [`SortedDocumentsIterator`]. - fn build(self) -> heed::Result> { - let SortedDocumentsIteratorBuilder { rtxn, number_db, string_db, fields, candidates } = - self; - let size = candidates.len() as usize; + fn build(self) -> crate::Result> { + let size = self.candidates.len() as usize; // There is no point sorting a 1-element array if size <= 1 { return Ok(SortedDocumentsIterator::Leaf { size, - values: Box::new(candidates.into_iter()), + values: Box::new(self.candidates.into_iter()), }); } - // There is no variable to sort on - let Some((field_id, ascending)) = fields.first().copied() else { - return Ok(SortedDocumentsIterator::Leaf { + match self.fields.first().copied() { + Some(AscDescId::Facet { field_id, ascending }) => self.build_facet(field_id, ascending), + Some(AscDescId::Geo { field_ids, target_point, ascending }) => { + self.build_geo(field_ids, target_point, ascending) + } + None => Ok(SortedDocumentsIterator::Leaf { size, - values: Box::new(candidates.into_iter()), - }); - }; + values: Box::new(self.candidates.into_iter()), + }), + } + } + + fn build_facet( + self, + field_id: u16, + ascending: bool, + ) -> crate::Result> { + let SortedDocumentsIteratorBuilder { + index, + rtxn, + number_db, + string_db, + fields, + candidates, + geo_candidates, + } = self; + let size = candidates.len() as usize; // Perform the sort on the first field let (number_iter, string_iter) = if ascending { @@ -62,25 +91,29 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { let number_db2 = number_db; let string_db2 = string_db; let number_iter = - number_iter.map(move |r| -> heed::Result { + number_iter.map(move |r| -> crate::Result { let (docids, _bytes) = r?; Ok(SortedDocumentsIteratorBuilder { + index, rtxn, number_db, string_db, fields: &fields[1..], candidates: docids, + geo_candidates, }) }); let string_iter = - string_iter.map(move |r| -> heed::Result { + string_iter.map(move |r| -> crate::Result { let (docids, _bytes) = r?; Ok(SortedDocumentsIteratorBuilder { + index, rtxn, number_db: number_db2, string_db: string_db2, fields: &fields[1..], candidates: docids, + geo_candidates, }) }); @@ -90,6 +123,60 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { next_children: Box::new(number_iter.chain(string_iter)), }) } + + fn build_geo( + self, + field_ids: [u16; 2], + target_point: [f64; 2], + ascending: bool, + ) -> crate::Result> { + let SortedDocumentsIteratorBuilder { + index, + rtxn, + number_db, + string_db, + fields, + candidates, + geo_candidates, + } = self; + + let mut cache = VecDeque::new(); + let mut rtree = None; + let size = candidates.len() as usize; + + let next_children = std::iter::from_fn(move || { + match next_bucket( + index, + rtxn, + &candidates, + ascending, + target_point, + &Some(field_ids), + &mut rtree, + &mut cache, + geo_candidates, + GeoSortParameter::default(), + ) { + Ok(Some((docids, _point))) => Some(Ok(SortedDocumentsIteratorBuilder { + index, + rtxn, + number_db, + string_db, + fields: &fields[1..], + candidates: docids, + geo_candidates, + })), + Ok(None) => None, + Err(e) => Some(Err(e)), + } + }); + + Ok(SortedDocumentsIterator::Branch { + current_child: None, + next_children_size: size, // TODO: confirm all candidates will be included + next_children: Box::new(next_children), + }) + } } /// A [`SortedDocumentsIterator`] allows efficient access to a continuous range of sorted documents. @@ -108,7 +195,7 @@ pub enum SortedDocumentsIterator<'ctx> { next_children_size: usize, /// Iterators to become the current child once it is exhausted next_children: - Box>> + 'ctx>, + Box>> + 'ctx>, }, } @@ -118,9 +205,9 @@ impl SortedDocumentsIterator<'_> { current_child: &mut Option>>, next_children_size: &mut usize, next_children: &mut Box< - dyn Iterator>> + 'ctx, + dyn Iterator>> + 'ctx, >, - ) -> heed::Result<()> { + ) -> crate::Result<()> { if current_child.is_none() { *current_child = match next_children.next() { Some(Ok(builder)) => { @@ -137,7 +224,7 @@ impl SortedDocumentsIterator<'_> { } impl Iterator for SortedDocumentsIterator<'_> { - type Item = heed::Result; + type Item = crate::Result; fn nth(&mut self, n: usize) -> Option { // If it's at the leaf level, just forward the call to the values iterator @@ -241,21 +328,25 @@ impl Iterator for SortedDocumentsIterator<'_> { /// A structure owning the data needed during the lifetime of a [`SortedDocumentsIterator`]. pub struct SortedDocuments<'ctx> { + index: &'ctx crate::Index, rtxn: &'ctx heed::RoTxn<'ctx>, - fields: Vec<(u16, bool)>, + fields: Vec, number_db: Database, FacetGroupValueCodec>, string_db: Database, FacetGroupValueCodec>, candidates: &'ctx RoaringBitmap, + geo_candidates: RoaringBitmap, } impl<'ctx> SortedDocuments<'ctx> { - pub fn iter(&'ctx self) -> heed::Result> { + pub fn iter(&'ctx self) -> crate::Result> { let builder = SortedDocumentsIteratorBuilder { + index: self.index, rtxn: self.rtxn, number_db: self.number_db, string_db: self.string_db, fields: &self.fields, candidates: self.candidates.clone(), + geo_candidates: &self.geo_candidates, }; builder.build() } @@ -264,28 +355,55 @@ impl<'ctx> SortedDocuments<'ctx> { pub fn recursive_facet_sort<'ctx>( index: &'ctx crate::Index, rtxn: &'ctx heed::RoTxn<'ctx>, - sort: &[AscDesc], + sort: Vec, candidates: &'ctx RoaringBitmap, ) -> crate::Result> { - check_sort_criteria(index, rtxn, Some(sort))?; + check_sort_criteria(index, rtxn, Some(&sort))?; let mut fields = Vec::new(); let fields_ids_map = index.fields_ids_map(rtxn)?; + let geo_candidates = index.geo_faceted_documents_ids(rtxn)?; // TODO: skip when no geo sort for sort in sort { - let (field_id, ascending) = match sort { - AscDesc::Asc(Member::Field(field)) => (fields_ids_map.id(field), true), - AscDesc::Desc(Member::Field(field)) => (fields_ids_map.id(field), false), - AscDesc::Asc(Member::Geo(_)) => todo!(), - AscDesc::Desc(Member::Geo(_)) => todo!(), + match sort { + AscDesc::Asc(Member::Field(field)) => { + if let Some(field_id) = fields_ids_map.id(&field) { + fields.push(AscDescId::Facet { field_id, ascending: true }); + } + } + AscDesc::Desc(Member::Field(field)) => { + if let Some(field_id) = fields_ids_map.id(&field) { + fields.push(AscDescId::Facet { field_id, ascending: false }); + } + } + AscDesc::Asc(Member::Geo(target_point)) => { + if let (Some(lat), Some(lng)) = + (fields_ids_map.id("_geo.lat"), fields_ids_map.id("_geo.lng")) + { + fields.push(AscDescId::Geo { + field_ids: [lat, lng], + target_point, + ascending: true, + }); + } + } + AscDesc::Desc(Member::Geo(target_point)) => { + if let (Some(lat), Some(lng)) = + (fields_ids_map.id("_geo.lat"), fields_ids_map.id("_geo.lng")) + { + fields.push(AscDescId::Geo { + field_ids: [lat, lng], + target_point, + ascending: false, + }); + } + } }; - if let Some(field_id) = field_id { - fields.push((field_id, ascending)); // FIXME: Should this return an error if the field is not found? - } + // FIXME: Should this return an error if the field is not found? } let number_db = index.facet_id_f64_docids.remap_key_type::>(); let string_db = index.facet_id_string_docids.remap_key_type::>(); - Ok(SortedDocuments { rtxn, fields, number_db, string_db, candidates }) + Ok(SortedDocuments { index, rtxn, fields, number_db, string_db, candidates, geo_candidates }) } diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index 48013b2ee..b073d271c 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -10,7 +10,6 @@ pub use self::facet::{FacetDistribution, Filter, OrderBy, DEFAULT_VALUES_PER_FAC pub use self::new::matches::{FormatOptions, MatchBounds, MatcherBuilder, MatchingWords}; use self::new::{execute_vector_search, PartialSearchResult, VectorStoreStats}; use crate::documents::GeoSortParameter; -use crate::documents::GeoSortStrategy; use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features}; use crate::index::MatchingStrategy; use crate::score_details::{ScoreDetails, ScoringStrategy}; @@ -147,7 +146,7 @@ impl<'a> Search<'a> { } #[cfg(test)] - pub fn geo_sort_strategy(&mut self, strategy: GeoSortStrategy) -> &mut Search<'a> { + pub fn geo_sort_strategy(&mut self, strategy: crate::GeoSortStrategy) -> &mut Search<'a> { self.geo_param.strategy = strategy; self } diff --git a/crates/milli/src/search/new/geo_sort.rs b/crates/milli/src/search/new/geo_sort.rs index 47001267d..6c7d7b03b 100644 --- a/crates/milli/src/search/new/geo_sort.rs +++ b/crates/milli/src/search/new/geo_sort.rs @@ -1,25 +1,13 @@ use std::collections::VecDeque; -use heed::types::{Bytes, Unit}; -use heed::{RoPrefix, RoTxn}; use roaring::RoaringBitmap; use rstar::RTree; -use super::facet_string_values; use super::ranking_rules::{RankingRule, RankingRuleOutput, RankingRuleQueryTrait}; use crate::documents::geo_sort::{fill_cache, next_bucket}; use crate::documents::{GeoSortParameter, GeoSortStrategy}; -use crate::heed_codec::facet::{FieldDocIdFacetCodec, OrderedF64Codec}; use crate::score_details::{self, ScoreDetails}; -use crate::{GeoPoint, Index, Result, SearchContext, SearchLogger}; - -const FID_SIZE: usize = 2; -const DOCID_SIZE: usize = 4; - -#[allow(clippy::drop_non_drop)] -fn facet_values_prefix_key(distinct: u16, id: u32) -> [u8; FID_SIZE + DOCID_SIZE] { - concat_arrays::concat_arrays!(distinct.to_be_bytes(), id.to_be_bytes()) -} +use crate::{GeoPoint, Result, SearchContext, SearchLogger}; pub struct GeoSort { query: Option, From f6803dd7d100b9d419747c69fc03f2207a64dfa9 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 30 Jun 2025 14:05:23 +0200 Subject: [PATCH 034/312] Simplify iterator chaining in facet sort --- .../milli/src/facet/facet_sort_recursive.rs | 43 ++++++------------- 1 file changed, 14 insertions(+), 29 deletions(-) diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/facet/facet_sort_recursive.rs index 87da20391..6f26ad16f 100644 --- a/crates/milli/src/facet/facet_sort_recursive.rs +++ b/crates/milli/src/facet/facet_sort_recursive.rs @@ -88,39 +88,24 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { }; // Create builders for the next level of the tree - let number_db2 = number_db; - let string_db2 = string_db; - let number_iter = - number_iter.map(move |r| -> crate::Result { - let (docids, _bytes) = r?; - Ok(SortedDocumentsIteratorBuilder { - index, - rtxn, - number_db, - string_db, - fields: &fields[1..], - candidates: docids, - geo_candidates, - }) - }); - let string_iter = - string_iter.map(move |r| -> crate::Result { - let (docids, _bytes) = r?; - Ok(SortedDocumentsIteratorBuilder { - index, - rtxn, - number_db: number_db2, - string_db: string_db2, - fields: &fields[1..], - candidates: docids, - geo_candidates, - }) - }); + let number_iter = number_iter.map(|r| r.map(|(d, _)| d)); + let string_iter = string_iter.map(|r| r.map(|(d, _)| d)); + let next_children = number_iter.chain(string_iter).map(move |r| { + Ok(SortedDocumentsIteratorBuilder { + index, + rtxn, + number_db, + string_db, + fields: &fields[1..], + candidates: r?, + geo_candidates, + }) + }); Ok(SortedDocumentsIterator::Branch { current_child: None, next_children_size: size, - next_children: Box::new(number_iter.chain(string_iter)), + next_children: Box::new(next_children), }) } From 29e9c74a49dd152c1db37e6b54d547dfeebc3746 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 30 Jun 2025 16:17:04 +0200 Subject: [PATCH 035/312] Merge two ifs --- crates/meilisearch/src/routes/indexes/documents.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index be6d647f7..d9b3f106f 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -1554,13 +1554,10 @@ fn retrieve_documents>( })? } - let mut facet_sort = None; - if let Some(sort) = sort_criteria { - facet_sort = Some(recursive_facet_sort(index, &rtxn, sort, &candidates)?) - } - - let (it, number_of_documents) = if let Some(facet_sort) = &facet_sort { + let facet_sort; + let (it, number_of_documents) = if let Some(sort) = sort_criteria { let number_of_documents = candidates.len(); + facet_sort = recursive_facet_sort(index, &rtxn, sort, &candidates)?; let iter = facet_sort.iter()?; ( itertools::Either::Left(some_documents( From eb2c2815b63c049d5e2aeabe6b67283bb2759ca6 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 10:00:10 +0200 Subject: [PATCH 036/312] Fix panic --- .../milli/src/facet/facet_sort_recursive.rs | 65 +++++++++++++------ 1 file changed, 44 insertions(+), 21 deletions(-) diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/facet/facet_sort_recursive.rs index 6f26ad16f..213d18624 100644 --- a/crates/milli/src/facet/facet_sort_recursive.rs +++ b/crates/milli/src/facet/facet_sort_recursive.rs @@ -128,37 +128,60 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { let mut cache = VecDeque::new(); let mut rtree = None; let size = candidates.len() as usize; + let not_geo_candidates = candidates.clone() - geo_candidates; + let mut geo_remaining = size - not_geo_candidates.len() as usize; + let mut not_geo_candidates = Some(not_geo_candidates); let next_children = std::iter::from_fn(move || { - match next_bucket( - index, - rtxn, - &candidates, - ascending, - target_point, - &Some(field_ids), - &mut rtree, - &mut cache, - geo_candidates, - GeoSortParameter::default(), - ) { - Ok(Some((docids, _point))) => Some(Ok(SortedDocumentsIteratorBuilder { + // Find the next bucket of geo-sorted documents. + // next_bucket loops and will go back to the beginning so we use a variable to track how many are left. + if geo_remaining > 0 { + if let Ok(Some((docids, _point))) = next_bucket( index, rtxn, - number_db, - string_db, - fields: &fields[1..], - candidates: docids, + &candidates, + ascending, + target_point, + &Some(field_ids), + &mut rtree, + &mut cache, geo_candidates, - })), - Ok(None) => None, - Err(e) => Some(Err(e)), + GeoSortParameter::default(), + ) { + geo_remaining -= docids.len() as usize; + return Some(Ok(SortedDocumentsIteratorBuilder { + index, + rtxn, + number_db, + string_db, + fields: &fields[1..], + candidates: docids, + geo_candidates, + })); + } } + + // Once all geo candidates have been processed, we can return the others + if let Some(not_geo_candidates) = not_geo_candidates.take() { + if !not_geo_candidates.is_empty() { + return Some(Ok(SortedDocumentsIteratorBuilder { + index, + rtxn, + number_db, + string_db, + fields: &fields[1..], + candidates: not_geo_candidates, + geo_candidates, + })); + } + } + + None }); Ok(SortedDocumentsIterator::Branch { current_child: None, - next_children_size: size, // TODO: confirm all candidates will be included + next_children_size: size, next_children: Box::new(next_children), }) } From f4a908669cb4db3df0ce9ff68c5cd13d0b26cf5c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 10:02:15 +0200 Subject: [PATCH 037/312] Add tests --- crates/meilisearch/tests/common/index.rs | 2 + .../tests/documents/get_documents.rs | 289 +++++++++++++++++- crates/meilisearch/tests/vector/settings.rs | 9 +- .../milli/src/facet/facet_sort_recursive.rs | 2 +- 4 files changed, 291 insertions(+), 11 deletions(-) diff --git a/crates/meilisearch/tests/common/index.rs b/crates/meilisearch/tests/common/index.rs index e324d2ff5..f1fdeba91 100644 --- a/crates/meilisearch/tests/common/index.rs +++ b/crates/meilisearch/tests/common/index.rs @@ -562,5 +562,7 @@ pub struct GetAllDocumentsOptions { pub offset: Option, #[serde(skip_serializing_if = "Option::is_none")] pub fields: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + pub sort: Option>, pub retrieve_vectors: bool, } diff --git a/crates/meilisearch/tests/documents/get_documents.rs b/crates/meilisearch/tests/documents/get_documents.rs index 63dc224c2..2267b8f5d 100644 --- a/crates/meilisearch/tests/documents/get_documents.rs +++ b/crates/meilisearch/tests/documents/get_documents.rs @@ -5,8 +5,8 @@ use urlencoding::encode as urlencode; use crate::common::encoder::Encoder; use crate::common::{ - shared_does_not_exists_index, shared_empty_index, shared_index_with_test_set, - GetAllDocumentsOptions, Server, Value, + shared_does_not_exists_index, shared_empty_index, shared_index_with_geo_documents, + shared_index_with_test_set, GetAllDocumentsOptions, Server, Value, }; use crate::json; @@ -83,6 +83,291 @@ async fn get_document() { ); } +#[actix_rt::test] +async fn get_document_sorted() { + let server = Server::new_shared(); + let index = server.unique_index(); + index.load_test_set().await; + + let (task, _status_code) = + index.update_settings_sortable_attributes(json!(["age", "email", "gender", "name"])).await; + server.wait_task(task.uid()).await.succeeded(); + + let (response, _code) = index + .get_all_documents(GetAllDocumentsOptions { + fields: Some(vec!["id", "age", "email"]), + sort: Some(vec!["age:asc", "email:desc"]), + ..Default::default() + }) + .await; + let results = response["results"].as_array().unwrap(); + snapshot!(json_string!(results), @r#" + [ + { + "id": 5, + "age": 20, + "email": "warrenwatson@chorizon.com" + }, + { + "id": 6, + "age": 20, + "email": "sheliaberry@chorizon.com" + }, + { + "id": 57, + "age": 20, + "email": "kaitlinconner@chorizon.com" + }, + { + "id": 45, + "age": 20, + "email": "irenebennett@chorizon.com" + }, + { + "id": 40, + "age": 21, + "email": "staffordemerson@chorizon.com" + }, + { + "id": 41, + "age": 21, + "email": "salinasgamble@chorizon.com" + }, + { + "id": 63, + "age": 21, + "email": "knowleshebert@chorizon.com" + }, + { + "id": 50, + "age": 21, + "email": "guerramcintyre@chorizon.com" + }, + { + "id": 44, + "age": 22, + "email": "jonispears@chorizon.com" + }, + { + "id": 56, + "age": 23, + "email": "tuckerbarry@chorizon.com" + }, + { + "id": 51, + "age": 23, + "email": "keycervantes@chorizon.com" + }, + { + "id": 60, + "age": 23, + "email": "jodyherrera@chorizon.com" + }, + { + "id": 70, + "age": 23, + "email": "glassperkins@chorizon.com" + }, + { + "id": 75, + "age": 24, + "email": "emmajacobs@chorizon.com" + }, + { + "id": 68, + "age": 24, + "email": "angelinadyer@chorizon.com" + }, + { + "id": 17, + "age": 25, + "email": "ortegabrennan@chorizon.com" + }, + { + "id": 76, + "age": 25, + "email": "claricegardner@chorizon.com" + }, + { + "id": 43, + "age": 25, + "email": "arnoldbender@chorizon.com" + }, + { + "id": 12, + "age": 25, + "email": "aidakirby@chorizon.com" + }, + { + "id": 9, + "age": 26, + "email": "kellimendez@chorizon.com" + } + ] + "#); + + let (response, _code) = index + .get_all_documents(GetAllDocumentsOptions { + fields: Some(vec!["id", "gender", "name"]), + sort: Some(vec!["gender:asc", "name:asc"]), + ..Default::default() + }) + .await; + let results = response["results"].as_array().unwrap(); + snapshot!(json_string!(results), @r#" + [ + { + "id": 3, + "name": "Adeline Flynn", + "gender": "female" + }, + { + "id": 12, + "name": "Aida Kirby", + "gender": "female" + }, + { + "id": 68, + "name": "Angelina Dyer", + "gender": "female" + }, + { + "id": 15, + "name": "Aurelia Contreras", + "gender": "female" + }, + { + "id": 36, + "name": "Barbra Valenzuela", + "gender": "female" + }, + { + "id": 23, + "name": "Blanca Mcclain", + "gender": "female" + }, + { + "id": 53, + "name": "Caitlin Burnett", + "gender": "female" + }, + { + "id": 71, + "name": "Candace Sawyer", + "gender": "female" + }, + { + "id": 65, + "name": "Carole Rowland", + "gender": "female" + }, + { + "id": 33, + "name": "Cecilia Greer", + "gender": "female" + }, + { + "id": 1, + "name": "Cherry Orr", + "gender": "female" + }, + { + "id": 38, + "name": "Christina Short", + "gender": "female" + }, + { + "id": 7, + "name": "Chrystal Boyd", + "gender": "female" + }, + { + "id": 76, + "name": "Clarice Gardner", + "gender": "female" + }, + { + "id": 73, + "name": "Eleanor Shepherd", + "gender": "female" + }, + { + "id": 75, + "name": "Emma Jacobs", + "gender": "female" + }, + { + "id": 16, + "name": "Estella Bass", + "gender": "female" + }, + { + "id": 62, + "name": "Estelle Ramirez", + "gender": "female" + }, + { + "id": 20, + "name": "Florence Long", + "gender": "female" + }, + { + "id": 42, + "name": "Graciela Russell", + "gender": "female" + } + ] + "#); +} + +#[actix_rt::test] +async fn get_document_geosorted() { + let index = shared_index_with_geo_documents().await; + + let (response, _code) = index + .get_all_documents(GetAllDocumentsOptions { + sort: Some(vec!["_geoPoint(45.4777599, 9.1967508):asc"]), + ..Default::default() + }) + .await; + let results = response["results"].as_array().unwrap(); + snapshot!(json_string!(results), @r#" + [ + { + "id": 2, + "name": "La Bella Italia", + "address": "456 Elm Street, Townsville", + "type": "Italian", + "rating": 9, + "_geo": { + "lat": "45.4777599", + "lng": "9.1967508" + } + }, + { + "id": 1, + "name": "Taco Truck", + "address": "444 Salsa Street, Burritoville", + "type": "Mexican", + "rating": 9, + "_geo": { + "lat": 34.0522, + "lng": -118.2437 + } + }, + { + "id": 3, + "name": "Crêpe Truck", + "address": "2 Billig Avenue, Rouenville", + "type": "French", + "rating": 10 + } + ] + "#); +} + +// TODO test on not sortable attributes + #[actix_rt::test] async fn error_get_unexisting_index_all_documents() { let index = shared_does_not_exists_index().await; diff --git a/crates/meilisearch/tests/vector/settings.rs b/crates/meilisearch/tests/vector/settings.rs index 50253f930..d26174faf 100644 --- a/crates/meilisearch/tests/vector/settings.rs +++ b/crates/meilisearch/tests/vector/settings.rs @@ -101,14 +101,7 @@ async fn reset_embedder_documents() { server.wait_task(response.uid()).await; // Make sure the documents are still present - let (documents, _code) = index - .get_all_documents(GetAllDocumentsOptions { - limit: None, - offset: None, - retrieve_vectors: false, - fields: None, - }) - .await; + let (documents, _code) = index.get_all_documents(GetAllDocumentsOptions::default()).await; snapshot!(json_string!(documents), @r###" { "results": [ diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/facet/facet_sort_recursive.rs index 213d18624..504f80d12 100644 --- a/crates/milli/src/facet/facet_sort_recursive.rs +++ b/crates/milli/src/facet/facet_sort_recursive.rs @@ -160,7 +160,7 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { })); } } - + // Once all geo candidates have been processed, we can return the others if let Some(not_geo_candidates) = not_geo_candidates.take() { if !not_geo_candidates.is_empty() { From 8326f34ad127629bf33f81945fa621bfc1e16fc0 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 11:35:28 +0200 Subject: [PATCH 038/312] Add analytics --- .../src/routes/indexes/documents.rs | 31 +++++++++++++++---- 1 file changed, 25 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index d9b3f106f..c8198d9a7 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -138,6 +138,8 @@ pub struct DocumentsFetchAggregator { per_document_id: bool, // if a filter was used per_filter: bool, + // if documents were sorted + sort: bool, #[serde(rename = "vector.retrieve_vectors")] retrieve_vectors: bool, @@ -156,16 +158,28 @@ pub struct DocumentsFetchAggregator { #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum DocumentFetchKind { - PerDocumentId { retrieve_vectors: bool }, - Normal { with_filter: bool, limit: usize, offset: usize, retrieve_vectors: bool, ids: usize }, + PerDocumentId { + retrieve_vectors: bool, + sort: bool, + }, + Normal { + with_filter: bool, + limit: usize, + offset: usize, + retrieve_vectors: bool, + sort: bool, + ids: usize, + }, } impl DocumentsFetchAggregator { pub fn from_query(query: &DocumentFetchKind) -> Self { - let (limit, offset, retrieve_vectors) = match query { - DocumentFetchKind::PerDocumentId { retrieve_vectors } => (1, 0, *retrieve_vectors), - DocumentFetchKind::Normal { limit, offset, retrieve_vectors, .. } => { - (*limit, *offset, *retrieve_vectors) + let (limit, offset, retrieve_vectors, sort) = match query { + DocumentFetchKind::PerDocumentId { retrieve_vectors, sort } => { + (1, 0, *retrieve_vectors, *sort) + } + DocumentFetchKind::Normal { limit, offset, retrieve_vectors, sort, .. } => { + (*limit, *offset, *retrieve_vectors, *sort) } }; @@ -179,6 +193,7 @@ impl DocumentsFetchAggregator { per_filter: matches!(query, DocumentFetchKind::Normal { with_filter, .. } if *with_filter), max_limit: limit, max_offset: offset, + sort, retrieve_vectors, max_document_ids: ids, @@ -196,6 +211,7 @@ impl Aggregate for DocumentsFetchAggregator { Box::new(Self { per_document_id: self.per_document_id | new.per_document_id, per_filter: self.per_filter | new.per_filter, + sort: self.sort | new.sort, retrieve_vectors: self.retrieve_vectors | new.retrieve_vectors, max_limit: self.max_limit.max(new.max_limit), max_offset: self.max_offset.max(new.max_offset), @@ -279,6 +295,7 @@ pub async fn get_document( retrieve_vectors: param_retrieve_vectors.0, per_document_id: true, per_filter: false, + sort: false, max_limit: 0, max_offset: 0, max_document_ids: 0, @@ -503,6 +520,7 @@ pub async fn documents_by_query_post( analytics.publish( DocumentsFetchAggregator:: { per_filter: body.filter.is_some(), + sort: body.sort.is_some(), retrieve_vectors: body.retrieve_vectors, max_limit: body.limit, max_offset: body.offset, @@ -603,6 +621,7 @@ pub async fn get_documents( analytics.publish( DocumentsFetchAggregator:: { per_filter: query.filter.is_some(), + sort: query.sort.is_some(), retrieve_vectors: query.retrieve_vectors, max_limit: query.limit, max_offset: query.offset, From 8aacd6374acd2387d2c66f46a09105ec2867e441 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 11:50:01 +0200 Subject: [PATCH 039/312] Optimize geo sort --- crates/milli/src/facet/facet_sort_recursive.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/facet/facet_sort_recursive.rs index 504f80d12..a4d65f91d 100644 --- a/crates/milli/src/facet/facet_sort_recursive.rs +++ b/crates/milli/src/facet/facet_sort_recursive.rs @@ -370,7 +370,7 @@ pub fn recursive_facet_sort<'ctx>( let mut fields = Vec::new(); let fields_ids_map = index.fields_ids_map(rtxn)?; - let geo_candidates = index.geo_faceted_documents_ids(rtxn)?; // TODO: skip when no geo sort + let mut need_geo_candidates = false; for sort in sort { match sort { AscDesc::Asc(Member::Field(field)) => { @@ -387,6 +387,7 @@ pub fn recursive_facet_sort<'ctx>( if let (Some(lat), Some(lng)) = (fields_ids_map.id("_geo.lat"), fields_ids_map.id("_geo.lng")) { + need_geo_candidates = true; fields.push(AscDescId::Geo { field_ids: [lat, lng], target_point, @@ -398,6 +399,7 @@ pub fn recursive_facet_sort<'ctx>( if let (Some(lat), Some(lng)) = (fields_ids_map.id("_geo.lat"), fields_ids_map.id("_geo.lng")) { + need_geo_candidates = true; fields.push(AscDescId::Geo { field_ids: [lat, lng], target_point, @@ -409,6 +411,12 @@ pub fn recursive_facet_sort<'ctx>( // FIXME: Should this return an error if the field is not found? } + let geo_candidates = if need_geo_candidates { + index.geo_faceted_documents_ids(rtxn)? + } else { + RoaringBitmap::new() + }; + let number_db = index.facet_id_f64_docids.remap_key_type::>(); let string_db = index.facet_id_string_docids.remap_key_type::>(); From 283944ea8979e007afb47f6dcd63e245d1542fcb Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 12:03:50 +0200 Subject: [PATCH 040/312] Differentiate between document sort error and search sort error --- crates/meilisearch-types/src/error.rs | 3 ++- .../meilisearch/src/routes/indexes/documents.rs | 2 +- .../meilisearch/src/search/federated/perform.rs | 7 +++---- crates/meilisearch/src/search/mod.rs | 2 +- crates/milli/src/asc_desc.rs | 16 ++++++++++------ crates/milli/src/error.rs | 4 ++-- crates/milli/src/facet/facet_sort_recursive.rs | 2 +- 7 files changed, 20 insertions(+), 16 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 2eb22035e..9cf1b93a0 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -483,7 +483,8 @@ impl ErrorCode for milli::Error { UserError::InvalidVectorsMapType { .. } | UserError::InvalidVectorsEmbedderConf { .. } => Code::InvalidVectorsType, UserError::TooManyVectors(_, _) => Code::TooManyVectors, - UserError::SortError(_) => Code::InvalidSearchSort, + UserError::SortError { search: true, .. } => Code::InvalidSearchSort, + UserError::SortError { search: false, .. } => Code::InvalidDocumentSort, UserError::InvalidMinTypoWordLenSetting(_, _) => { Code::InvalidSettingsTypoTolerance } diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index c8198d9a7..b66eec535 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -663,7 +663,7 @@ fn documents_by_query( let sorts: Vec<_> = match sort.iter().map(|s| milli::AscDesc::from_str(s)).collect() { Ok(sorts) => sorts, Err(asc_desc_error) => { - return Err(milli::Error::from(milli::SortError::from(asc_desc_error)).into()) + return Err(milli::SortError::from(asc_desc_error).into_documents_error().into()) } }; Some(sorts) diff --git a/crates/meilisearch/src/search/federated/perform.rs b/crates/meilisearch/src/search/federated/perform.rs index 5ad64d63c..c0fec01e8 100644 --- a/crates/meilisearch/src/search/federated/perform.rs +++ b/crates/meilisearch/src/search/federated/perform.rs @@ -745,10 +745,9 @@ impl SearchByIndex { match sort.iter().map(|s| milli::AscDesc::from_str(s)).collect() { Ok(sorts) => sorts, Err(asc_desc_error) => { - return Err(milli::Error::from(milli::SortError::from( - asc_desc_error, - )) - .into()) + return Err(milli::SortError::from(asc_desc_error) + .into_search_error() + .into()) } }; Some(sorts) diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 5e543c53f..f57bc9b9a 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -1060,7 +1060,7 @@ pub fn prepare_search<'t>( let sort = match sort.iter().map(|s| AscDesc::from_str(s)).collect() { Ok(sorts) => sorts, Err(asc_desc_error) => { - return Err(milli::Error::from(SortError::from(asc_desc_error)).into()) + return Err(SortError::from(asc_desc_error).into_search_error().into()) } }; diff --git a/crates/milli/src/asc_desc.rs b/crates/milli/src/asc_desc.rs index e75adf83d..999b02511 100644 --- a/crates/milli/src/asc_desc.rs +++ b/crates/milli/src/asc_desc.rs @@ -168,6 +168,16 @@ pub enum SortError { ReservedNameForFilter { name: String }, } +impl SortError { + pub fn into_search_error(self) -> Error { + Error::UserError(UserError::SortError { error: self, search: true }) + } + + pub fn into_documents_error(self) -> Error { + Error::UserError(UserError::SortError { error: self, search: false }) + } +} + impl From for SortError { fn from(error: AscDescError) -> Self { match error { @@ -190,12 +200,6 @@ impl From for SortError { } } -impl From for Error { - fn from(error: SortError) -> Self { - Self::UserError(UserError::SortError(error)) - } -} - #[cfg(test)] mod tests { use big_s::S; diff --git a/crates/milli/src/error.rs b/crates/milli/src/error.rs index 2136ec97e..2624a9824 100644 --- a/crates/milli/src/error.rs +++ b/crates/milli/src/error.rs @@ -272,8 +272,8 @@ and can not be more than 511 bytes.", .document_id.to_string() PrimaryKeyCannotBeChanged(String), #[error(transparent)] SerdeJson(serde_json::Error), - #[error(transparent)] - SortError(#[from] SortError), + #[error("{error}")] + SortError { error: SortError, search: bool }, #[error("An unknown internal document id have been used: `{document_id}`.")] UnknownInternalDocumentId { document_id: DocumentId }, #[error("`minWordSizeForTypos` setting is invalid. `oneTypo` and `twoTypos` fields should be between `0` and `255`, and `twoTypos` should be greater or equals to `oneTypo` but found `oneTypo: {0}` and twoTypos: {1}`.")] diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/facet/facet_sort_recursive.rs index a4d65f91d..19bf5afb9 100644 --- a/crates/milli/src/facet/facet_sort_recursive.rs +++ b/crates/milli/src/facet/facet_sort_recursive.rs @@ -411,7 +411,7 @@ pub fn recursive_facet_sort<'ctx>( // FIXME: Should this return an error if the field is not found? } - let geo_candidates = if need_geo_candidates { + let geo_candidates = if need_geo_candidates { index.geo_faceted_documents_ids(rtxn)? } else { RoaringBitmap::new() From 8419fd9b3b8dbadf8edfed87c6dca3bb1d798f07 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 13:42:38 +0200 Subject: [PATCH 041/312] Ditch usage of check_sort_criteria --- crates/meilisearch-types/src/error.rs | 3 +- crates/milli/src/error.rs | 18 +++- .../milli/src/facet/facet_sort_recursive.rs | 89 +++++++++---------- crates/milli/src/search/new/mod.rs | 4 +- 4 files changed, 63 insertions(+), 51 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 9cf1b93a0..e43b28fc6 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -467,7 +467,8 @@ impl ErrorCode for milli::Error { UserError::InvalidDistinctAttribute { .. } => Code::InvalidSearchDistinct, UserError::SortRankingRuleMissing => Code::InvalidSearchSort, UserError::InvalidFacetsDistribution { .. } => Code::InvalidSearchFacets, - UserError::InvalidSortableAttribute { .. } => Code::InvalidSearchSort, + UserError::InvalidSearchSortableAttribute { .. } => Code::InvalidSearchSort, + UserError::InvalidDocumentSortableAttribute { .. } => Code::InvalidDocumentSort, UserError::InvalidSearchableAttribute { .. } => { Code::InvalidSearchAttributesToSearchOn } diff --git a/crates/milli/src/error.rs b/crates/milli/src/error.rs index 2624a9824..f3b390690 100644 --- a/crates/milli/src/error.rs +++ b/crates/milli/src/error.rs @@ -191,7 +191,21 @@ and can not be more than 511 bytes.", .document_id.to_string() ), } )] - InvalidSortableAttribute { field: String, valid_fields: BTreeSet, hidden_fields: bool }, + InvalidSearchSortableAttribute { + field: String, + valid_fields: BTreeSet, + hidden_fields: bool, + }, + #[error("Attribute `{}` is not sortable. {}", + .field, + match .sortable_fields.is_empty() { + true => "This index does not have configured sortable attributes.".to_string(), + false => format!("Available sortable attributes are: `{}`.", + sortable_fields.iter().map(AsRef::as_ref).collect::>().join(", ") + ), + } + )] + InvalidDocumentSortableAttribute { field: String, sortable_fields: BTreeSet }, #[error("Attribute `{}` is not filterable and thus, cannot be used as distinct attribute. {}", .field, match (.valid_patterns.is_empty(), .matching_rule_index) { @@ -614,7 +628,7 @@ fn conditionally_lookup_for_error_message() { ]; for (list, suffix) in messages { - let err = UserError::InvalidSortableAttribute { + let err = UserError::InvalidSearchSortableAttribute { field: "name".to_string(), valid_fields: list, hidden_fields: false, diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/facet/facet_sort_recursive.rs index 19bf5afb9..ab62ebcfd 100644 --- a/crates/milli/src/facet/facet_sort_recursive.rs +++ b/crates/milli/src/facet/facet_sort_recursive.rs @@ -1,16 +1,15 @@ -use std::collections::VecDeque; +use std::collections::{BTreeSet, VecDeque}; use crate::{ + constants::RESERVED_GEO_FIELD_NAME, documents::{geo_sort::next_bucket, GeoSortParameter}, heed_codec::{ facet::{FacetGroupKeyCodec, FacetGroupValueCodec}, BytesRefCodec, }, - search::{ - facet::{ascending_facet_sort, descending_facet_sort}, - new::check_sort_criteria, - }, - AscDesc, DocumentId, Member, + is_faceted, + search::facet::{ascending_facet_sort, descending_facet_sort}, + AscDesc, DocumentId, Member, UserError, }; use heed::Database; use roaring::RoaringBitmap; @@ -366,49 +365,47 @@ pub fn recursive_facet_sort<'ctx>( sort: Vec, candidates: &'ctx RoaringBitmap, ) -> crate::Result> { - check_sort_criteria(index, rtxn, Some(&sort))?; - - let mut fields = Vec::new(); + let sortable_fields: BTreeSet<_> = index.sortable_fields(rtxn)?.into_iter().collect(); let fields_ids_map = index.fields_ids_map(rtxn)?; + + let mut fields = Vec::new(); let mut need_geo_candidates = false; - for sort in sort { - match sort { - AscDesc::Asc(Member::Field(field)) => { - if let Some(field_id) = fields_ids_map.id(&field) { - fields.push(AscDescId::Facet { field_id, ascending: true }); - } - } - AscDesc::Desc(Member::Field(field)) => { - if let Some(field_id) = fields_ids_map.id(&field) { - fields.push(AscDescId::Facet { field_id, ascending: false }); - } - } - AscDesc::Asc(Member::Geo(target_point)) => { - if let (Some(lat), Some(lng)) = - (fields_ids_map.id("_geo.lat"), fields_ids_map.id("_geo.lng")) - { - need_geo_candidates = true; - fields.push(AscDescId::Geo { - field_ids: [lat, lng], - target_point, - ascending: true, - }); - } - } - AscDesc::Desc(Member::Geo(target_point)) => { - if let (Some(lat), Some(lng)) = - (fields_ids_map.id("_geo.lat"), fields_ids_map.id("_geo.lng")) - { - need_geo_candidates = true; - fields.push(AscDescId::Geo { - field_ids: [lat, lng], - target_point, - ascending: false, - }); - } - } + for asc_desc in sort { + let (field, geofield) = match asc_desc { + AscDesc::Asc(Member::Field(field)) => (Some((field, true)), None), + AscDesc::Desc(Member::Field(field)) => (Some((field, false)), None), + AscDesc::Asc(Member::Geo(target_point)) => (None, Some((target_point, true))), + AscDesc::Desc(Member::Geo(target_point)) => (None, Some((target_point, false))), }; - // FIXME: Should this return an error if the field is not found? + if let Some((field, ascending)) = field { + if is_faceted(&field, &sortable_fields) { + if let Some(field_id) = fields_ids_map.id(&field) { + fields.push(AscDescId::Facet { field_id, ascending }); + continue; + } + } + return Err(UserError::InvalidDocumentSortableAttribute { + field: field.to_string(), + sortable_fields: sortable_fields.clone(), + } + .into()); + } + if let Some((target_point, ascending)) = geofield { + if sortable_fields.contains(RESERVED_GEO_FIELD_NAME) { + if let (Some(lat), Some(lng)) = + (fields_ids_map.id("_geo.lat"), fields_ids_map.id("_geo.lng")) + { + need_geo_candidates = true; + fields.push(AscDescId::Geo { field_ids: [lat, lng], target_point, ascending }); + continue; + } + } + return Err(UserError::InvalidDocumentSortableAttribute { + field: RESERVED_GEO_FIELD_NAME.to_string(), + sortable_fields: sortable_fields.clone(), + } + .into()); + } } let geo_candidates = if need_geo_candidates { diff --git a/crates/milli/src/search/new/mod.rs b/crates/milli/src/search/new/mod.rs index b5258413e..3983aa07a 100644 --- a/crates/milli/src/search/new/mod.rs +++ b/crates/milli/src/search/new/mod.rs @@ -903,7 +903,7 @@ pub(crate) fn check_sort_criteria( let (valid_fields, hidden_fields) = index.remove_hidden_fields(rtxn, sortable_fields)?; - return Err(UserError::InvalidSortableAttribute { + return Err(UserError::InvalidSearchSortableAttribute { field: field.to_string(), valid_fields, hidden_fields, @@ -914,7 +914,7 @@ pub(crate) fn check_sort_criteria( let (valid_fields, hidden_fields) = index.remove_hidden_fields(rtxn, sortable_fields)?; - return Err(UserError::InvalidSortableAttribute { + return Err(UserError::InvalidSearchSortableAttribute { field: RESERVED_GEO_FIELD_NAME.to_string(), valid_fields, hidden_fields, From 280c3907bebc9ad0077e4e3372428c8c2ce5b810 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 13:58:37 +0200 Subject: [PATCH 042/312] Add test to sort the unsortable --- .../tests/documents/get_documents.rs | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/documents/get_documents.rs b/crates/meilisearch/tests/documents/get_documents.rs index 2267b8f5d..5ee838232 100644 --- a/crates/meilisearch/tests/documents/get_documents.rs +++ b/crates/meilisearch/tests/documents/get_documents.rs @@ -366,7 +366,27 @@ async fn get_document_geosorted() { "#); } -// TODO test on not sortable attributes +#[actix_rt::test] +async fn get_document_sort_the_unsortable() { + let index = shared_index_with_test_set().await; + + let (response, _code) = index + .get_all_documents(GetAllDocumentsOptions { + fields: Some(vec!["id", "name"]), + sort: Some(vec!["name:asc"]), + ..Default::default() + }) + .await; + + snapshot!(json_string!(response), @r#" + { + "message": "Attribute `name` is not sortable. This index does not have configured sortable attributes.", + "code": "invalid_document_sort", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_document_sort" + } + "#); +} #[actix_rt::test] async fn error_get_unexisting_index_all_documents() { From 9f55708d84ddd55cdcca5e442a97ab32fa34ad66 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 13:58:56 +0200 Subject: [PATCH 043/312] Format --- crates/milli/src/facet/facet_sort_recursive.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/facet/facet_sort_recursive.rs index ab62ebcfd..596ce6335 100644 --- a/crates/milli/src/facet/facet_sort_recursive.rs +++ b/crates/milli/src/facet/facet_sort_recursive.rs @@ -367,7 +367,7 @@ pub fn recursive_facet_sort<'ctx>( ) -> crate::Result> { let sortable_fields: BTreeSet<_> = index.sortable_fields(rtxn)?.into_iter().collect(); let fields_ids_map = index.fields_ids_map(rtxn)?; - + let mut fields = Vec::new(); let mut need_geo_candidates = false; for asc_desc in sort { From d85480de898d5f6b1c829fdd1fbed2bc381a1864 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 14:05:47 +0200 Subject: [PATCH 044/312] Move sort code out of facet --- crates/meilisearch/src/routes/indexes/documents.rs | 4 ++-- crates/milli/src/documents/mod.rs | 1 + .../src/{facet/facet_sort_recursive.rs => documents/sort.rs} | 2 +- crates/milli/src/facet/mod.rs | 1 - 4 files changed, 4 insertions(+), 4 deletions(-) rename crates/milli/src/{facet/facet_sort_recursive.rs => documents/sort.rs} (99%) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index b66eec535..e8499a789 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -18,7 +18,7 @@ use meilisearch_types::error::deserr_codes::*; use meilisearch_types::error::{Code, ResponseError}; use meilisearch_types::heed::RoTxn; use meilisearch_types::index_uid::IndexUid; -use meilisearch_types::milli::facet::facet_sort_recursive::recursive_facet_sort; +use meilisearch_types::milli::documents::sort::recursive_sort; use meilisearch_types::milli::update::IndexDocumentsMethod; use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors; use meilisearch_types::milli::{AscDesc, DocumentId}; @@ -1576,7 +1576,7 @@ fn retrieve_documents>( let facet_sort; let (it, number_of_documents) = if let Some(sort) = sort_criteria { let number_of_documents = candidates.len(); - facet_sort = recursive_facet_sort(index, &rtxn, sort, &candidates)?; + facet_sort = recursive_sort(index, &rtxn, sort, &candidates)?; let iter = facet_sort.iter()?; ( itertools::Either::Left(some_documents( diff --git a/crates/milli/src/documents/mod.rs b/crates/milli/src/documents/mod.rs index b515c4e98..7a4babfa8 100644 --- a/crates/milli/src/documents/mod.rs +++ b/crates/milli/src/documents/mod.rs @@ -4,6 +4,7 @@ pub mod geo_sort; mod primary_key; mod reader; mod serde_impl; +pub mod sort; use std::fmt::Debug; use std::io; diff --git a/crates/milli/src/facet/facet_sort_recursive.rs b/crates/milli/src/documents/sort.rs similarity index 99% rename from crates/milli/src/facet/facet_sort_recursive.rs rename to crates/milli/src/documents/sort.rs index 596ce6335..4008a37a4 100644 --- a/crates/milli/src/facet/facet_sort_recursive.rs +++ b/crates/milli/src/documents/sort.rs @@ -359,7 +359,7 @@ impl<'ctx> SortedDocuments<'ctx> { } } -pub fn recursive_facet_sort<'ctx>( +pub fn recursive_sort<'ctx>( index: &'ctx crate::Index, rtxn: &'ctx heed::RoTxn<'ctx>, sort: Vec, diff --git a/crates/milli/src/facet/mod.rs b/crates/milli/src/facet/mod.rs index 8b0b9a25e..274d2588d 100644 --- a/crates/milli/src/facet/mod.rs +++ b/crates/milli/src/facet/mod.rs @@ -1,4 +1,3 @@ -pub mod facet_sort_recursive; mod facet_type; mod facet_value; pub mod value_encoding; From 73dfeefc7ce3fbab172730bc1bfc37404dc21e87 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 14:08:46 +0200 Subject: [PATCH 045/312] Remove plural form --- crates/meilisearch/src/routes/indexes/documents.rs | 2 +- crates/milli/src/asc_desc.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index e8499a789..9c8d28e04 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -663,7 +663,7 @@ fn documents_by_query( let sorts: Vec<_> = match sort.iter().map(|s| milli::AscDesc::from_str(s)).collect() { Ok(sorts) => sorts, Err(asc_desc_error) => { - return Err(milli::SortError::from(asc_desc_error).into_documents_error().into()) + return Err(milli::SortError::from(asc_desc_error).into_document_error().into()) } }; Some(sorts) diff --git a/crates/milli/src/asc_desc.rs b/crates/milli/src/asc_desc.rs index 999b02511..d7288faa3 100644 --- a/crates/milli/src/asc_desc.rs +++ b/crates/milli/src/asc_desc.rs @@ -173,7 +173,7 @@ impl SortError { Error::UserError(UserError::SortError { error: self, search: true }) } - pub fn into_documents_error(self) -> Error { + pub fn into_document_error(self) -> Error { Error::UserError(UserError::SortError { error: self, search: false }) } } From 27cc3573624d6c3bcd80066c4a95b4d455690ba3 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 14:21:55 +0200 Subject: [PATCH 046/312] Document code --- crates/milli/src/documents/sort.rs | 302 +++++++++++++++-------------- 1 file changed, 155 insertions(+), 147 deletions(-) diff --git a/crates/milli/src/documents/sort.rs b/crates/milli/src/documents/sort.rs index 4008a37a4..59858caad 100644 --- a/crates/milli/src/documents/sort.rs +++ b/crates/milli/src/documents/sort.rs @@ -20,6 +20,158 @@ enum AscDescId { Geo { field_ids: [u16; 2], target_point: [f64; 2], ascending: bool }, } +/// A [`SortedDocumentsIterator`] allows efficient access to a continuous range of sorted documents. +/// This is ideal in the context of paginated queries in which only a small number of documents are needed at a time. +/// Search operations will only be performed upon access. +pub enum SortedDocumentsIterator<'ctx> { + Leaf { + /// The exact number of documents remaining + size: usize, + values: Box + 'ctx>, + }, + Branch { + /// The current child, got from the children iterator + current_child: Option>>, + /// The exact number of documents remaining, excluding documents in the current child + next_children_size: usize, + /// Iterators to become the current child once it is exhausted + next_children: + Box>> + 'ctx>, + }, +} + +impl SortedDocumentsIterator<'_> { + /// Takes care of updating the current child if it is `None`, and also updates the size + fn update_current<'ctx>( + current_child: &mut Option>>, + next_children_size: &mut usize, + next_children: &mut Box< + dyn Iterator>> + 'ctx, + >, + ) -> crate::Result<()> { + if current_child.is_none() { + *current_child = match next_children.next() { + Some(Ok(builder)) => { + let next_child = Box::new(builder.build()?); + *next_children_size -= next_child.size_hint().0; + Some(next_child) + } + Some(Err(e)) => return Err(e), + None => return Ok(()), + }; + } + Ok(()) + } +} + +impl Iterator for SortedDocumentsIterator<'_> { + type Item = crate::Result; + + /// Implementing the `nth` method allows for efficient access to the nth document in the sorted order. + /// It's used by `skip` internally. + /// The default implementation of `nth` would iterate over all children, which is inefficient for large datasets. + /// This implementation will jump over whole chunks of children until it gets close. + fn nth(&mut self, n: usize) -> Option { + // If it's at the leaf level, just forward the call to the values iterator + let (current_child, next_children, next_children_size) = match self { + SortedDocumentsIterator::Leaf { values, size } => { + *size = size.saturating_sub(n); + return values.nth(n).map(Ok); + } + SortedDocumentsIterator::Branch { + current_child, + next_children, + next_children_size, + } => (current_child, next_children, next_children_size), + }; + + // Otherwise don't directly iterate over children, skip them if we know we will go further + let mut to_skip = n - 1; + while to_skip > 0 { + if let Err(e) = SortedDocumentsIterator::update_current( + current_child, + next_children_size, + next_children, + ) { + return Some(Err(e)); + } + let Some(inner) = current_child else { + return None; // No more inner iterators, everything has been consumed. + }; + + if to_skip >= inner.size_hint().0 { + // The current child isn't large enough to contain the nth element. + // Skip it and continue with the next one. + to_skip -= inner.size_hint().0; + *current_child = None; + continue; + } else { + // The current iterator is large enough, so we can forward the call to it. + return inner.nth(to_skip + 1); + } + } + + self.next() + } + + /// Iterators need to keep track of their size so that they can be skipped efficiently by the `nth` method. + fn size_hint(&self) -> (usize, Option) { + let size = match self { + SortedDocumentsIterator::Leaf { size, .. } => *size, + SortedDocumentsIterator::Branch { + next_children_size, + current_child: Some(current_child), + .. + } => current_child.size_hint().0 + next_children_size, + SortedDocumentsIterator::Branch { next_children_size, current_child: None, .. } => { + *next_children_size + } + }; + + (size, Some(size)) + } + + fn next(&mut self) -> Option { + match self { + SortedDocumentsIterator::Leaf { values, size } => { + let result = values.next().map(Ok); + if result.is_some() { + *size -= 1; + } + result + } + SortedDocumentsIterator::Branch { + current_child, + next_children_size, + next_children, + } => { + let mut result = None; + while result.is_none() { + // Ensure we have selected an iterator to work with + if let Err(e) = SortedDocumentsIterator::update_current( + current_child, + next_children_size, + next_children, + ) { + return Some(Err(e)); + } + let Some(inner) = current_child else { + return None; + }; + + result = inner.next(); + + // If the current iterator is exhausted, we need to try the next one + if result.is_none() { + *current_child = None; + } + } + result + } + } + } +} + /// Builder for a [`SortedDocumentsIterator`]. /// Most builders won't ever be built, because pagination will skip them. pub struct SortedDocumentsIteratorBuilder<'ctx> { @@ -57,6 +209,7 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { } } + /// Builds a [`SortedDocumentsIterator`] based on the results of a facet sort. fn build_facet( self, field_id: u16, @@ -108,6 +261,7 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { }) } + /// Builds a [`SortedDocumentsIterator`] based on the (lazy) results of a geo sort. fn build_geo( self, field_ids: [u16; 2], @@ -186,153 +340,6 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { } } -/// A [`SortedDocumentsIterator`] allows efficient access to a continuous range of sorted documents. -/// This is ideal in the context of paginated queries in which only a small number of documents are needed at a time. -/// Search operations will only be performed upon access. -pub enum SortedDocumentsIterator<'ctx> { - Leaf { - /// The exact number of documents remaining - size: usize, - values: Box + 'ctx>, - }, - Branch { - /// The current child, got from the children iterator - current_child: Option>>, - /// The exact number of documents remaining, excluding documents in the current child - next_children_size: usize, - /// Iterators to become the current child once it is exhausted - next_children: - Box>> + 'ctx>, - }, -} - -impl SortedDocumentsIterator<'_> { - /// Takes care of updating the current child if it is `None`, and also updates the size - fn update_current<'ctx>( - current_child: &mut Option>>, - next_children_size: &mut usize, - next_children: &mut Box< - dyn Iterator>> + 'ctx, - >, - ) -> crate::Result<()> { - if current_child.is_none() { - *current_child = match next_children.next() { - Some(Ok(builder)) => { - let next_child = Box::new(builder.build()?); - *next_children_size -= next_child.size_hint().0; - Some(next_child) - } - Some(Err(e)) => return Err(e), - None => return Ok(()), - }; - } - Ok(()) - } -} - -impl Iterator for SortedDocumentsIterator<'_> { - type Item = crate::Result; - - fn nth(&mut self, n: usize) -> Option { - // If it's at the leaf level, just forward the call to the values iterator - let (current_child, next_children, next_children_size) = match self { - SortedDocumentsIterator::Leaf { values, size } => { - *size = size.saturating_sub(n); - return values.nth(n).map(Ok); - } - SortedDocumentsIterator::Branch { - current_child, - next_children, - next_children_size, - } => (current_child, next_children, next_children_size), - }; - - // Otherwise don't directly iterate over children, skip them if we know we will go further - let mut to_skip = n - 1; - while to_skip > 0 { - if let Err(e) = SortedDocumentsIterator::update_current( - current_child, - next_children_size, - next_children, - ) { - return Some(Err(e)); - } - let Some(inner) = current_child else { - return None; // No more inner iterators, everything has been consumed. - }; - - if to_skip >= inner.size_hint().0 { - // The current child isn't large enough to contain the nth element. - // Skip it and continue with the next one. - to_skip -= inner.size_hint().0; - *current_child = None; - continue; - } else { - // The current iterator is large enough, so we can forward the call to it. - return inner.nth(to_skip + 1); - } - } - - self.next() - } - - fn size_hint(&self) -> (usize, Option) { - let size = match self { - SortedDocumentsIterator::Leaf { size, .. } => *size, - SortedDocumentsIterator::Branch { - next_children_size, - current_child: Some(current_child), - .. - } => current_child.size_hint().0 + next_children_size, - SortedDocumentsIterator::Branch { next_children_size, current_child: None, .. } => { - *next_children_size - } - }; - - (size, Some(size)) - } - - fn next(&mut self) -> Option { - match self { - SortedDocumentsIterator::Leaf { values, size } => { - let result = values.next().map(Ok); - if result.is_some() { - *size -= 1; - } - result - } - SortedDocumentsIterator::Branch { - current_child, - next_children_size, - next_children, - } => { - let mut result = None; - while result.is_none() { - // Ensure we have selected an iterator to work with - if let Err(e) = SortedDocumentsIterator::update_current( - current_child, - next_children_size, - next_children, - ) { - return Some(Err(e)); - } - let Some(inner) = current_child else { - return None; - }; - - result = inner.next(); - - // If the current iterator is exhausted, we need to try the next one - if result.is_none() { - *current_child = None; - } - } - result - } - } - } -} - /// A structure owning the data needed during the lifetime of a [`SortedDocumentsIterator`]. pub struct SortedDocuments<'ctx> { index: &'ctx crate::Index, @@ -368,6 +375,7 @@ pub fn recursive_sort<'ctx>( let sortable_fields: BTreeSet<_> = index.sortable_fields(rtxn)?.into_iter().collect(); let fields_ids_map = index.fields_ids_map(rtxn)?; + // Retrieve the field ids that are used for sorting let mut fields = Vec::new(); let mut need_geo_candidates = false; for asc_desc in sort { From e92b6beb2011f8488d197d9a5e31d3e9b3f88dd6 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 14:26:55 +0200 Subject: [PATCH 047/312] Revert making check_sort_criteria usable without a search context --- crates/milli/src/search/new/mod.rs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/crates/milli/src/search/new/mod.rs b/crates/milli/src/search/new/mod.rs index 3983aa07a..ecc51161d 100644 --- a/crates/milli/src/search/new/mod.rs +++ b/crates/milli/src/search/new/mod.rs @@ -638,7 +638,7 @@ pub fn execute_vector_search( time_budget: TimeBudget, ranking_score_threshold: Option, ) -> Result { - check_sort_criteria(ctx.index, ctx.txn, sort_criteria.as_deref())?; + check_sort_criteria(ctx, sort_criteria.as_ref())?; // FIXME: input universe = universe & documents_with_vectors // for now if we're computing embeddings for ALL documents, we can assume that this is just universe @@ -702,7 +702,7 @@ pub fn execute_search( ranking_score_threshold: Option, locales: Option<&Vec>, ) -> Result { - check_sort_criteria(ctx.index, ctx.txn, sort_criteria.as_deref())?; + check_sort_criteria(ctx, sort_criteria.as_ref())?; let mut used_negative_operator = false; let mut located_query_terms = None; @@ -873,9 +873,8 @@ pub fn execute_search( } pub(crate) fn check_sort_criteria( - index: &Index, - rtxn: &RoTxn<'_>, - sort_criteria: Option<&[AscDesc]>, + ctx: &SearchContext<'_>, + sort_criteria: Option<&Vec>, ) -> Result<()> { let sort_criteria = if let Some(sort_criteria) = sort_criteria { sort_criteria @@ -889,19 +888,19 @@ pub(crate) fn check_sort_criteria( // We check that the sort ranking rule exists and throw an // error if we try to use it and that it doesn't. - let sort_ranking_rule_missing = !index.criteria(rtxn)?.contains(&crate::Criterion::Sort); + let sort_ranking_rule_missing = !ctx.index.criteria(ctx.txn)?.contains(&crate::Criterion::Sort); if sort_ranking_rule_missing { return Err(UserError::SortRankingRuleMissing.into()); } // We check that we are allowed to use the sort criteria, we check // that they are declared in the sortable fields. - let sortable_fields = index.sortable_fields(rtxn)?; + let sortable_fields = ctx.index.sortable_fields(ctx.txn)?; for asc_desc in sort_criteria { match asc_desc.member() { Member::Field(ref field) if !crate::is_faceted(field, &sortable_fields) => { let (valid_fields, hidden_fields) = - index.remove_hidden_fields(rtxn, sortable_fields)?; + ctx.index.remove_hidden_fields(ctx.txn, sortable_fields)?; return Err(UserError::InvalidSearchSortableAttribute { field: field.to_string(), @@ -912,7 +911,7 @@ pub(crate) fn check_sort_criteria( } Member::Geo(_) if !sortable_fields.contains(RESERVED_GEO_FIELD_NAME) => { let (valid_fields, hidden_fields) = - index.remove_hidden_fields(rtxn, sortable_fields)?; + ctx.index.remove_hidden_fields(ctx.txn, sortable_fields)?; return Err(UserError::InvalidSearchSortableAttribute { field: RESERVED_GEO_FIELD_NAME.to_string(), From 7ae9a4afee3981ee98bfbde8777b0f03f11388f6 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 15:42:43 +0200 Subject: [PATCH 048/312] Add a test for issue #5274 --- crates/meilisearch/tests/search/pagination.rs | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/crates/meilisearch/tests/search/pagination.rs b/crates/meilisearch/tests/search/pagination.rs index c0752e7ec..6dd8b3181 100644 --- a/crates/meilisearch/tests/search/pagination.rs +++ b/crates/meilisearch/tests/search/pagination.rs @@ -1,6 +1,7 @@ use super::shared_index_with_documents; use crate::common::Server; use crate::json; +use meili_snap::{json_string, snapshot}; #[actix_rt::test] async fn default_search_should_return_estimated_total_hit() { @@ -133,3 +134,61 @@ async fn ensure_placeholder_search_hit_count_valid() { .await; } } + +#[actix_rt::test] +async fn test_issue_5274() { + let server = Server::new_shared(); + let index = server.unique_index(); + + let documents = json!([ + { + "id": 1, + "title": "Document 1", + "content": "This is the first." + }, + { + "id": 2, + "title": "Document 2", + "content": "This is the second doc." + } + ]); + let (task, _code) = index.add_documents(documents, None).await; + server.wait_task(task.uid()).await.succeeded(); + + // Find out the lowest ranking score among the documents + let (rep, _status) = index + .search_post(json!({"q": "doc", "page": 1, "hitsPerPage": 2, "showRankingScore": true})) + .await; + let hits = rep["hits"].as_array().expect("Missing hits array"); + let second_hit = hits.get(1).expect("Missing second hit"); + let ranking_score = second_hit + .get("_rankingScore") + .expect("Missing _rankingScore field") + .as_f64() + .expect("Expected _rankingScore to be a f64"); + + // Search with a ranking score threshold just above and expect to be a single hit + let (rep, _status) = index + .search_post(json!({"q": "doc", "page": 1, "hitsPerPage": 1, "rankingScoreThreshold": ranking_score + 0.0001})) + .await; + + snapshot!(json_string!(rep, { + ".processingTimeMs" => "[ignored]", + }), @r#" + { + "hits": [ + { + "id": 2, + "title": "Document 2", + "content": "This is the second doc." + } + ], + "query": "doc", + "processingTimeMs": "[ignored]", + "hitsPerPage": 1, + "page": 1, + "totalPages": 1, + "totalHits": 1 + } + "#); +} From dedae94102f7960d22befe0bf68edfba40fea3ee Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 16:22:25 +0200 Subject: [PATCH 049/312] Fix #5274 --- crates/milli/src/search/mod.rs | 1 + crates/milli/src/search/new/bucket_sort.rs | 5 ++++- crates/milli/src/search/new/mod.rs | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index 62183afc3..ecb9af852 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -236,6 +236,7 @@ impl<'a> Search<'a> { &mut ctx, vector, self.scoring_strategy, + self.exhaustive_number_hits, universe, &self.sort_criteria, &self.distinct, diff --git a/crates/milli/src/search/new/bucket_sort.rs b/crates/milli/src/search/new/bucket_sort.rs index 3c26cad5c..f4fd62825 100644 --- a/crates/milli/src/search/new/bucket_sort.rs +++ b/crates/milli/src/search/new/bucket_sort.rs @@ -32,6 +32,7 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>( logger: &mut dyn SearchLogger, time_budget: TimeBudget, ranking_score_threshold: Option, + exhaustive_number_hits: bool, ) -> Result { logger.initial_query(query); logger.ranking_rules(&ranking_rules); @@ -159,7 +160,9 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>( }; } - while valid_docids.len() < length { + while valid_docids.len() < length + || (exhaustive_number_hits && ranking_score_threshold.is_some()) + { if time_budget.exceeded() { loop { let bucket = std::mem::take(&mut ranking_rule_universes[cur_ranking_rule_index]); diff --git a/crates/milli/src/search/new/mod.rs b/crates/milli/src/search/new/mod.rs index a65b4076b..2c6fe5c3c 100644 --- a/crates/milli/src/search/new/mod.rs +++ b/crates/milli/src/search/new/mod.rs @@ -626,6 +626,7 @@ pub fn execute_vector_search( ctx: &mut SearchContext<'_>, vector: &[f32], scoring_strategy: ScoringStrategy, + exhaustive_number_hits: bool, universe: RoaringBitmap, sort_criteria: &Option>, distinct: &Option, @@ -669,6 +670,7 @@ pub fn execute_vector_search( placeholder_search_logger, time_budget, ranking_score_threshold, + exhaustive_number_hits, )?; Ok(PartialSearchResult { @@ -825,6 +827,7 @@ pub fn execute_search( query_graph_logger, time_budget, ranking_score_threshold, + exhaustive_number_hits, )? } else { let ranking_rules = @@ -841,6 +844,7 @@ pub fn execute_search( placeholder_search_logger, time_budget, ranking_score_threshold, + exhaustive_number_hits, )? }; From 600178c5abb02d493937597ba12fa31419c933ab Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 1 Jul 2025 18:33:09 +0200 Subject: [PATCH 050/312] Still limit to max hits --- crates/meilisearch/src/search/mod.rs | 1 + crates/milli/src/search/hybrid.rs | 1 + crates/milli/src/search/mod.rs | 11 +++++++++++ crates/milli/src/search/new/bucket_sort.rs | 6 +++++- crates/milli/src/search/new/matches/mod.rs | 1 + crates/milli/src/search/new/mod.rs | 5 +++++ 6 files changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 5e543c53f..e1cfc542b 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -1020,6 +1020,7 @@ pub fn prepare_search<'t>( .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS); search.exhaustive_number_hits(is_finite_pagination); + search.max_total_hits(Some(max_total_hits)); search.scoring_strategy( if query.show_ranking_score || query.show_ranking_score_details diff --git a/crates/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs index b63f6288f..5fc228807 100644 --- a/crates/milli/src/search/hybrid.rs +++ b/crates/milli/src/search/hybrid.rs @@ -209,6 +209,7 @@ impl Search<'_> { scoring_strategy: ScoringStrategy::Detailed, words_limit: self.words_limit, exhaustive_number_hits: self.exhaustive_number_hits, + max_total_hits: self.max_total_hits, rtxn: self.rtxn, index: self.index, semantic: self.semantic.clone(), diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index ecb9af852..2192ea9fd 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -51,6 +51,7 @@ pub struct Search<'a> { scoring_strategy: ScoringStrategy, words_limit: usize, exhaustive_number_hits: bool, + max_total_hits: Option, rtxn: &'a heed::RoTxn<'a>, index: &'a Index, semantic: Option, @@ -73,6 +74,7 @@ impl<'a> Search<'a> { terms_matching_strategy: TermsMatchingStrategy::default(), scoring_strategy: Default::default(), exhaustive_number_hits: false, + max_total_hits: None, words_limit: 10, rtxn, index, @@ -163,6 +165,11 @@ impl<'a> Search<'a> { self } + pub fn max_total_hits(&mut self, max_total_hits: Option) -> &mut Search<'a> { + self.max_total_hits = max_total_hits; + self + } + pub fn time_budget(&mut self, time_budget: TimeBudget) -> &mut Search<'a> { self.time_budget = time_budget; self @@ -237,6 +244,7 @@ impl<'a> Search<'a> { vector, self.scoring_strategy, self.exhaustive_number_hits, + self.max_total_hits, universe, &self.sort_criteria, &self.distinct, @@ -256,6 +264,7 @@ impl<'a> Search<'a> { self.terms_matching_strategy, self.scoring_strategy, self.exhaustive_number_hits, + self.max_total_hits, universe, &self.sort_criteria, &self.distinct, @@ -309,6 +318,7 @@ impl fmt::Debug for Search<'_> { scoring_strategy, words_limit, exhaustive_number_hits, + max_total_hits, rtxn: _, index: _, semantic, @@ -328,6 +338,7 @@ impl fmt::Debug for Search<'_> { .field("terms_matching_strategy", terms_matching_strategy) .field("scoring_strategy", scoring_strategy) .field("exhaustive_number_hits", exhaustive_number_hits) + .field("max_total_hits", max_total_hits) .field("words_limit", words_limit) .field( "semantic.embedder_name", diff --git a/crates/milli/src/search/new/bucket_sort.rs b/crates/milli/src/search/new/bucket_sort.rs index f4fd62825..298983091 100644 --- a/crates/milli/src/search/new/bucket_sort.rs +++ b/crates/milli/src/search/new/bucket_sort.rs @@ -33,6 +33,7 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>( time_budget: TimeBudget, ranking_score_threshold: Option, exhaustive_number_hits: bool, + max_total_hits: Option, ) -> Result { logger.initial_query(query); logger.ranking_rules(&ranking_rules); @@ -160,8 +161,11 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>( }; } + let max_total_hits = max_total_hits.unwrap_or(usize::MAX); while valid_docids.len() < length - || (exhaustive_number_hits && ranking_score_threshold.is_some()) + || (exhaustive_number_hits + && ranking_score_threshold.is_some() + && valid_docids.len() < max_total_hits) { if time_budget.exceeded() { loop { diff --git a/crates/milli/src/search/new/matches/mod.rs b/crates/milli/src/search/new/matches/mod.rs index 2d6f2cf17..66f65f5e5 100644 --- a/crates/milli/src/search/new/matches/mod.rs +++ b/crates/milli/src/search/new/matches/mod.rs @@ -510,6 +510,7 @@ mod tests { crate::TermsMatchingStrategy::default(), crate::score_details::ScoringStrategy::Skip, false, + None, universe, &None, &None, diff --git a/crates/milli/src/search/new/mod.rs b/crates/milli/src/search/new/mod.rs index 2c6fe5c3c..047d08202 100644 --- a/crates/milli/src/search/new/mod.rs +++ b/crates/milli/src/search/new/mod.rs @@ -627,6 +627,7 @@ pub fn execute_vector_search( vector: &[f32], scoring_strategy: ScoringStrategy, exhaustive_number_hits: bool, + max_total_hits: Option, universe: RoaringBitmap, sort_criteria: &Option>, distinct: &Option, @@ -671,6 +672,7 @@ pub fn execute_vector_search( time_budget, ranking_score_threshold, exhaustive_number_hits, + max_total_hits, )?; Ok(PartialSearchResult { @@ -691,6 +693,7 @@ pub fn execute_search( terms_matching_strategy: TermsMatchingStrategy, scoring_strategy: ScoringStrategy, exhaustive_number_hits: bool, + max_total_hits: Option, mut universe: RoaringBitmap, sort_criteria: &Option>, distinct: &Option, @@ -828,6 +831,7 @@ pub fn execute_search( time_budget, ranking_score_threshold, exhaustive_number_hits, + max_total_hits, )? } else { let ranking_rules = @@ -845,6 +849,7 @@ pub fn execute_search( time_budget, ranking_score_threshold, exhaustive_number_hits, + max_total_hits, )? }; From 5a675bcb827300632262223b0c9d16525de17320 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 2 Jul 2025 11:50:32 +0200 Subject: [PATCH 051/312] Add benchmarks --- crates/benchmarks/Cargo.toml | 5 ++ crates/benchmarks/benches/sort.rs | 108 +++++++++++++++++++++++++++++ crates/benchmarks/benches/utils.rs | 98 +++++++++++++++++++++----- 3 files changed, 195 insertions(+), 16 deletions(-) create mode 100644 crates/benchmarks/benches/sort.rs diff --git a/crates/benchmarks/Cargo.toml b/crates/benchmarks/Cargo.toml index 9dccc444b..68ed5aff4 100644 --- a/crates/benchmarks/Cargo.toml +++ b/crates/benchmarks/Cargo.toml @@ -51,3 +51,8 @@ harness = false [[bench]] name = "indexing" harness = false + +[[bench]] +name = "sort" +harness = false + diff --git a/crates/benchmarks/benches/sort.rs b/crates/benchmarks/benches/sort.rs new file mode 100644 index 000000000..0dd392cb2 --- /dev/null +++ b/crates/benchmarks/benches/sort.rs @@ -0,0 +1,108 @@ +//! This benchmark module is used to compare the performance of sorting documents in /search VS /documents +//! +//! The tests/benchmarks were designed in the context of a query returning only 20 documents. + +mod datasets_paths; +mod utils; + +use criterion::{criterion_group, criterion_main}; +use milli::update::Settings; +use utils::Conf; + +#[cfg(not(windows))] +#[global_allocator] +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; + +fn base_conf(builder: &mut Settings) { + let displayed_fields = + ["geonameid", "name", "asciiname", "alternatenames", "_geo", "population"] + .iter() + .map(|s| s.to_string()) + .collect(); + builder.set_displayed_fields(displayed_fields); + + let sortable_fields = + ["_geo", "name", "population", "elevation", "timezone", "modification-date"] + .iter() + .map(|s| s.to_string()) + .collect(); + builder.set_sortable_fields(sortable_fields); +} + +#[rustfmt::skip] +const BASE_CONF: Conf = Conf { + dataset: datasets_paths::SMOL_ALL_COUNTRIES, + dataset_format: "jsonl", + configure: base_conf, + primary_key: Some("geonameid"), + queries: &[""], + offsets: &[ + Some((0, 20)), // The most common query in the real world + Some((0, 500)), // A query that ranges over many documents + Some((980, 20)), // The worst query that could happen in the real world + Some((800_000, 20)) // The worst query + ], + get_documents: true, + ..Conf::BASE +}; + +fn bench_sort(c: &mut criterion::Criterion) { + #[rustfmt::skip] + let confs = &[ + // utils::Conf { + // group_name: "without sort", + // sort: None, + // ..BASE_CONF + // }, + + // utils::Conf { + // group_name: "sort on many different values", + // sort: Some(vec!["name:asc"]), + // ..BASE_CONF + // }, + + // utils::Conf { + // group_name: "sort on many similar values", + // sort: Some(vec!["timezone:desc"]), + // ..BASE_CONF + // }, + + // utils::Conf { + // group_name: "sort on many similar then different values", + // sort: Some(vec!["timezone:desc", "name:asc"]), + // ..BASE_CONF + // }, + + // utils::Conf { + // group_name: "sort on many different then similar values", + // sort: Some(vec!["timezone:desc", "name:asc"]), + // ..BASE_CONF + // }, + + utils::Conf { + group_name: "geo sort", + sample_size: Some(10), + sort: Some(vec!["_geoPoint(45.4777599, 9.1967508):asc"]), + ..BASE_CONF + }, + + utils::Conf { + group_name: "sort on many similar values then geo sort", + sample_size: Some(10), + sort: Some(vec!["timezone:desc", "_geoPoint(45.4777599, 9.1967508):asc"]), + ..BASE_CONF + }, + + utils::Conf { + group_name: "sort on many different values then geo sort", + sample_size: Some(10), + sort: Some(vec!["name:desc", "_geoPoint(45.4777599, 9.1967508):asc"]), + ..BASE_CONF + }, + ]; + + utils::run_benches(c, confs); +} + +criterion_group!(benches, bench_sort); +criterion_main!(benches); diff --git a/crates/benchmarks/benches/utils.rs b/crates/benchmarks/benches/utils.rs index aaa2d50a0..93fa7506f 100644 --- a/crates/benchmarks/benches/utils.rs +++ b/crates/benchmarks/benches/utils.rs @@ -9,6 +9,7 @@ use anyhow::Context; use bumpalo::Bump; use criterion::BenchmarkId; use memmap2::Mmap; +use milli::documents::sort::recursive_sort; use milli::heed::EnvOpenOptions; use milli::progress::Progress; use milli::update::new::indexer; @@ -35,6 +36,12 @@ pub struct Conf<'a> { pub configure: fn(&mut Settings), pub filter: Option<&'a str>, pub sort: Option>, + /// set to skip documents (offset, limit) + pub offsets: &'a [Option<(usize, usize)>], + /// enable if you want to bench getting documents without querying + pub get_documents: bool, + /// configure the benchmark sample size + pub sample_size: Option, /// enable or disable the optional words on the query pub optional_words: bool, /// primary key, if there is None we'll auto-generate docids for every documents @@ -52,6 +59,9 @@ impl Conf<'_> { configure: |_| (), filter: None, sort: None, + offsets: &[None], + get_documents: false, + sample_size: None, optional_words: true, primary_key: None, }; @@ -144,25 +154,81 @@ pub fn run_benches(c: &mut criterion::Criterion, confs: &[Conf]) { let file_name = Path::new(conf.dataset).file_name().and_then(|f| f.to_str()).unwrap(); let name = format!("{}: {}", file_name, conf.group_name); let mut group = c.benchmark_group(&name); + if let Some(sample_size) = conf.sample_size { + group.sample_size(sample_size); + } for &query in conf.queries { - group.bench_with_input(BenchmarkId::from_parameter(query), &query, |b, &query| { - b.iter(|| { - let rtxn = index.read_txn().unwrap(); - let mut search = index.search(&rtxn); - search.query(query).terms_matching_strategy(TermsMatchingStrategy::default()); - if let Some(filter) = conf.filter { - let filter = Filter::from_str(filter).unwrap().unwrap(); - search.filter(filter); - } - if let Some(sort) = &conf.sort { - let sort = sort.iter().map(|sort| sort.parse().unwrap()).collect(); - search.sort_criteria(sort); - } - let _ids = search.execute().unwrap(); - }); - }); + for offset in conf.offsets { + let parameter = match (query.is_empty(), offset) { + (true, None) => String::from("placeholder"), + (true, Some((offset, limit))) => format!("placeholder[{offset}:{limit}]"), + (false, None) => query.to_string(), + (false, Some((offset, limit))) => format!("{query}[{offset}:{limit}]"), + }; + group.bench_with_input( + BenchmarkId::from_parameter(parameter), + &query, + |b, &query| { + b.iter(|| { + let rtxn = index.read_txn().unwrap(); + let mut search = index.search(&rtxn); + search + .query(query) + .terms_matching_strategy(TermsMatchingStrategy::default()); + if let Some(filter) = conf.filter { + let filter = Filter::from_str(filter).unwrap().unwrap(); + search.filter(filter); + } + if let Some(sort) = &conf.sort { + let sort = sort.iter().map(|sort| sort.parse().unwrap()).collect(); + search.sort_criteria(sort); + } + if let Some((offset, limit)) = offset { + search.offset(*offset).limit(*limit); + } + + let _ids = search.execute().unwrap(); + }); + }, + ); + } } + + if conf.get_documents { + for offset in conf.offsets { + let parameter = match offset { + None => String::from("get_documents"), + Some((offset, limit)) => format!("get_documents[{offset}:{limit}]"), + }; + group.bench_with_input(BenchmarkId::from_parameter(parameter), &(), |b, &()| { + b.iter(|| { + let rtxn = index.read_txn().unwrap(); + if let Some(sort) = &conf.sort { + let sort = sort.iter().map(|sort| sort.parse().unwrap()).collect(); + let all_docs = index.documents_ids(&rtxn).unwrap(); + let facet_sort = + recursive_sort(&index, &rtxn, sort, &all_docs).unwrap(); + let iter = facet_sort.iter().unwrap(); + if let Some((offset, limit)) = offset { + let _results = iter.skip(*offset).take(*limit).collect::>(); + } else { + let _results = iter.collect::>(); + } + } else { + let all_docs = index.documents_ids(&rtxn).unwrap(); + if let Some((offset, limit)) = offset { + let _results = + all_docs.iter().skip(*offset).take(*limit).collect::>(); + } else { + let _results = all_docs.iter().collect::>(); + } + } + }); + }); + } + } + group.finish(); index.prepare_for_closing().wait(); From f60814b319b97beec0e2c98594d4c2cc55c2281f Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 2 Jul 2025 12:06:00 +0200 Subject: [PATCH 052/312] Add benchmark --- crates/benchmarks/benches/sort.rs | 60 +++++++++++++++++-------------- 1 file changed, 33 insertions(+), 27 deletions(-) diff --git a/crates/benchmarks/benches/sort.rs b/crates/benchmarks/benches/sort.rs index 0dd392cb2..c3e934432 100644 --- a/crates/benchmarks/benches/sort.rs +++ b/crates/benchmarks/benches/sort.rs @@ -49,35 +49,35 @@ const BASE_CONF: Conf = Conf { fn bench_sort(c: &mut criterion::Criterion) { #[rustfmt::skip] let confs = &[ - // utils::Conf { - // group_name: "without sort", - // sort: None, - // ..BASE_CONF - // }, + utils::Conf { + group_name: "without sort", + sort: None, + ..BASE_CONF + }, - // utils::Conf { - // group_name: "sort on many different values", - // sort: Some(vec!["name:asc"]), - // ..BASE_CONF - // }, + utils::Conf { + group_name: "sort on many different values", + sort: Some(vec!["name:asc"]), + ..BASE_CONF + }, - // utils::Conf { - // group_name: "sort on many similar values", - // sort: Some(vec!["timezone:desc"]), - // ..BASE_CONF - // }, + utils::Conf { + group_name: "sort on many similar values", + sort: Some(vec!["timezone:desc"]), + ..BASE_CONF + }, - // utils::Conf { - // group_name: "sort on many similar then different values", - // sort: Some(vec!["timezone:desc", "name:asc"]), - // ..BASE_CONF - // }, + utils::Conf { + group_name: "sort on many similar then different values", + sort: Some(vec!["timezone:desc", "name:asc"]), + ..BASE_CONF + }, - // utils::Conf { - // group_name: "sort on many different then similar values", - // sort: Some(vec!["timezone:desc", "name:asc"]), - // ..BASE_CONF - // }, + utils::Conf { + group_name: "sort on many different then similar values", + sort: Some(vec!["timezone:desc", "name:asc"]), + ..BASE_CONF + }, utils::Conf { group_name: "geo sort", @@ -88,17 +88,23 @@ fn bench_sort(c: &mut criterion::Criterion) { utils::Conf { group_name: "sort on many similar values then geo sort", - sample_size: Some(10), + sample_size: Some(50), sort: Some(vec!["timezone:desc", "_geoPoint(45.4777599, 9.1967508):asc"]), ..BASE_CONF }, utils::Conf { group_name: "sort on many different values then geo sort", - sample_size: Some(10), + sample_size: Some(50), sort: Some(vec!["name:desc", "_geoPoint(45.4777599, 9.1967508):asc"]), ..BASE_CONF }, + + utils::Conf { + group_name: "sort on many fields", + sort: Some(vec!["population:asc", "name:asc", "elevation:asc", "timezone:asc"]), + ..BASE_CONF + }, ]; utils::run_benches(c, confs); From 8af76a65bfe4430075bded3ff95e8a343c46b2f1 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 2 Jul 2025 18:04:44 +0200 Subject: [PATCH 053/312] Add test_fragment_indexing --- crates/meilisearch/tests/vector/fragments.rs | 198 +++++++++++++++++++ crates/meilisearch/tests/vector/mod.rs | 1 + 2 files changed, 199 insertions(+) create mode 100644 crates/meilisearch/tests/vector/fragments.rs diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs new file mode 100644 index 000000000..5f6b1095e --- /dev/null +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -0,0 +1,198 @@ +use std::collections::BTreeMap; + +use meili_snap::{json_string, snapshot}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, Request, ResponseTemplate}; + +use crate::common::Value; +use crate::json; +use crate::vector::{get_server_vector, GetAllDocumentsOptions}; + +async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (MockServer, Value) { + let mock_server = MockServer::start().await; + + let text_to_embedding: BTreeMap<_, _> = vec![ + ("kefir", [0.5, -0.5, 2.0]), + ("intel", [1.0, 1.0, 1.0]), + ("bulldog", [1.5, -2.5, 0.0]), + ("dustin", [-0.5, 0.5, 2.5]), + ("labrador", [-3.5, 0.5, -1.0]), + ] + .into_iter() + .collect(); + + Mock::given(method("POST")) + .and(path("/")) + .respond_with(move |req: &Request| { + let text = String::from_utf8_lossy(&req.body).to_string(); + let mut data = [0.0, 0.0, 0.0]; + for (inner_text, inner_data) in &text_to_embedding { + if text.contains(inner_text) { + for (i, &value) in inner_data.iter().enumerate() { + data[i] += value; + } + } + } + ResponseTemplate::new(200).set_body_json( + json!({ "data": data }) + ) + }) + .mount(&mock_server) + .await; + let url = mock_server.uri(); + + let embedder_settings = json!({ + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": indexing_fragments, + "searchFragments": search_fragments, + "documentTemplate": "document template: {{dog.name}}", + }); + + (mock_server, embedder_settings) +} + + +#[actix_rt::test] +async fn test_fragment_indexing() { + let (_mock, settings) = create_mock( + json!({ + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }), + json!({ + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }) + ).await; + let server = get_server_vector().await; + let index = server.index("doggo"); + + // Enable the experimental feature + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + + // Configure the index to use our mock embedder + let (response, code) = index + .update_settings(json!({ + "embedders": { + "rest": settings, + }, + })) + .await; + snapshot!(code, @"202 Accepted"); + + let task = server.wait_task(response.uid()).await; + println!("[task] {:?}", task); + snapshot!(task["status"], @r###""succeeded""###); + + // Send documents + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel", "breed": "labrador"}, + {"id": 3, "name": "dustin", "breed": "bulldog"}, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + + let task = index.wait_task(value.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + // Make sure the documents have been indexed and their embeddings retrieved + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 2.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ], + [ + -2.5, + 1.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 2.5 + ], + [ + 1.0, + -2.0, + 2.5 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} + diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 98555dfac..837c34289 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -4,6 +4,7 @@ mod ollama; mod openai; mod rest; mod settings; +mod fragments; use std::str::FromStr; From 65ba7b47af77015d1baab22ef61e2693c13cc74c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 10:43:27 +0200 Subject: [PATCH 054/312] Test search fragments --- crates/meilisearch/tests/vector/fragments.rs | 169 ++++++++++++++++++- 1 file changed, 164 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 5f6b1095e..876e18ffe 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -12,11 +12,11 @@ async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (Moc let mock_server = MockServer::start().await; let text_to_embedding: BTreeMap<_, _> = vec![ - ("kefir", [0.5, -0.5, 2.0]), - ("intel", [1.0, 1.0, 1.0]), - ("bulldog", [1.5, -2.5, 0.0]), - ("dustin", [-0.5, 0.5, 2.5]), - ("labrador", [-3.5, 0.5, -1.0]), + ("kefir", [0.5, -0.5, 0.0]), + ("intel", [1.0, 1.0, 0.0]), + ("dustin", [-0.5, 0.5, 0.0]), + ("bulldog", [0.0, 0.0, 1.0]), + ("labrador", [0.0, 0.0, -1.0]), ] .into_iter() .collect(); @@ -196,3 +196,162 @@ async fn test_fragment_indexing() { "#); } +#[actix_rt::test] +async fn test_search_fragments() { + let (_mock, settings) = create_mock( + json!({ + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }), + json!({ + "justBreed": {"value": "It's a {{ media.breed }}"}, + "justName": {"value": "{{ media.name }} is a dog"}, + "query": {"value": "Some pre-prompt for query {{ q }}"}, + }) + ).await; + let server = get_server_vector().await; + let index = server.index("doggo"); + + // Enable the experimental feature + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + + // Configure the index to use our mock embedder + let (response, code) = index + .update_settings(json!({ + "embedders": { + "rest": settings, + }, + })) + .await; + snapshot!(code, @"202 Accepted"); + + let task = server.wait_task(response.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + // Send documents + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel", "breed": "labrador"}, + {"id": 3, "name": "dustin", "breed": "bulldog"}, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + + let task = index.wait_task(value.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + // Perform a search with a provided vector + let (value, code) = index.search_post( + json!({"vector": [1.0, 1.0, 1.0], "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, "limit": 1} + )).await; + snapshot!(code, @"200 OK"); + snapshot!(value, @r#" + { + "hits": [ + { + "id": 1, + "name": "echo" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 1, + "offset": 0, + "estimatedTotalHits": 4, + "semanticHitCount": 1 + } + "#); + + // Perform a search with some media + let (value, code) = index.search_post( + json!({ + "media": { "breed": "labrador" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )).await; + snapshot!(code, @"200 OK"); + snapshot!(value, @r#" + { + "hits": [ + { + "id": 2, + "name": "intel", + "breed": "labrador" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 1, + "offset": 0, + "estimatedTotalHits": 4, + "semanticHitCount": 1 + } + "#); + + // Perform a search that matches multiple media + let (value, code) = index.search_post( + json!({ + "media": { "name": "dustin", "breed": "labrador" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Error while generating embeddings: user error: Query matches multiple search fragments.\n - Note: First matched fragment `justBreed`.\n - Note: Second matched fragment `justName`.\n - Note: {\"q\":null,\"media\":{\"name\":\"dustin\",\"breed\":\"labrador\"}}", + "code": "vector_embedding_error", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#vector_embedding_error" + } + "#); + + // Perform a search that matches no media + let (value, code) = index.search_post( + json!({ + "media": { "ticker": "GME", "section": "portfolio" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Error while generating embeddings: user error: Query matches no search fragment.\n - Note: {\"q\":null,\"media\":{\"ticker\":\"GME\",\"section\":\"portfolio\"}}", + "code": "vector_embedding_error", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#vector_embedding_error" + } + "#); + + // Perform a search with a query media + let (value, code) = index.search_post( + json!({ + "q": "bulldog", + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )).await; + snapshot!(code, @"200 OK"); + snapshot!(value, @r#" + { + "hits": [ + { + "id": 3, + "name": "dustin", + "breed": "bulldog" + } + ], + "query": "bulldog", + "processingTimeMs": "[duration]", + "limit": 1, + "offset": 0, + "estimatedTotalHits": 4, + "semanticHitCount": 1 + } + "#); +} + From 0b89ef1fd7c11c2909bc906911394bdc9ffd10fc Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 11:24:09 +0200 Subject: [PATCH 055/312] Make tests use a shared index --- crates/meilisearch/tests/common/server.rs | 4 +- crates/meilisearch/tests/vector/fragments.rs | 261 +++++++++---------- 2 files changed, 128 insertions(+), 137 deletions(-) diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 4367650c5..e3839855b 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -35,7 +35,7 @@ pub struct Server { pub static TEST_TEMP_DIR: Lazy = Lazy::new(|| TempDir::new().unwrap()); impl Server { - fn into_shared(self) -> Server { + pub fn into_shared(self) -> Server { Server { service: self.service, _dir: self._dir, _marker: PhantomData } } @@ -327,7 +327,7 @@ impl Server { self.service.get(url).await } - pub(super) fn _index(&self, uid: impl AsRef) -> Index<'_> { + pub fn _index(&self, uid: impl AsRef) -> Index<'_> { Index { uid: uid.as_ref().to_string(), service: &self.service, diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 876e18ffe..027a4069d 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -1,13 +1,73 @@ use std::collections::BTreeMap; use meili_snap::{json_string, snapshot}; +use tokio::sync::OnceCell; use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, Request, ResponseTemplate}; +use crate::common::index::Index; +use crate::common::Shared; use crate::common::Value; use crate::json; +use crate::vector::Server; use crate::vector::{get_server_vector, GetAllDocumentsOptions}; +async fn shared_index_for_fragments() -> Index<'static, Shared> { + static INDEX: OnceCell<(Server, String)> = OnceCell::const_new(); + let (server, uid) = INDEX + .get_or_init(|| async { + let (_mock, settings) = create_mock( + json!({ + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }), + json!({ + "justBreed": {"value": "It's a {{ media.breed }}"}, + "justName": {"value": "{{ media.name }} is a dog"}, + "query": {"value": "Some pre-prompt for query {{ q }}"}, + }), + ) + .await; + + let server = Server::new().await; + let index = server.unique_index(); + + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + + // Configure the index to use our mock embedder + let (response, code) = index + .update_settings(json!({ + "embedders": { + "rest": settings, + }, + })) + .await; + snapshot!(code, @"202 Accepted"); + + let task = server.wait_task(response.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + // Send documents + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel", "breed": "labrador"}, + {"id": 3, "name": "dustin", "breed": "bulldog"}, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + + let task = index.wait_task(value.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + let uid = index.uid.clone(); + (server.into_shared(), uid) + }) + .await; + server._index(uid).to_shared() +} + async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (MockServer, Value) { let mock_server = MockServer::start().await; @@ -33,9 +93,7 @@ async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (Moc } } } - ResponseTemplate::new(200).set_body_json( - json!({ "data": data }) - ) + ResponseTemplate::new(200).set_body_json(json!({ "data": data })) }) .mount(&mock_server) .await; @@ -57,52 +115,9 @@ async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (Moc (mock_server, embedder_settings) } - #[actix_rt::test] -async fn test_fragment_indexing() { - let (_mock, settings) = create_mock( - json!({ - "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, - "basic": {"value": "{{ doc.name }} is a dog"}, - }), - json!({ - "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, - "basic": {"value": "{{ doc.name }} is a dog"}, - }) - ).await; - let server = get_server_vector().await; - let index = server.index("doggo"); - - // Enable the experimental feature - let (_response, code) = server.set_features(json!({"multimodal": true})).await; - snapshot!(code, @"200 OK"); - - // Configure the index to use our mock embedder - let (response, code) = index - .update_settings(json!({ - "embedders": { - "rest": settings, - }, - })) - .await; - snapshot!(code, @"202 Accepted"); - - let task = server.wait_task(response.uid()).await; - println!("[task] {:?}", task); - snapshot!(task["status"], @r###""succeeded""###); - - // Send documents - let documents = json!([ - {"id": 0, "name": "kefir"}, - {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, - {"id": 2, "name": "intel", "breed": "labrador"}, - {"id": 3, "name": "dustin", "breed": "bulldog"}, - ]); - let (value, code) = index.add_documents(documents, None).await; - snapshot!(code, @"202 Accepted"); - - let task = index.wait_task(value.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); +async fn indexing_fragments() { + let index = shared_index_for_fragments().await; // Make sure the documents have been indexed and their embeddings retrieved let (documents, code) = index @@ -121,7 +136,7 @@ async fn test_fragment_indexing() { [ 0.5, -0.5, - 2.0 + 0.0 ] ], "regenerate": true @@ -154,12 +169,12 @@ async fn test_fragment_indexing() { [ 1.0, 1.0, - 1.0 + 0.0 ], [ - -2.5, - 1.5, - 0.0 + 1.0, + 1.0, + -1.0 ] ], "regenerate": true @@ -176,12 +191,12 @@ async fn test_fragment_indexing() { [ -0.5, 0.5, - 2.5 + 0.0 ], [ - 1.0, - -2.0, - 2.5 + -0.5, + 0.5, + 1.0 ] ], "regenerate": true @@ -197,52 +212,9 @@ async fn test_fragment_indexing() { } #[actix_rt::test] -async fn test_search_fragments() { - let (_mock, settings) = create_mock( - json!({ - "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, - "basic": {"value": "{{ doc.name }} is a dog"}, - }), - json!({ - "justBreed": {"value": "It's a {{ media.breed }}"}, - "justName": {"value": "{{ media.name }} is a dog"}, - "query": {"value": "Some pre-prompt for query {{ q }}"}, - }) - ).await; - let server = get_server_vector().await; - let index = server.index("doggo"); +async fn search_with_vector() { + let index = shared_index_for_fragments().await; - // Enable the experimental feature - let (_response, code) = server.set_features(json!({"multimodal": true})).await; - snapshot!(code, @"200 OK"); - - // Configure the index to use our mock embedder - let (response, code) = index - .update_settings(json!({ - "embedders": { - "rest": settings, - }, - })) - .await; - snapshot!(code, @"202 Accepted"); - - let task = server.wait_task(response.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); - - // Send documents - let documents = json!([ - {"id": 0, "name": "kefir"}, - {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, - {"id": 2, "name": "intel", "breed": "labrador"}, - {"id": 3, "name": "dustin", "breed": "bulldog"}, - ]); - let (value, code) = index.add_documents(documents, None).await; - snapshot!(code, @"202 Accepted"); - - let task = index.wait_task(value.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); - - // Perform a search with a provided vector let (value, code) = index.search_post( json!({"vector": [1.0, 1.0, 1.0], "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, "limit": 1} )).await; @@ -263,15 +235,20 @@ async fn test_search_fragments() { "semanticHitCount": 1 } "#); +} - // Perform a search with some media - let (value, code) = index.search_post( - json!({ - "media": { "breed": "labrador" }, - "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, - "limit": 1 - } - )).await; +#[actix_rt::test] +async fn search_with_media() { + let index = shared_index_for_fragments().await; + + let (value, code) = index + .search_post(json!({ + "media": { "breed": "labrador" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )) + .await; snapshot!(code, @"200 OK"); snapshot!(value, @r#" { @@ -290,15 +267,20 @@ async fn test_search_fragments() { "semanticHitCount": 1 } "#); +} - // Perform a search that matches multiple media - let (value, code) = index.search_post( - json!({ - "media": { "name": "dustin", "breed": "labrador" }, - "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, - "limit": 1 - } - )).await; +#[actix_rt::test] +async fn search_with_media_matching_multiple_fragments() { + let index = shared_index_for_fragments().await; + + let (value, code) = index + .search_post(json!({ + "media": { "name": "dustin", "breed": "labrador" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )) + .await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { @@ -308,15 +290,20 @@ async fn test_search_fragments() { "link": "https://docs.meilisearch.com/errors#vector_embedding_error" } "#); +} - // Perform a search that matches no media - let (value, code) = index.search_post( - json!({ - "media": { "ticker": "GME", "section": "portfolio" }, - "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, - "limit": 1 - } - )).await; +#[actix_rt::test] +async fn search_with_media_matching_no_fragment() { + let index = shared_index_for_fragments().await; + + let (value, code) = index + .search_post(json!({ + "media": { "ticker": "GME", "section": "portfolio" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )) + .await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { @@ -326,15 +313,20 @@ async fn test_search_fragments() { "link": "https://docs.meilisearch.com/errors#vector_embedding_error" } "#); +} - // Perform a search with a query media - let (value, code) = index.search_post( - json!({ - "q": "bulldog", - "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, - "limit": 1 - } - )).await; +#[actix_rt::test] +async fn search_with_query() { + let index = shared_index_for_fragments().await; + + let (value, code) = index + .search_post(json!({ + "q": "bulldog", + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )) + .await; snapshot!(code, @"200 OK"); snapshot!(value, @r#" { @@ -354,4 +346,3 @@ async fn test_search_fragments() { } "#); } - From b45eea0d3e101938ed7eaa58c839b754922504d4 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 13:26:44 +0200 Subject: [PATCH 056/312] Add test for fragment deletion --- crates/meilisearch/tests/common/mod.rs | 2 +- crates/meilisearch/tests/vector/fragments.rs | 258 ++++++++++++++----- 2 files changed, 196 insertions(+), 64 deletions(-) diff --git a/crates/meilisearch/tests/common/mod.rs b/crates/meilisearch/tests/common/mod.rs index 1a73a7532..2fafbd11f 100644 --- a/crates/meilisearch/tests/common/mod.rs +++ b/crates/meilisearch/tests/common/mod.rs @@ -3,7 +3,7 @@ pub mod index; pub mod server; pub mod service; -use std::fmt::{self, Display}; +use std::{fmt::{self, Display}, future::Future}; #[allow(unused)] pub use index::GetAllDocumentsOptions; diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 027a4069d..c3fef8c79 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -6,69 +6,23 @@ use wiremock::matchers::{method, path}; use wiremock::{Mock, MockServer, Request, ResponseTemplate}; use crate::common::index::Index; -use crate::common::Shared; -use crate::common::Value; +use crate::common::{Owned, Shared}; use crate::json; use crate::vector::Server; -use crate::vector::{get_server_vector, GetAllDocumentsOptions}; +use crate::vector::GetAllDocumentsOptions; async fn shared_index_for_fragments() -> Index<'static, Shared> { static INDEX: OnceCell<(Server, String)> = OnceCell::const_new(); let (server, uid) = INDEX .get_or_init(|| async { - let (_mock, settings) = create_mock( - json!({ - "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, - "basic": {"value": "{{ doc.name }} is a dog"}, - }), - json!({ - "justBreed": {"value": "It's a {{ media.breed }}"}, - "justName": {"value": "{{ media.name }} is a dog"}, - "query": {"value": "Some pre-prompt for query {{ q }}"}, - }), - ) - .await; - - let server = Server::new().await; - let index = server.unique_index(); - - let (_response, code) = server.set_features(json!({"multimodal": true})).await; - snapshot!(code, @"200 OK"); - - // Configure the index to use our mock embedder - let (response, code) = index - .update_settings(json!({ - "embedders": { - "rest": settings, - }, - })) - .await; - snapshot!(code, @"202 Accepted"); - - let task = server.wait_task(response.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); - - // Send documents - let documents = json!([ - {"id": 0, "name": "kefir"}, - {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, - {"id": 2, "name": "intel", "breed": "labrador"}, - {"id": 3, "name": "dustin", "breed": "bulldog"}, - ]); - let (value, code) = index.add_documents(documents, None).await; - snapshot!(code, @"202 Accepted"); - - let task = index.wait_task(value.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); - - let uid = index.uid.clone(); + let (server, uid, _) = init_fragments_index().await; (server.into_shared(), uid) }) .await; server._index(uid).to_shared() } -async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (MockServer, Value) { +pub async fn init_fragments_index() -> (Server, String, crate::common::Value) { let mock_server = MockServer::start().await; let text_to_embedding: BTreeMap<_, _> = vec![ @@ -99,22 +53,62 @@ async fn create_mock(indexing_fragments: Value, search_fragments: Value) -> (Moc .await; let url = mock_server.uri(); - let embedder_settings = json!({ - "source": "rest", - "url": url, - "dimensions": 3, - "request": "{{fragment}}", - "response": { - "data": "{{embedding}}" - }, - "indexingFragments": indexing_fragments, - "searchFragments": search_fragments, - "documentTemplate": "document template: {{dog.name}}", - }); + let server = Server::new().await; + let index = server.unique_index(); - (mock_server, embedder_settings) + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + + // Configure the index to use our mock embedder + let settings = json!({ + "embedders": { + "rest": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }, + "searchFragments": { + "justBreed": {"value": "It's a {{ media.breed }}"}, + "justName": {"value": "{{ media.name }} is a dog"}, + "query": {"value": "Some pre-prompt for query {{ q }}"}, + } + }, + }, + }); + let (response, code) = index + .update_settings(settings.clone()) + .await; + snapshot!(code, @"202 Accepted"); + + let task = server.wait_task(response.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + // Send documents + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel", "breed": "labrador"}, + {"id": 3, "name": "dustin", "breed": "bulldog"}, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + + let task = index.wait_task(value.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + let uid = index.uid.clone(); + (server, uid, settings) } +// TODO: Test cannot pass both fragments and document + #[actix_rt::test] async fn indexing_fragments() { let index = shared_index_for_fragments().await; @@ -346,3 +340,141 @@ async fn search_with_query() { } "#); } + +#[actix_rt::test] +async fn deleting_fragments_deletes_vectors() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"]["rest"]["indexingFragments"]["basic"] = serde_json::Value::Null; + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + println!("Documents before update: {documents:?}"); + + let (response, code) = index + .update_settings(settings) + .await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": null, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} From 5c792737486899d2af03aaf7cb9e0acba2fcc883 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 14:42:49 +0200 Subject: [PATCH 057/312] Add TODOs --- crates/meilisearch/tests/vector/fragments.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index c3fef8c79..338f9e7c5 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -109,6 +109,12 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va // TODO: Test cannot pass both fragments and document +// TODO: test with 2 embedders + +// TODO: edit fragment + +// TODO: document fragment replaced + #[actix_rt::test] async fn indexing_fragments() { let index = shared_index_for_fragments().await; From 90683d0e4e77a1f8d9924a4a9e8065767b49c76d Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 3 Jul 2025 14:33:45 +0200 Subject: [PATCH 058/312] add snapshot of get settings --- crates/meilisearch/tests/vector/fragments.rs | 99 +++++++++++++++++--- 1 file changed, 87 insertions(+), 12 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 338f9e7c5..a21855d8d 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -8,8 +8,7 @@ use wiremock::{Mock, MockServer, Request, ResponseTemplate}; use crate::common::index::Index; use crate::common::{Owned, Shared}; use crate::json; -use crate::vector::Server; -use crate::vector::GetAllDocumentsOptions; +use crate::vector::{GetAllDocumentsOptions, Server}; async fn shared_index_for_fragments() -> Index<'static, Shared> { static INDEX: OnceCell<(Server, String)> = OnceCell::const_new(); @@ -82,9 +81,7 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va }, }, }); - let (response, code) = index - .update_settings(settings.clone()) - .await; + let (response, code) = index.update_settings(settings.clone()).await; snapshot!(code, @"202 Accepted"); let task = server.wait_task(response.uid()).await; @@ -359,9 +356,7 @@ async fn deleting_fragments_deletes_vectors() { .await; println!("Documents before update: {documents:?}"); - let (response, code) = index - .update_settings(settings) - .await; + let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); let value = server.wait_task(response.uid()).await.succeeded(); snapshot!(value, @r#" @@ -410,11 +405,91 @@ async fn deleting_fragments_deletes_vectors() { } "#); + let (value, code) = index.settings().await; + snapshot!(value, @r###" + { + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [], + "sortableAttributes": [], + "rankingRules": [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness" + ], + "stopWords": [], + "nonSeparatorTokens": [], + "separatorTokens": [], + "dictionary": [], + "synonyms": {}, + "distinctAttribute": null, + "proximityPrecision": "byWord", + "typoTolerance": { + "enabled": true, + "minWordSizeForTypos": { + "oneTypo": 5, + "twoTypos": 9 + }, + "disableOnWords": [], + "disableOnAttributes": [], + "disableOnNumbers": false + }, + "faceting": { + "maxValuesPerFacet": 100, + "sortFacetValuesBy": { + "*": "alpha" + } + }, + "pagination": { + "maxTotalHits": 1000 + }, + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "http://127.0.0.1:53832", + "indexingFragments": { + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "headers": {} + } + }, + "searchCutoffMs": null, + "localizedAttributes": null, + "facetSearch": true, + "prefixSearch": "indexingTime" + } + "###); + let (documents, code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(documents), @r#" + snapshot!(json_string!(documents), @r###" { "results": [ { @@ -453,7 +528,7 @@ async fn deleting_fragments_deletes_vectors() { [ 1.0, 1.0, - 0.0 + -1.0 ] ], "regenerate": true @@ -470,7 +545,7 @@ async fn deleting_fragments_deletes_vectors() { [ -0.5, 0.5, - 0.0 + 1.0 ] ], "regenerate": true @@ -482,5 +557,5 @@ async fn deleting_fragments_deletes_vectors() { "limit": 20, "total": 4 } - "#); + "###); } From a3af9fe0578903895c6c210e66760d0ab48205a9 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 3 Jul 2025 14:35:02 +0200 Subject: [PATCH 059/312] new extractor bugfixes: - fix old_has_fragments - new_is_user_provided is always false when generating fragments, even if no fragment ever matches --- .../src/update/new/extract/vectors/mod.rs | 24 +++++-------------- 1 file changed, 6 insertions(+), 18 deletions(-) diff --git a/crates/milli/src/update/new/extract/vectors/mod.rs b/crates/milli/src/update/new/extract/vectors/mod.rs index 72a07dea6..4ca68027c 100644 --- a/crates/milli/src/update/new/extract/vectors/mod.rs +++ b/crates/milli/src/update/new/extract/vectors/mod.rs @@ -357,7 +357,7 @@ impl<'extractor, SD: SettingsDelta + Sync> SettingsChangeExtractor<'extractor> chunks.is_user_provided_must_regenerate(document.docid()); let old_has_fragments = old_embedders .get(embedder_name) - .map(|embedder| embedder.fragments().is_empty()) + .map(|embedder| !embedder.fragments().is_empty()) .unwrap_or_default(); let new_has_fragments = chunks.has_fragments(); @@ -628,9 +628,6 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { session.on_embed_mut().clear_vectors(docid); } - let mut extracted = false; - let extracted = &mut extracted; - settings_delta.try_for_each_fragment_diff( session.embedder_name(), |fragment_diff| { @@ -660,7 +657,6 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { ); } ExtractorDiff::Added(input) | ExtractorDiff::Updated(input) => { - *extracted = true; session.request_embedding( metadata, input, @@ -673,13 +669,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { Result::Ok(()) }, )?; - self.set_status( - docid, - old_is_user_provided, - true, - old_is_user_provided & !*extracted, - true, - ); + self.set_status(docid, old_is_user_provided, true, false, true); } ChunkType::DocumentTemplate { document_template, session } => { let doc_alloc = session.doc_alloc(); @@ -732,7 +722,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { where 'a: 'doc, { - let extracted = match &mut self.kind { + match &mut self.kind { ChunkType::DocumentTemplate { document_template, session } => { let doc_alloc = session.doc_alloc(); let ex = DocumentTemplateExtractor::new( @@ -785,7 +775,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { docid, old_is_user_provided, old_must_regenerate, - old_is_user_provided && !extracted, + false, new_must_regenerate, ); @@ -968,7 +958,7 @@ fn update_autogenerated<'doc, 'a: 'doc, 'b, E, OD, ND>( old_must_regenerate: bool, session: &mut EmbedSession<'a, OnEmbeddingDocumentUpdates<'a, 'b>, E::Input>, unused_vectors_distribution: &UnusedVectorsDistributionBump<'a>, -) -> Result +) -> Result<()> where OD: Document<'doc> + Debug, ND: Document<'doc> + Debug, @@ -976,7 +966,6 @@ where E::Input: Input, crate::Error: From, { - let mut extracted = false; for extractor in extractors { let new_rendered = extractor.extract(&new_document, meta)?; let must_regenerate = if !old_must_regenerate { @@ -995,7 +984,6 @@ where }; if must_regenerate { - extracted = true; let metadata = Metadata { docid, external_docid, extractor_id: extractor.extractor_id() }; @@ -1011,7 +999,7 @@ where } } - Ok(extracted) + Ok(()) } fn insert_autogenerated<'a, 'b, E, D: Document<'a> + Debug>( From de24e75be8a34da406d84656698c68185d7644a7 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 15:00:11 +0200 Subject: [PATCH 060/312] Update test --- crates/meilisearch/tests/vector/fragments.rs | 101 +++++-------------- 1 file changed, 28 insertions(+), 73 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index a21855d8d..863d0127b 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -406,84 +406,39 @@ async fn deleting_fragments_deletes_vectors() { "#); let (value, code) = index.settings().await; - snapshot!(value, @r###" + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value["embedders"], { + ".rest.url" => "[url]", + }), @r#" { - "displayedAttributes": [ - "*" - ], - "searchableAttributes": [ - "*" - ], - "filterableAttributes": [], - "sortableAttributes": [], - "rankingRules": [ - "words", - "typo", - "proximity", - "attribute", - "sort", - "exactness" - ], - "stopWords": [], - "nonSeparatorTokens": [], - "separatorTokens": [], - "dictionary": [], - "synonyms": {}, - "distinctAttribute": null, - "proximityPrecision": "byWord", - "typoTolerance": { - "enabled": true, - "minWordSizeForTypos": { - "oneTypo": 5, - "twoTypos": 9 + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } }, - "disableOnWords": [], - "disableOnAttributes": [], - "disableOnNumbers": false - }, - "faceting": { - "maxValuesPerFacet": 100, - "sortFacetValuesBy": { - "*": "alpha" - } - }, - "pagination": { - "maxTotalHits": 1000 - }, - "embedders": { - "rest": { - "source": "rest", - "dimensions": 3, - "url": "http://127.0.0.1:53832", - "indexingFragments": { - "withBreed": { - "value": "{{ doc.name }} is a {{ doc.breed }}" - } + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" }, - "searchFragments": { - "justBreed": { - "value": "It's a {{ media.breed }}" - }, - "justName": { - "value": "{{ media.name }} is a dog" - }, - "query": { - "value": "Some pre-prompt for query {{ q }}" - } + "justName": { + "value": "{{ media.name }} is a dog" }, - "request": "{{fragment}}", - "response": { - "data": "{{embedding}}" - }, - "headers": {} - } - }, - "searchCutoffMs": null, - "localizedAttributes": null, - "facetSearch": true, - "prefixSearch": "indexingTime" + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "headers": {} + } } - "###); + "#); let (documents, code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) From 2bcd69750f61fb3f46f8e8759fac80bd3b2b170f Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 15:08:27 +0200 Subject: [PATCH 061/312] Add fragment modification test --- crates/meilisearch/tests/vector/fragments.rs | 158 ++++++++++++++++++- 1 file changed, 153 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 863d0127b..2b7592316 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -112,6 +112,8 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va // TODO: document fragment replaced +// TODO: not setting to null but ommitting settings + #[actix_rt::test] async fn indexing_fragments() { let index = shared_index_for_fragments().await; @@ -351,11 +353,6 @@ async fn deleting_fragments_deletes_vectors() { settings["embedders"]["rest"]["indexingFragments"]["basic"] = serde_json::Value::Null; - let (documents, code) = index - .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) - .await; - println!("Documents before update: {documents:?}"); - let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); let value = server.wait_task(response.uid()).await.succeeded(); @@ -514,3 +511,154 @@ async fn deleting_fragments_deletes_vectors() { } "###); } + +#[actix_rt::test] +async fn modifying_fragments_modifies_vectors() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"]["rest"]["indexingFragments"]["basic"]["value"] = + serde_json::Value::String("{{ doc.name }} is a dog (maybe bulldog?)".to_string()); + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog (maybe bulldog?)" + }, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 1.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 1.0 + ], + [ + -0.5, + 0.5, + 1.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} + From 2faad504c6d7c2195bd99cd947cc5a4a3d1db38b Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 15:12:47 +0200 Subject: [PATCH 062/312] Add test --- crates/meilisearch/tests/vector/fragments.rs | 92 +++++++++++++++++++- 1 file changed, 90 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 2b7592316..00682299b 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -112,8 +112,6 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va // TODO: document fragment replaced -// TODO: not setting to null but ommitting settings - #[actix_rt::test] async fn indexing_fragments() { let index = shared_index_for_fragments().await; @@ -662,3 +660,93 @@ async fn modifying_fragments_modifies_vectors() { "#); } +#[actix_rt::test] +async fn ommitted_fragment_isnt_removed() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"]["rest"]["indexingFragments"]["basic"] = serde_json::Value::Null; // basic is removed + settings["embedders"]["rest"]["indexingFragments"].as_object_mut().unwrap().remove("withBreed"); // withBreed isn't specified + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": null + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (value, code) = index.settings().await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value["embedders"], { + ".rest.url" => "[url]", + }), @r#" + { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "headers": {} + } + } + "#); +} + From 5690700601b4ef3970c8c9e4c7da57b6c3e6bbb0 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 15:19:31 +0200 Subject: [PATCH 063/312] Add fragment addition test --- crates/meilisearch/tests/vector/fragments.rs | 168 +++++++++++++++++++ 1 file changed, 168 insertions(+) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 00682299b..71bab6ea0 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -750,3 +750,171 @@ async fn ommitted_fragment_isnt_removed() { "#); } +#[actix_rt::test] +async fn fragment_insertion() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"]["rest"]["indexingFragments"].as_object_mut().unwrap().insert(String::from("useless"), serde_json::json!({ + "value": "This fragment is useless" + })); + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog" + }, + "useless": { + "value": "This fragment is useless" + }, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ], + [ + 1.0, + 1.0, + -1.0 + ], + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ], + [ + -0.5, + 0.5, + 1.0 + ], + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} From 7423243be0bc373a42b5518d6cc625e762187629 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 15:52:18 +0200 Subject: [PATCH 064/312] Add test with multiple embedders --- crates/meilisearch/tests/vector/fragments.rs | 447 ++++++++++++++++++- 1 file changed, 445 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 71bab6ea0..03d0ffc7a 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -106,12 +106,14 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va // TODO: Test cannot pass both fragments and document -// TODO: test with 2 embedders - // TODO: edit fragment // TODO: document fragment replaced +// TODO: complex value + +// TODO: swapping fragments + #[actix_rt::test] async fn indexing_fragments() { let index = shared_index_for_fragments().await; @@ -918,3 +920,444 @@ async fn fragment_insertion() { } "#); } + +#[actix_rt::test] +async fn multiple_embedders() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + let url = settings["embedders"]["rest"]["url"].as_str().unwrap(); + + let settings2 = json!({ + "embedders": { + "rest2": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }, + "searchFragments": { + "query": {"value": "Some pre-prompt for query {{ q }}"}, + } + }, + "rest3": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "basic": {"value": "{{ doc.name }} is a dog"}, + }, + "searchFragments": { + "query": {"value": "Some pre-prompt for query {{ q }}"}, + } + }, + }, + }); + let (response, code) = index.update_settings(settings2).await; + snapshot!(code, @"202 Accepted"); + let task = server.wait_task(response.uid()).await; + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest2": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog" + }, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + }, + "rest3": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog" + } + }, + "searchFragments": { + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + }, + "rest2": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + }, + "rest2": { + "embeddings": [ + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ], + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": true + }, + "rest2": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ], + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ], + [ + -0.5, + 0.5, + 1.0 + ] + ], + "regenerate": true + }, + "rest2": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ], + [ + -0.5, + 0.5, + 1.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); + + // Remove Rest2 + + settings["embedders"]["rest2"] = serde_json::Value::Null; + + let (response, code) = index.update_settings(settings.clone()).await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value["status"], @r###""succeeded""###); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + }, + "rest3": { + "embeddings": [ + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ], + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ], + [ + -0.5, + 0.5, + 1.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); + + // Remove rest's basic fragment + + settings["embedders"]["rest"]["indexingFragments"]["basic"] = serde_json::Value::Null; + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value["status"], @r###""succeeded""###); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r""); +} From cf9b311f71d2316905a0d25487e953c68d6a345f Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 15:53:09 +0200 Subject: [PATCH 065/312] Format --- crates/meilisearch/tests/common/mod.rs | 2 +- crates/meilisearch/tests/vector/fragments.rs | 13 ++++++++----- crates/meilisearch/tests/vector/mod.rs | 2 +- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch/tests/common/mod.rs b/crates/meilisearch/tests/common/mod.rs index 2fafbd11f..1a73a7532 100644 --- a/crates/meilisearch/tests/common/mod.rs +++ b/crates/meilisearch/tests/common/mod.rs @@ -3,7 +3,7 @@ pub mod index; pub mod server; pub mod service; -use std::{fmt::{self, Display}, future::Future}; +use std::fmt::{self, Display}; #[allow(unused)] pub use index::GetAllDocumentsOptions; diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 03d0ffc7a..7cfa0c1af 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -757,9 +757,12 @@ async fn fragment_insertion() { let (server, uid, mut settings) = init_fragments_index().await; let index = server.index(uid); - settings["embedders"]["rest"]["indexingFragments"].as_object_mut().unwrap().insert(String::from("useless"), serde_json::json!({ - "value": "This fragment is useless" - })); + settings["embedders"]["rest"]["indexingFragments"].as_object_mut().unwrap().insert( + String::from("useless"), + serde_json::json!({ + "value": "This fragment is useless" + }), + ); let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); @@ -1215,7 +1218,7 @@ async fn multiple_embedders() { snapshot!(code, @"202 Accepted"); let value = server.wait_task(response.uid()).await.succeeded(); snapshot!(value["status"], @r###""succeeded""###); - + let (documents, code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; @@ -1354,7 +1357,7 @@ async fn multiple_embedders() { snapshot!(code, @"202 Accepted"); let value = server.wait_task(response.uid()).await.succeeded(); snapshot!(value["status"], @r###""succeeded""###); - + let (documents, code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 837c34289..7f54489b6 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -1,10 +1,10 @@ mod binary_quantized; +mod fragments; #[cfg(feature = "test-ollama")] mod ollama; mod openai; mod rest; mod settings; -mod fragments; use std::str::FromStr; From caccb5181449fcde12bf967688608b6ef9fc7188 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 16:10:23 +0200 Subject: [PATCH 066/312] Add a complex value test --- crates/meilisearch/tests/vector/fragments.rs | 186 ++++++++++++++++++- 1 file changed, 184 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 7cfa0c1af..20bc4e7ce 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -30,6 +30,7 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va ("dustin", [-0.5, 0.5, 0.0]), ("bulldog", [0.0, 0.0, 1.0]), ("labrador", [0.0, 0.0, -1.0]), + ("{", [-9999.0, -9999.0, -9999.0]), // That wouldn't be nice ] .into_iter() .collect(); @@ -110,8 +111,6 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va // TODO: document fragment replaced -// TODO: complex value - // TODO: swapping fragments #[actix_rt::test] @@ -1364,3 +1363,186 @@ async fn multiple_embedders() { snapshot!(code, @"200 OK"); snapshot!(json_string!(documents), @r""); } + +#[actix_rt::test] +async fn complex_fragment() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"]["rest"]["indexingFragments"].as_object_mut().unwrap().insert( + String::from("complex"), + serde_json::json!({ + "value": { + "breed": "{{ doc.breed }}", + "breeds": [ + "{{ doc.breed }}", + { + "breed": "{{ doc.breed }}", + } + ] + } + }), + ); + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let task = server.wait_task(response.uid()).await; + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog" + }, + "complex": { + "value": { + "breed": "{{ doc.breed }}", + "breeds": [ + "{{ doc.breed }}", + { + "breed": "{{ doc.breed }}" + } + ] + } + }, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ], + [ + 1.0, + 1.0, + -1.0 + ], + [ + 0.0, + 0.0, + -1.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ], + [ + -0.5, + 0.5, + 1.0 + ], + [ + 0.0, + 0.0, + 1.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} From d0cd3cacecbaa01fe3a273924232fe9d119e37f6 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 3 Jul 2025 18:18:04 +0200 Subject: [PATCH 067/312] Add a way to reproduce the bug --- crates/meilisearch/tests/vector/fragments.rs | 51 ++++++++++++++++++++ 1 file changed, 51 insertions(+) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 20bc4e7ce..0135e2044 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -1351,6 +1351,7 @@ async fn multiple_embedders() { // Remove rest's basic fragment settings["embedders"]["rest"]["indexingFragments"]["basic"] = serde_json::Value::Null; + //settings["embedders"].as_object_mut().unwrap().remove("rest2"); let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); @@ -1364,6 +1365,56 @@ async fn multiple_embedders() { snapshot!(json_string!(documents), @r""); } +#[actix_rt::test] +async fn remove_non_existant_embedder() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"].as_object_mut().unwrap().insert(String::from("non-existant"), serde_json::Value::Null); + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let task = server.wait_task(response.uid()).await; + snapshot!(task, @r""); +} + +#[actix_rt::test] +async fn double_remove_embedder() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + settings["embedders"].as_object_mut().unwrap().insert(String::from("rest"), serde_json::Value::Null); + + let (response, code) = index.update_settings(settings.clone()).await; + snapshot!(code, @"202 Accepted"); + let task = server.wait_task(response.uid()).await; + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": null + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (response, code) = index.update_settings(settings.clone()).await; + snapshot!(code, @"202 Accepted"); + let task = server.wait_task(response.uid()).await; + snapshot!(task, @r#""#); +} + #[actix_rt::test] async fn complex_fragment() { let (server, uid, mut settings) = init_fragments_index().await; From 3714f166967a37adc7f04a101bed4ca62e121762 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 10:40:50 +0200 Subject: [PATCH 068/312] Fix bug --- crates/milli/src/update/settings.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 911f51865..4124aa540 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -101,6 +101,10 @@ impl Setting { matches!(self, Self::NotSet) } + pub const fn is_reset(&self) -> bool { + matches!(self, Self::Reset) + } + /// If `Self` is `Reset`, then map self to `Set` with the provided `val`. pub fn or_reset(self, val: T) -> Self { match self { @@ -1213,6 +1217,10 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { // new config EitherOrBoth::Right((name, mut setting)) => { tracing::debug!(embedder = name, "new embedder"); + // if we are asked to reset an embedder that doesn't exist, just ignore it + if setting.is_reset() { + continue; + } // apply the default source in case the source was not set so that it gets validated crate::vector::settings::EmbeddingSettings::apply_default_source(&mut setting); crate::vector::settings::EmbeddingSettings::apply_default_openai_model( From 8dfded2993a13ea958593d8e0616660efd104758 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 10:49:03 +0200 Subject: [PATCH 069/312] Update tests --- crates/meilisearch/tests/vector/fragments.rs | 70 +++++++++++++++++++- 1 file changed, 68 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 0135e2044..915f1c79d 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -1375,7 +1375,54 @@ async fn remove_non_existant_embedder() { let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); let task = server.wait_task(response.uid()).await; - snapshot!(task, @r""); + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "non-existant": null, + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog" + }, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); } #[actix_rt::test] @@ -1412,7 +1459,26 @@ async fn double_remove_embedder() { let (response, code) = index.update_settings(settings.clone()).await; snapshot!(code, @"202 Accepted"); let task = server.wait_task(response.uid()).await; - snapshot!(task, @r#""#); + snapshot!(task, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": null + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); } #[actix_rt::test] From 6792d048b8aafd13f799fe9254a7c67bb9f07495 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 11:47:38 +0200 Subject: [PATCH 070/312] Test both fragments and document template --- crates/meilisearch/tests/vector/fragments.rs | 155 ++++++++++++++++++- 1 file changed, 152 insertions(+), 3 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 915f1c79d..25d7a7ffc 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -30,7 +30,7 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va ("dustin", [-0.5, 0.5, 0.0]), ("bulldog", [0.0, 0.0, 1.0]), ("labrador", [0.0, 0.0, -1.0]), - ("{", [-9999.0, -9999.0, -9999.0]), // That wouldn't be nice + ("{{ doc.", [-9999.0, -9999.0, -9999.0]), // If a template didn't render ] .into_iter() .collect(); @@ -68,7 +68,7 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va "dimensions": 3, "request": "{{fragment}}", "response": { - "data": "{{embedding}}" + "data": "{{embedding}}" }, "indexingFragments": { "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, @@ -1362,7 +1362,115 @@ async fn multiple_embedders() { .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(documents), @r""); + snapshot!(json_string!(documents), @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + }, + "rest3": { + "embeddings": [ + [ + 0.0, + 0.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 1.0 + ] + ], + "regenerate": true + }, + "rest3": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); } #[actix_rt::test] @@ -1663,3 +1771,44 @@ async fn complex_fragment() { } "#); } + +#[actix_rt::test] +async fn both_fragments_and_document_template() { + let server = Server::new().await; + let index = server.unique_index(); + + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + + let settings = json!({ + "embedders": { + "rest": { + "source": "rest", + "url": "http://localhost:1337", + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "basic": {"value": "{{ doc.name }} is a dog"}, + }, + "searchFragments": { + "justBreed": {"value": "It's a {{ media.breed }}"}, + }, + "documentTemplate": "{{ doc.name }} is a dog", + }, + }, + }); + + let (response, code) = index.update_settings(settings.clone()).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(response, @r#" + { + "message": "Error while generating embeddings: user error: cannot pass both fragments and a document template.\n - Note: 1 fragments declared in `indexingFragments` and 1 fragments declared in `search_fragments_len`.\n - Hint: remove the declared fragments or remove the `documentTemplate`", + "code": "vector_embedding_error", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#vector_embedding_error" + } + "#); +} From 48527761e72bd04e87f28269430bccb31395e173 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 12:01:15 +0200 Subject: [PATCH 071/312] Add test --- crates/meilisearch/tests/vector/fragments.rs | 113 +++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 25d7a7ffc..3c0c154d5 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -1812,3 +1812,116 @@ async fn both_fragments_and_document_template() { } "#); } + +#[actix_rt::test] +async fn set_fragments_then_document_template() { + let (server, uid, settings) = init_fragments_index().await; + let index = server.index(uid); + + let url = settings["embedders"]["rest"]["url"].as_str().unwrap(); + + let settings = json!({ + "embedders": { + "rest": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "documentTemplate": "{{ doc.name }} is a dog", + }, + }, + }); + + let (response, code) = index.update_settings(settings.clone()).await; + snapshot!(code, @"202 Accepted"); + let task = server.wait_task(response.uid()).await; + snapshot!(task, @r""); + + let (settings, code) = index.settings().await; + snapshot!(code, @"200 OK"); + snapshot!(settings, @r#" + { + "displayedAttributes": [ + "*" + ], + "searchableAttributes": [ + "*" + ], + "filterableAttributes": [], + "sortableAttributes": [], + "rankingRules": [ + "words", + "typo", + "proximity", + "attribute", + "sort", + "exactness" + ], + "stopWords": [], + "nonSeparatorTokens": [], + "separatorTokens": [], + "dictionary": [], + "synonyms": {}, + "distinctAttribute": null, + "proximityPrecision": "byWord", + "typoTolerance": { + "enabled": true, + "minWordSizeForTypos": { + "oneTypo": 5, + "twoTypos": 9 + }, + "disableOnWords": [], + "disableOnAttributes": [], + "disableOnNumbers": false + }, + "faceting": { + "maxValuesPerFacet": 100, + "sortFacetValuesBy": { + "*": "alpha" + } + }, + "pagination": { + "maxTotalHits": 1000 + }, + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "http://127.0.0.1:55578", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a dog" + }, + "withBreed": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "headers": {} + } + }, + "searchCutoffMs": null, + "localizedAttributes": null, + "facetSearch": true, + "prefixSearch": "indexingTime" + } + "#); +} + From b274106ad3adb3a3224d29114ecb2795a09676a7 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 13:05:52 +0200 Subject: [PATCH 072/312] Add test --- crates/meilisearch/tests/vector/fragments.rs | 24 ++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 3c0c154d5..cf4ed4ab4 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -267,6 +267,30 @@ async fn search_with_media() { "#); } +#[actix_rt::test] +async fn search_with_media_and_vector() { + let index = shared_index_for_fragments().await; + + let (value, code) = index + .search_post(json!({ + "vector": [1.0, 1.0, 1.0], + "media": { "breed": "labrador" }, + "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, + "limit": 1 + } + )) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid request: both `media` and `vector` parameters are present.", + "code": "invalid_search_media_and_vector", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_media_and_vector" + } + "#); +} + #[actix_rt::test] async fn search_with_media_matching_multiple_fragments() { let index = shared_index_for_fragments().await; From be9f4f96dfd295a90c92f286f2b5903af209abad Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 13:15:15 +0200 Subject: [PATCH 073/312] Add experimental feature test --- crates/meilisearch/tests/vector/fragments.rs | 40 +++++++++++++++++++- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index cf4ed4ab4..0dde9dfc2 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -105,14 +105,50 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va (server, uid, settings) } -// TODO: Test cannot pass both fragments and document - // TODO: edit fragment // TODO: document fragment replaced // TODO: swapping fragments +// TODO: consistency + +#[actix_rt::test] +async fn experimental_feature_not_enabled() { + let server = Server::new().await; + let index = server.unique_index(); + + let settings = json!({ + "embedders": { + "rest": { + "source": "rest", + "url": "http://localhost:1337", + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "basic": {"value": "{{ doc.name }} is a dog"}, + }, + "searchFragments": { + "query": {"value": "Some pre-prompt for query {{ q }}"}, + } + }, + }, + }); + let (response, code) = index.update_settings(settings.clone()).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(response, @r#" + { + "message": "setting `indexingFragments` requires enabling the `multimodal` experimental feature. See https://github.com/orgs/meilisearch/discussions/846", + "code": "feature_not_enabled", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#feature_not_enabled" + } + "#); +} + #[actix_rt::test] async fn indexing_fragments() { let index = shared_index_for_fragments().await; From 16234e1313c4d76b000c5c1ea1d229c6ff2c26c1 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 13:25:42 +0200 Subject: [PATCH 074/312] Add fragment swapping test --- crates/meilisearch/tests/vector/fragments.rs | 160 ++++++++++++++++++- 1 file changed, 153 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 0dde9dfc2..f083e40d0 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -105,14 +105,8 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va (server, uid, settings) } -// TODO: edit fragment - // TODO: document fragment replaced -// TODO: swapping fragments - -// TODO: consistency - #[actix_rt::test] async fn experimental_feature_not_enabled() { let server = Server::new().await; @@ -158,7 +152,7 @@ async fn indexing_fragments() { .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(documents), @r#" + snapshot!(documents, @r#" { "results": [ { @@ -721,6 +715,158 @@ async fn modifying_fragments_modifies_vectors() { "#); } +#[actix_rt::test] +async fn swapping_fragments() { + let (server, uid, mut settings) = init_fragments_index().await; + let index = server.index(uid); + + let basic = settings["embedders"]["rest"]["indexingFragments"]["basic"].clone(); + let with_breed = settings["embedders"]["rest"]["indexingFragments"]["withBreed"].clone(); + settings["embedders"]["rest"]["indexingFragments"]["basic"] = with_breed; + settings["embedders"]["rest"]["indexingFragments"]["withBreed"] = basic; + + let (response, code) = index.update_settings(settings).await; + snapshot!(code, @"202 Accepted"); + let value = server.wait_task(response.uid()).await.succeeded(); + snapshot!(value, @r#" + { + "uid": "[uid]", + "batchUid": "[batch_uid]", + "indexUid": "[uuid]", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "rest": { + "source": "rest", + "dimensions": 3, + "url": "[url]", + "indexingFragments": { + "basic": { + "value": "{{ doc.name }} is a {{ doc.breed }}" + }, + "withBreed": { + "value": "{{ doc.name }} is a dog" + } + }, + "searchFragments": { + "justBreed": { + "value": "It's a {{ media.breed }}" + }, + "justName": { + "value": "{{ media.name }} is a dog" + }, + "query": { + "value": "Some pre-prompt for query {{ q }}" + } + }, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + } + } + } + }, + "error": null, + "duration": "[duration]", + "enqueuedAt": "[date]", + "startedAt": "[date]", + "finishedAt": "[date]" + } + "#); + + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(documents, @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + -1.0 + ], + [ + 1.0, + 1.0, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 1.0 + ], + [ + -0.5, + 0.5, + 0.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} + #[actix_rt::test] async fn ommitted_fragment_isnt_removed() { let (server, uid, mut settings) = init_fragments_index().await; From c5993196b3c925dccd811afdbe3b9da5e112ed2b Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 13:32:55 +0200 Subject: [PATCH 075/312] Add test --- crates/meilisearch/tests/vector/fragments.rs | 115 ++++++++++++++++++- 1 file changed, 113 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index f083e40d0..70e4433ed 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -105,8 +105,6 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va (server, uid, settings) } -// TODO: document fragment replaced - #[actix_rt::test] async fn experimental_feature_not_enabled() { let server = Server::new().await; @@ -239,6 +237,119 @@ async fn indexing_fragments() { "#); } +#[actix_rt::test] +async fn replace_document() { + let (server, uid, _settings) = init_fragments_index().await; + let index = server.index(uid); + + let documents = json!([ + { "id": 0, "name": "kefir", "breed": "sorry-I-forgot" }, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + + let task = index.wait_task(value.uid()).await; + snapshot!(task["status"], @r###""succeeded""###); + + // Make sure kefir now has 2 vectors + let (documents, code) = index + .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) + .await; + snapshot!(code, @"200 OK"); + snapshot!(documents, @r#" + { + "results": [ + { + "id": 0, + "name": "kefir", + "breed": "sorry-I-forgot", + "_vectors": { + "rest": { + "embeddings": [ + [ + 0.5, + -0.5, + 0.0 + ], + [ + 0.5, + -0.5, + 0.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 1, + "name": "echo", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 1.0 + ] + ], + "regenerate": false + } + } + }, + { + "id": 2, + "name": "intel", + "breed": "labrador", + "_vectors": { + "rest": { + "embeddings": [ + [ + 1.0, + 1.0, + 0.0 + ], + [ + 1.0, + 1.0, + -1.0 + ] + ], + "regenerate": true + } + } + }, + { + "id": 3, + "name": "dustin", + "breed": "bulldog", + "_vectors": { + "rest": { + "embeddings": [ + [ + -0.5, + 0.5, + 0.0 + ], + [ + -0.5, + 0.5, + 1.0 + ] + ], + "regenerate": true + } + } + } + ], + "offset": 0, + "limit": 20, + "total": 4 + } + "#); +} + + #[actix_rt::test] async fn search_with_vector() { let index = shared_index_for_fragments().await; From fa3990daf90920bec99cf4051afb48ce46e79a66 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 4 Jul 2025 13:33:49 +0200 Subject: [PATCH 076/312] Format --- crates/meilisearch/tests/vector/fragments.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 70e4433ed..337f01ca6 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -349,7 +349,6 @@ async fn replace_document() { "#); } - #[actix_rt::test] async fn search_with_vector() { let index = shared_index_for_fragments().await; @@ -891,7 +890,7 @@ async fn swapping_fragments() { .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) .await; snapshot!(code, @"200 OK"); - snapshot!(documents, @r#" + snapshot!(documents, @r#" { "results": [ { @@ -1795,7 +1794,10 @@ async fn remove_non_existant_embedder() { let (server, uid, mut settings) = init_fragments_index().await; let index = server.index(uid); - settings["embedders"].as_object_mut().unwrap().insert(String::from("non-existant"), serde_json::Value::Null); + settings["embedders"] + .as_object_mut() + .unwrap() + .insert(String::from("non-existant"), serde_json::Value::Null); let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); @@ -1855,7 +1857,10 @@ async fn double_remove_embedder() { let (server, uid, mut settings) = init_fragments_index().await; let index = server.index(uid); - settings["embedders"].as_object_mut().unwrap().insert(String::from("rest"), serde_json::Value::Null); + settings["embedders"] + .as_object_mut() + .unwrap() + .insert(String::from("rest"), serde_json::Value::Null); let (response, code) = index.update_settings(settings.clone()).await; snapshot!(code, @"202 Accepted"); @@ -2241,4 +2246,3 @@ async fn set_fragments_then_document_template() { } "#); } - From 73c9c1ebdcd00d53483934e79ea6706e2d5a0586 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 7 Jul 2025 11:33:01 +0200 Subject: [PATCH 077/312] Add compile-time checks for dumpless upgrade --- crates/milli/src/update/upgrade/mod.rs | 78 ++++++++++++++++++-------- 1 file changed, 54 insertions(+), 24 deletions(-) diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index 9f64ca0e3..c23e5c8b1 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -24,6 +24,57 @@ trait UpgradeIndex { fn target_version(&self) -> (u32, u32, u32); } +const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[ + &V1_12_To_V1_12_3 {}, + &V1_12_3_To_V1_13_0 {}, + &V1_13_0_To_V1_13_1 {}, + &V1_13_1_To_Latest_V1_13 {}, + &Latest_V1_13_To_Latest_V1_14 {}, + &Latest_V1_14_To_Latest_V1_15 {}, + // This is the last upgrade function, it will be called when the index is up to date. + // any other upgrade function should be added before this one. + &ToCurrentNoOp {}, +]; + +/// Causes a compile-time error if the argument is not in range of `0..UPGRADE_FUNCTIONS.len()` +macro_rules! function_index { + ($start:expr) => {{ + const _CHECK_INDEX: () = { + if $start >= $crate::update::upgrade::UPGRADE_FUNCTIONS.len() { + panic!("upgrade functions out of range") + } + }; + + $start + }}; +} + +const fn start(from: (u32, u32, u32)) -> Option { + let start = match from { + (1, 12, 0..=2) => function_index!(0), + (1, 12, 3..) => function_index!(1), + (1, 13, 0) => function_index!(2), + (1, 13, _) => function_index!(4), + (1, 14, _) => function_index!(5), + // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. + (1, 15, _) => function_index!(6), + // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually + // considering dumpless upgrade. + (_major, _minor, _patch) => return None, + }; + + Some(start) +} + +/// Causes a compile-time error if the latest package cannot be upgraded. +/// +/// This serves as a reminder to consider the proper dumpless upgrade implementation when changing the package version. +const _CHECK_PACKAGE_CAN_UPGRADE: () = { + if start((VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)).is_none() { + panic!("cannot upgrade from latest package version") + } +}; + /// Return true if the cached stats of the index must be regenerated pub fn upgrade( wtxn: &mut RwTxn, @@ -36,33 +87,12 @@ where MSP: Fn() -> bool + Sync, { let from = index.get_version(wtxn)?.unwrap_or(db_version); - let upgrade_functions: &[&dyn UpgradeIndex] = &[ - &V1_12_To_V1_12_3 {}, - &V1_12_3_To_V1_13_0 {}, - &V1_13_0_To_V1_13_1 {}, - &V1_13_1_To_Latest_V1_13 {}, - &Latest_V1_13_To_Latest_V1_14 {}, - &Latest_V1_14_To_Latest_V1_15 {}, - // This is the last upgrade function, it will be called when the index is up to date. - // any other upgrade function should be added before this one. - &ToCurrentNoOp {}, - ]; - let start = match from { - (1, 12, 0..=2) => 0, - (1, 12, 3..) => 1, - (1, 13, 0) => 2, - (1, 13, _) => 4, - (1, 14, _) => 5, - // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. - (1, 15, _) => 6, - (major, minor, patch) => { - return Err(InternalError::CannotUpgradeToVersion(major, minor, patch).into()) - } - }; + let start = + start(from).ok_or_else(|| InternalError::CannotUpgradeToVersion(from.0, from.1, from.2))?; enum UpgradeVersion {} - let upgrade_path = &upgrade_functions[start..]; + let upgrade_path = &UPGRADE_FUNCTIONS[start..]; let mut current_version = from; let mut regenerate_stats = false; From a3254d7d7d9085aa8c83929cec1f64fb88c1e686 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 7 Jul 2025 11:57:08 +0200 Subject: [PATCH 078/312] Implement dumpless upgrade from v1.15 to v1.16 --- crates/milli/src/update/upgrade/mod.rs | 4 ++ crates/milli/src/update/upgrade/v1_15.rs | 13 +++++++ crates/milli/src/update/upgrade/v1_16.rs | 48 ++++++++++++++++++++++++ crates/milli/src/vector/db.rs | 7 ++++ 4 files changed, 72 insertions(+) create mode 100644 crates/milli/src/update/upgrade/v1_16.rs diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index c23e5c8b1..f53319a37 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -2,6 +2,7 @@ mod v1_12; mod v1_13; mod v1_14; mod v1_15; +mod v1_16; use heed::RwTxn; use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3}; use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13}; @@ -10,6 +11,7 @@ use v1_15::Latest_V1_14_To_Latest_V1_15; use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use crate::progress::{Progress, VariableNameStep}; +use crate::update::upgrade::v1_16::Latest_V1_15_To_V1_16_0; use crate::{Index, InternalError, Result}; trait UpgradeIndex { @@ -31,6 +33,7 @@ const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[ &V1_13_1_To_Latest_V1_13 {}, &Latest_V1_13_To_Latest_V1_14 {}, &Latest_V1_14_To_Latest_V1_15 {}, + &Latest_V1_15_To_V1_16_0 {}, // This is the last upgrade function, it will be called when the index is up to date. // any other upgrade function should be added before this one. &ToCurrentNoOp {}, @@ -58,6 +61,7 @@ const fn start(from: (u32, u32, u32)) -> Option { (1, 14, _) => function_index!(5), // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. (1, 15, _) => function_index!(6), + (1, 16, _) => function_index!(7), // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually // considering dumpless upgrade. (_major, _minor, _patch) => return None, diff --git a/crates/milli/src/update/upgrade/v1_15.rs b/crates/milli/src/update/upgrade/v1_15.rs index cea4783a1..9ca25d06b 100644 --- a/crates/milli/src/update/upgrade/v1_15.rs +++ b/crates/milli/src/update/upgrade/v1_15.rs @@ -1,4 +1,6 @@ use heed::RwTxn; +use roaring::RoaringBitmap; +use serde::{Deserialize, Serialize}; use super::UpgradeIndex; use crate::progress::Progress; @@ -26,3 +28,14 @@ impl UpgradeIndex for Latest_V1_14_To_Latest_V1_15 { (1, 15, 0) } } + +/// Parts of v1.15 `IndexingEmbeddingConfig` that are relevant for upgrade to v1.16 +/// +/// # Warning +/// +/// This object should not be rewritten to the DB, only read to get the name and `user_provided` roaring. +#[derive(Debug, Deserialize, Serialize)] +pub struct IndexEmbeddingConfig { + pub name: String, + pub user_provided: RoaringBitmap, +} diff --git a/crates/milli/src/update/upgrade/v1_16.rs b/crates/milli/src/update/upgrade/v1_16.rs new file mode 100644 index 000000000..f43efd77d --- /dev/null +++ b/crates/milli/src/update/upgrade/v1_16.rs @@ -0,0 +1,48 @@ +use heed::types::{SerdeJson, Str}; +use heed::RwTxn; + +use super::UpgradeIndex; +use crate::progress::Progress; +use crate::vector::db::{EmbedderInfo, EmbeddingStatus}; +use crate::{Index, InternalError, Result}; + +#[allow(non_camel_case_types)] +pub(super) struct Latest_V1_15_To_V1_16_0(); + +impl UpgradeIndex for Latest_V1_15_To_V1_16_0 { + fn upgrade( + &self, + wtxn: &mut RwTxn, + index: &Index, + _original: (u32, u32, u32), + _progress: Progress, + ) -> Result { + let v1_15_indexing_configs = index + .main + .remap_types::>>() + .get(wtxn, crate::index::main_key::EMBEDDING_CONFIGS)? + .unwrap_or_default(); + + let embedders = index.embedding_configs(); + for config in v1_15_indexing_configs { + let embedder_id = embedders.embedder_id(wtxn, &config.name)?.ok_or( + InternalError::DatabaseMissingEntry { + db_name: crate::index::db_name::VECTOR_EMBEDDER_CATEGORY_ID, + key: None, + }, + )?; + let info = EmbedderInfo { + embedder_id, + // v1.15 used not to make a difference between `user_provided` and `! regenerate`. + embedding_status: EmbeddingStatus::from_user_provided(config.user_provided), + }; + embedders.put_embedder_info(wtxn, &config.name, &info)?; + } + + Ok(false) + } + + fn target_version(&self) -> (u32, u32, u32) { + (1, 16, 0) + } +} diff --git a/crates/milli/src/vector/db.rs b/crates/milli/src/vector/db.rs index 0e890fac9..2fea75d68 100644 --- a/crates/milli/src/vector/db.rs +++ b/crates/milli/src/vector/db.rs @@ -117,6 +117,13 @@ impl EmbeddingStatus { Default::default() } + /// Create a new `EmbeddingStatus` that assumes that any `user_provided` docid is also skipping regenerate. + /// + /// Used for migration from v1.15 and earlier DBs. + pub(crate) fn from_user_provided(user_provided: RoaringBitmap) -> Self { + Self { user_provided, skip_regenerate_different_from_user_provided: Default::default() } + } + /// Whether the document contains user-provided vectors for that embedder. pub fn is_user_provided(&self, docid: DocumentId) -> bool { self.user_provided.contains(docid) From 132065afda95a6e4223de59f754e4155f162669e Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 7 Jul 2025 13:10:16 +0200 Subject: [PATCH 079/312] Minor improvements --- crates/meilisearch/tests/vector/fragments.rs | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 337f01ca6..db57f9f6e 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -85,8 +85,7 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va let (response, code) = index.update_settings(settings.clone()).await; snapshot!(code, @"202 Accepted"); - let task = server.wait_task(response.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); + server.wait_task(response.uid()).await.succeeded(); // Send documents let documents = json!([ @@ -1031,6 +1030,7 @@ async fn ommitted_fragment_isnt_removed() { } "#); + // Make sure withBreed is still here because it wasn't specified let (value, code) = index.settings().await; snapshot!(code, @"200 OK"); snapshot!(json_string!(value["embedders"], { @@ -1283,7 +1283,7 @@ async fn multiple_embedders() { }); let (response, code) = index.update_settings(settings2).await; snapshot!(code, @"202 Accepted"); - let task = server.wait_task(response.uid()).await; + let task = server.wait_task(response.uid()).await.succeeded(); snapshot!(task, @r#" { "uid": "[uid]", @@ -1531,8 +1531,7 @@ async fn multiple_embedders() { let (response, code) = index.update_settings(settings.clone()).await; snapshot!(code, @"202 Accepted"); - let value = server.wait_task(response.uid()).await.succeeded(); - snapshot!(value["status"], @r###""succeeded""###); + server.wait_task(response.uid()).await.succeeded(); let (documents, code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) @@ -1671,8 +1670,7 @@ async fn multiple_embedders() { let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); - let value = server.wait_task(response.uid()).await.succeeded(); - snapshot!(value["status"], @r###""succeeded""###); + server.wait_task(response.uid()).await.succeeded(); let (documents, code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) @@ -1801,7 +1799,7 @@ async fn remove_non_existant_embedder() { let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); - let task = server.wait_task(response.uid()).await; + let task = server.wait_task(response.uid()).await.succeeded(); snapshot!(task, @r#" { "uid": "[uid]", @@ -1864,7 +1862,7 @@ async fn double_remove_embedder() { let (response, code) = index.update_settings(settings.clone()).await; snapshot!(code, @"202 Accepted"); - let task = server.wait_task(response.uid()).await; + let task = server.wait_task(response.uid()).await.succeeded(); snapshot!(task, @r#" { "uid": "[uid]", @@ -1888,7 +1886,7 @@ async fn double_remove_embedder() { let (response, code) = index.update_settings(settings.clone()).await; snapshot!(code, @"202 Accepted"); - let task = server.wait_task(response.uid()).await; + let task = server.wait_task(response.uid()).await.succeeded(); snapshot!(task, @r#" { "uid": "[uid]", @@ -1933,7 +1931,7 @@ async fn complex_fragment() { let (response, code) = index.update_settings(settings).await; snapshot!(code, @"202 Accepted"); - let task = server.wait_task(response.uid()).await; + let task = server.wait_task(response.uid()).await.succeeded(); snapshot!(task, @r#" { "uid": "[uid]", From a9bb64c55a285b9cf90d29e85f9d622cbf893887 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 7 Jul 2025 15:28:10 +0200 Subject: [PATCH 080/312] Unrelated minor fixes --- crates/meilisearch/src/routes/indexes/documents.rs | 2 -- crates/milli/src/filterable_attributes_rules.rs | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index a93d736f7..bc5539081 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -1461,8 +1461,6 @@ fn some_documents<'a, 't: 'a>( document.remove("_vectors"); } RetrieveVectors::Retrieve => { - // Clippy is simply wrong - #[allow(clippy::manual_unwrap_or_default)] let mut vectors = match document.remove("_vectors") { Some(Value::Object(map)) => map, _ => Default::default(), diff --git a/crates/milli/src/filterable_attributes_rules.rs b/crates/milli/src/filterable_attributes_rules.rs index ae1a9755a..5ba8a99d8 100644 --- a/crates/milli/src/filterable_attributes_rules.rs +++ b/crates/milli/src/filterable_attributes_rules.rs @@ -111,7 +111,7 @@ impl FilterableAttributesFeatures { self.filter.is_filterable_null() } - /// Check if `IS EXISTS` is allowed + /// Check if `EXISTS` is allowed pub fn is_filterable_exists(&self) -> bool { self.filter.is_filterable_exists() } From 20525376815e6c0d6962744701f954bf9f03f7cb Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 7 Jul 2025 15:28:35 +0200 Subject: [PATCH 081/312] Implement core filter logic --- crates/milli/src/index.rs | 2 +- crates/milli/src/search/facet/filter.rs | 15 ++- .../milli/src/search/facet/filter_vector.rs | 123 ++++++++++++++++++ crates/milli/src/search/facet/mod.rs | 1 + .../src/update/index_documents/transform.rs | 2 +- 5 files changed, 138 insertions(+), 5 deletions(-) create mode 100644 crates/milli/src/search/facet/filter_vector.rs diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index b2ec992ba..2751498bf 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -1776,7 +1776,7 @@ impl Index { embedder_info.embedder_id, config.config.quantized(), ); - let embeddings = reader.item_vectors(rtxn, docid)?; + let embeddings = reader.item_vectors(rtxn, docid)?; // MARKER res.insert( config.name.to_owned(), (embeddings, embedder_info.embedding_status.must_regenerate(docid)), diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index c3eba8031..f80d1681f 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -10,7 +10,7 @@ use memchr::memmem::Finder; use roaring::{MultiOps, RoaringBitmap}; use serde_json::Value; -use super::facet_range_search; +use super::{facet_range_search, filter_vector::VectorFilter}; use crate::constants::RESERVED_GEO_FIELD_NAME; use crate::error::{Error, UserError}; use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features}; @@ -234,8 +234,11 @@ impl<'a> Filter<'a> { pub fn evaluate(&self, rtxn: &heed::RoTxn<'_>, index: &Index) -> Result { // to avoid doing this for each recursive call we're going to do it ONCE ahead of time let fields_ids_map = index.fields_ids_map(rtxn)?; - let filterable_attributes_rules = index.filterable_attributes_rules(rtxn)?; + let filterable_attributes_rules = dbg!(index.filterable_attributes_rules(rtxn)?); + for fid in self.condition.fids(MAX_FILTER_DEPTH) { + println!("{fid:?}"); + let attribute = fid.value(); if matching_features(attribute, &filterable_attributes_rules) .is_some_and(|(_, features)| features.is_filterable()) @@ -542,7 +545,13 @@ impl<'a> Filter<'a> { .union() } FilterCondition::Condition { fid, op } => { - let Some(field_id) = field_ids_map.id(fid.value()) else { + let value = fid.value(); + if VectorFilter::matches(value, op) { + let vector_filter = VectorFilter::parse(value)?; + return vector_filter.evaluate(rtxn, index, universe); + } + + let Some(field_id) = field_ids_map.id(value) else { return Ok(RoaringBitmap::new()); }; let Some((rule_index, features)) = diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs new file mode 100644 index 000000000..701ab561c --- /dev/null +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -0,0 +1,123 @@ +use filter_parser::Condition; +use roaring::RoaringBitmap; + +use crate::error::{Error, UserError}; +use crate::vector::{ArroyStats, ArroyWrapper}; +use crate::{Index, Result}; + +pub(super) struct VectorFilter<'a> { + embedder_name: &'a str, + fragment_name: Option<&'a str>, + user_provided: bool, + // TODO: not_user_provided: bool, +} + +impl<'a> VectorFilter<'a> { + pub(super) fn matches(value: &str, op: &Condition) -> bool { + matches!(op, Condition::Exists) && value.starts_with("_vectors.") + } + + /// Parses a vector filter string. + /// + /// Valid formats: + /// - `_vectors.{embedder_name}` + /// - `_vectors.{embedder_name}.userProvided` + /// - `_vectors.{embedder_name}.fragments.{fragment_name}` + /// - `_vectors.{embedder_name}.fragments.{fragment_name}.userProvided` + pub(super) fn parse(s: &'a str) -> Result { + let mut split = s.split('.').peekable(); + + if split.next() != Some("_vectors") { + return Err(Error::UserError(UserError::InvalidFilter(String::from( + "Vector filter must start with '_vectors'", + )))); + } + + let embedder_name = split.next().ok_or_else(|| { + Error::UserError(UserError::InvalidFilter(String::from( + "Vector filter must contain an embedder name", + ))) + })?; + + let mut fragment_name = None; + if split.peek() == Some(&"fragments") { + split.next(); + + fragment_name = Some(split.next().ok_or_else(|| { + Error::UserError(UserError::InvalidFilter( + String::from("Vector filter is inconsistent: either specify a fragment name or remove the 'fragments' part"), + )) + })?); + } + + let mut user_provided = false; + if split.peek() == Some(&"userProvided") || split.peek() == Some(&"user_provided") { + split.next(); + user_provided = true; + } + + if let Some(next) = split.next() { + return Err(Error::UserError(UserError::InvalidFilter(format!( + "Unexpected part in vector filter: '{next}'" + )))); + } + + Ok(Self { embedder_name, fragment_name, user_provided }) + } + + pub(super) fn evaluate( + &self, + rtxn: &heed::RoTxn<'_>, + index: &Index, + universe: Option<&RoaringBitmap>, + ) -> Result { + let index_embedding_configs = index.embedding_configs(); + let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; + + let Some(embedder_config) = + embedding_configs.iter().find(|config| config.name == self.embedder_name) + else { + return Ok(RoaringBitmap::new()); + }; + let Some(embedder_info) = + index_embedding_configs.embedder_info(rtxn, self.embedder_name)? + else { + return Ok(RoaringBitmap::new()); + }; + + let arroy_wrapper = ArroyWrapper::new( + index.vector_arroy, + embedder_info.embedder_id, + embedder_config.config.quantized(), + ); + + let mut docids = if let Some(fragment_name) = self.fragment_name { + let Some(fragment_config) = embedder_config + .fragments + .as_slice() + .iter() + .find(|fragment| fragment.name == fragment_name) + else { + return Ok(RoaringBitmap::new()); + }; + + arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + } else { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents + }; + + // FIXME: performance + if self.user_provided { + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + docids &= user_provided_docsids; + } + + if let Some(universe) = universe { + docids &= universe; + } + + Ok(docids) + } +} diff --git a/crates/milli/src/search/facet/mod.rs b/crates/milli/src/search/facet/mod.rs index a5e65c95d..fac85df59 100644 --- a/crates/milli/src/search/facet/mod.rs +++ b/crates/milli/src/search/facet/mod.rs @@ -17,6 +17,7 @@ mod facet_range_search; mod facet_sort_ascending; mod facet_sort_descending; mod filter; +mod filter_vector; mod search; fn facet_extreme_value<'t>( diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index e07483aff..d69768d4b 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -966,7 +966,7 @@ impl<'a, 'i> Transform<'a, 'i> { // some user provided, remove only the ids that are not user provided let to_delete = arroy.items_in_store(wtxn, *fragment_id, |items| { items - infos.embedding_status.user_provided_docids() - })?; + })?; // MARKER for to_delete in to_delete { arroy.del_item_in_store(wtxn, to_delete, *fragment_id, dimensions)?; From f7c8a77f89c2d449492421de2f812613bb8e4234 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 7 Jul 2025 16:01:50 +0200 Subject: [PATCH 082/312] Update v1.12.0 DB to contain vectors --- .../upgrade/v1_12/v1_12_0.ms/auth/lock.mdb | Bin 8192 -> 8192 bytes .../data.mdb | Bin 163840 -> 229376 bytes .../lock.mdb | Bin 65664 -> 65664 bytes .../upgrade/v1_12/v1_12_0.ms/tasks/data.mdb | Bin 212992 -> 225280 bytes .../upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb | Bin 8192 -> 8192 bytes 5 files changed, 0 insertions(+), 0 deletions(-) diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/auth/lock.mdb index 4c80ffe2c03fba62008aca5da473370d7b421a38..80fb2b9d5b76d935cb8bddeabccb21fee83a51a6 100644 GIT binary patch delta 53 zcmZp0XmAj}ci{a#rUjiEtPBvq2&C3BF;2ACohYL*F+pTvg8(ao!H;~#jSJ)DCnoR! E0Ha9{P5=M^ delta 42 ucmZp0XmAj`ci{a#rUZQ*Rt5-QoM>r0(Zz@DfY@b+`_&s4#>-Dk;06F8#|_f} diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/data.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/data.mdb index c31db3415612b78106346996d5ff20d3fd293ae9..95ca0a9da78bb8e4d843ec3e7b4450b4b92fe076 100644 GIT binary patch literal 229376 zcmeF42YeL8`~Uau(v!PA0!S6Y(IOB`g#g*PP(%a-R74-BC2yQ6xQaDulDgKC4I{^SapgzKwmb z0j2;`fGNNfUqSTcj+W9^ot^vqE~)&xqHm!^EH@)0@&L(-_t z`dp&bl9E6S1d%F|*TmR(i#^C@&$Ol`xvZ>OZgr%E+N@Kx&BQk0E)A-s)+rTNN?Dtu z#Ukai+JY>p8TL`MxGoKOX z)yu8X>kyP^vr|IVhby~v)Pkr-8mbAjrlxXRY3Z4%u_@NkmIUSU{@JVCpi(I-)LAK| zd;__>BI-T!cqMBU>P+qvpnPhbYgMjuJ#J)rWY4%Dc)e8Rpa;syS0g7#UwQJCtS6Ssp z+fqNO)GJfTR+UwKv@N|-9*;`ps;gX)SZgM^Ea}{x96!;HT>cyb7^{;f#{2fcxDweFGp+}3R3h%8JmzL=et&POZ%0q%@x6Z}yAo?$Y-kvH)L{6fAvUurP$zZw2xD$EMX6krN41(*U%0j2;`fGNNf zU;xa6DEAiKn4^zg2vq&C16;eDod!~hq0}&UKHElZcyaN`R*Rab+6}2xRu3r$sTocdN_a~t z<%vpaSxMwUeoE?T>4=Ytg;a(5RjD@O(zD2aMBQttxqMcx_S-Mj<5?|2n^t@XhBfo<>htSbBE^~ z&t%VN&)S~XJPJLQdt`g`@d)(bJdV3>cAw@x)IHMO=w9kp;I_yu!>yZJfZH8iv2Lv{ zN7r8$tn<>H*6z^G(I#u7wY9a^G=-YwnruxUO`wL;99QZET~U7s;&^w5o)J&Hhte!$ zTa!axyxL3t8PZb5{Y>{=az6pN+d%FXkh=-wZUDLKK<*lly9(s40J+OR?h=su5y)Kx zauZfAwwQTS>7$<^QjG=-01YqCe;VAF7f0VIaSl z58%W3Q~W`GGe4P6HBwfvqt(gK$0j2;`fGNNfU!|q=E0Ui=1zPbesE|@V{;zg#NbP^bP4_;`_u{8}NBO1rdHF8Z zH{pLWg!|U^Tkm(pcLZPTJID99p}*cwf5K<4Ur$3W&*J|nLbDZ9fGNNfU@i4vyvV@zt| zkn{{G_?b;g>z0+2MC^1WQ3Ivj)bs>P3bA4%p%N`=@s=QaW^9@*kwl)eWZIsLC2{ti zwWe8nJ{#1tP0)x?Vm;F`!fGSN$T^nqcuTv;wrykM62mMBiI$MyxZs4y(6*5gmdLQU zg!s62ZNn^~p%!vh0*SGgX^qR0ieYLm#qS$Cwv~xQ79)`XrDE7gh`+K^TzE6FQltfjqtb?b5|>G3pX-;h{4t+K^7Br_|; zB3-hwq#+q(lSB$5)rm1xsH8D zZqyi43nE%i62&k+Jt;FbBiR~HMX^lGmRyF%F?3eY?j!}_~aq+S@!fq;#Djy z4WpH#b-0XJa%05GQSyk|LH2YziDQ;*RU>F}yp`IiL1Kz!2Pp-}AVDqVP><9c7f*d+ zjY>?lvI^T22N_OnTiCKloU~L6*HS*ww?Ygyo7_@T?8z(R)WWZKpI0rY&I1>-V>4)4 zH7-4k+_|GN(rI|P6ib>LwOf;u?uwuU()>x2!YGa+Uf82j%L=0obkgROnt|%vBR85l z$R+UN9zIott2Ut?tz^n7c{@msMmBqThFsDrsydM~&d|7Jr5Ls9sIVGq^4%v}|FA^1 zU{LQ!5?d}PH8z8O7TAW6UFppvU;HrgrYnCBk#`NfQ-fP6pBv=PRW5<4eJFW9k#mPg z7euxYOl~Q1U$-)akPZ2Brj;p_Y^rCpGKG;nDW6uRaI#e~Ct@p&gux?)F}05v%Z-ss z>@CN#vq^C=e}8qSAKJ$m=j}aFqVY^8^(F%BEM1 z8asWxyMb}?p<{&&jMH}Q=8RKz?{BT~@U7OoC2?NQvS*|Yx5USVhZud-fobJF=V~0e z{tLbLc=>%48<%3Cp(brHa!m$N|CX9uvXz8tO`~@;xo7B`Kc+(6rFtuLHF70V$Q${0 z*2>i#M16(yJ6kbsg8xq6e(j~T^7-%#r3Te|fq)q$qi1s0ES_g*)w`mg=647>`iN4RJ z)1>%+rF^OCT_9UxFH8ZZ08@Y|z!YE#Fa?+bOaZ0yDxUx6x=0VrAFgWtKimi{1z`R^+@vzc6!QE(xKUb8m-GMO29;8}n*R?sNK5GA z`G0V8Wixflo=;fC^Z(%HXz41x>!3#5Fe>M%!|(y}zE@eu8tUawUuD03IX<8|TfnDM z96&WyMvg|9DX9myL>#Qy@rB149un<2n(M_;U8tIyH* z*9Ysp^r!uH_|5T4_KWuWuZjP+hzjpA{?!{mZXu9c0OaNaxp_b?56I00a&v&(Y#=ua z$jt&<@L1y?Ii%Ycpw)CjUI^1G!#6?pYw$6Ug-da@~R4GeE8zkb4@) zbp>)y0l6+ft}~G91mrpbxo9Bw0LcBuar7flZZm6$3x6o5@ibY<+^G~3r>z%bA`@=N zW-F6@&pNtcP0u3#ra_#}EhMCM1m_k=@c{CGfq<;E05WGMEg>Mz5}TA_39x7GS!)ZR z4+}8SFL601QanwFl#@Iqh2+#MM%pC#kk_&eYo;9QpQO^Jyp*Xfq;)vw7DP`}^OT== z5FkH&AV7{pP&wnWehH~3y#{xVzrmm5dl^Fc|Lpd^S9!etyTI#x2gv;loK+XZ=@`2pPKyD|H+X3W0 z0&?4d+%_QhA&}b&}c&a%+Ly8X&hC$gKi$ z?*h4%K<*tNw*ts52Xf1R+)^O-HjsM@$WccT*W*8nf%e{1NBuQiC^Cjk0U zGvxKJJ^!!lbhg11Uue_s{pl7mH)rC zEM$|r3M*^%`G3uneQE>X|6g93@-1pZ-S<+nqBbG^|LUpbA*NItp2~(nSyP)X{r?|r zP*u;U=0GjF^8a^jux_Oiy72#ZZLUt`@>TGKufqS|wK@MvrK{iz|FHl6qfMzt!Yh{y z?*FeEc=S~ct5nJgbyiAA0xgnIWE663ty(2(73%!3|No;+sdvj`mFxU} zwGfY9KQt$(E{(fK;_S zK)g!VK;<93u55`!SIv%%x6@1d=PqVrH4j>*zx({Z>WVpP?f?Gye{Gcf%f1R9^8Z(B z%hpT*rT|lbDZmt93NQtj0!#s>08@Y|z!dneR^Wfc|6lDLmU_|~aNYPH_#^ygejz`U zAI*2;|EsSss~l5+DZmt93NQtj0!#s>08@Y|z!YE#Fa`d!0@O>)S87HRCmlI7fc)Yi zTOwZeaypF=Kws$*GX5?fv0FZ!K%xlHSOQ$7v!v*KW&b|1JtIAdaD<9WAeRp0(tsTO z>EQbNEd^+AIFK6#jvbW26A12+*3fV z3y|v!j&421anjJgrl2|9k5<-|j&jbQ66M9L ztkp;#lDj|yE{?y*&*6{q4t_PC&ByVDUdz3*z4~|sk_R9j_uT9`&2y+{q^Hrd)T6*- zkw=C{H;(|1JMP8qYu$6)`@09bd%2%>+u=6HE!i#Ft+v}WU7>EdE?d_}7pUWO$F-Za z)3igiky@j+R8yc?q{+~9(*$VlaK+qOTF5_mttB7x;<$Dca{2o%63Dd$a&3TI1dvl= z!MLnP7|>oQkP88F!9eawAlDkm1p&EMK&~Z_YXRh*0CIsqPI;HQybqcI?KK5*O@Lfu zAZG${jeuMLkZTC!8UVS+fn0qcR}aW3L9krj2TF7+mt1XNzVw6Db^DQgy-178d}{)^ z8bGc(kn;y}$`_u?dKiKBcpzs0a(W==2jqN#oDY!m26E)9T3TG{>NU|n)hNdHzG>(vnk1YHeMt-~Hj1;hKS0CI_C;yC z@&(`4tA)Xl1;UoQTJsuI*BMgdT>kanj))s>FU4K7Ik^5e+i}d%kE1rCrGo#BkHy6^ zHsG{X7|*!R;S{URJn`ICaY{m4p=U@F+;|`%GllNS!; z*Bdu3s$ldy$I_fQ^!=d&_=U)Wj*L$eP~5@yaLc$B_~p|l@h|Td;hyWZpfN`qn-dTF zi{GW5MA`>c3!}FHXmUwzK@^d&x%VLzLCGCiJS1%;=6+5M2y&?b0a)! z-yqZ_{B;y~eKej_P$;Z=#^ET6UW!kyU*7Kaov%?(XFGA9Pdd64XvOy%j6$BEZHFJ@sGgAl4_!-zHimxr3nT2hmBLCO0I>YaoS(; z%GU?tfnR*(h#z&!UH+!{h6ss5)_I>7i$E$B!mmp);#(3buskrsTk8sH+ zCj|fIE6^{NtB(AiH={GQ%VVtJA~*(xLNI9;>X)&p=cl8yxWwA2YWRaHTBc+l{@Q2r{`nn)QC+{TS8BZ z_o6=%4!qa_^|@RJAC6lgTNvziS= zVM7a1$<6b4<+cIh;`B*qbtKD+MUkB*XDBm{7=)X+78~gyx zcl#N?W(YtRFSm;dTabvFX5Z`3_fi@ze#^`J+Y{69ygSpympXZwyYC!{pI@;IMWlpc z-H}F)2K`>ZTRR`Z$8WAeZA<)wxAiaL13L@Q(OuI}UH2k1b?!_d`ooFh`h6|L)q%0N z@#jyVfKIE#+%YZDSIPU(3DgcX*p!NL4y+OH@7J3{b+6<1hc>`XJ%dnm?>PK%b~hAJ z*c3NBvs|cuv6?d{=nDR3u^Tqea-d;n3iDrVX~oB$e+}6>zK`oVzs7N=rr@;a4v2jk z&%|q@_^40M=~0h+YlPYZr-+=xAWY;1bhZDpcyyP?oIN&pp=pPkq3)sng1yl#p~VFY zKELK2H2cM_LW8hrxX0sLqWlWBi!p2baHM52zByYfa`SwhpZC=ZQNvBn@AOaNRR`Y? zX0&dC&E34Q$8@dOM>|LOG2|VwU)~JIjG20KO-&Mdcj~iZ=Ku||BouGmT6YgBxoQ`4 z@2$iKzukz|KksJN&8_QPxB6T0*U^V?&({*fQJ(8{m#lwnQoYbMZFgGiVn5LL#5&T27u3ppm`-QfMmq($#W`>%W}dkEW9^ zSuIDW3CPJ6Qt3Hy@Czu`Y$IdKbuIAr1Afg{7L=_ z`8%cjs-fRi?3F3N6krN41(*U%0j2;`fGNNfU08@Y| zz!YE#{EHNzj&a1DO)C%c$C%`mX%Kb5lhW-L65%h#q>T5Q22qzg$uocs^~ad#y5bQ! z+#h42Ytta=Mo66hv?N7|_$MD#woQYmLme6Imxvf?{yTMb({PfDxTb86I)W>1nB<5Q z|L+EA|Cbu1_W!>~M3ya6fGNNfUgVK z|El$4^Z%>Xh0Xu3S{F9|ziM6B{Qs(TVe|jl{C_t8pUwYg^Z(iW|GzZ>G?Bq@=KnvzY`!(ZB}+#>U@;tYvW!a9*H$^XBA zKg}1&{{PZ3wlD>l0!#s>08@Y|z!YE#Fa?+bOaZ0AJ}53<=at!YVry0WyKb*i?R*e3i> z8&pbJo1?`d<+Iv?EU6jxQM9;!+8`}qtxERNvQkq1qB*5}kCofAWLs@^+g~(COIN+z z8odrdi8lK~m+_Ii4_#kVxvjMH%+%Nv>u5`Ya(VyWRjyEHrIhjw^cSy4g*ub_{X6|eN)x&Qnt-7|m3QniXD z`)h{EK2e z{VR_KNGGhxPY^6Sx3WF@_&@rpJWiUBlK=lD{wMwtznDD#?;z;~n9QO8O7gRXDZmt9 z3NQtj0!#s>08@Y|z!YE#Fa?+be@OuvkKc_8Q|9o~eQ%|mug>9DQ)xS$;{xQoXsTXf zm6kt8#_gqr9`T|_l9&RNs!mK1qB zU3$iz)KIq2s2&y4+&FutMV{X;B~s3VOSLAYrKd^fNU5s5q|Df~1o>Q=;=#pPVv|xV z<&1`lsBm1vg;voV$#BVhne7aV-AW^lsJX?Z#HPhpV&apn7P(MHm67KQNQ}~x zp!N_vE}tj8Nb&zx^B4J}q}{)o?`{}oILt?rwtpGGHkblT0j2;`fGNNfU|yXsJ@Tx+;o#0;Xt&73cnkx8F&wVZ@c zBZ2#fVN?6Cv0QUGcN$;NrUa$5*rbr2@=$rz*Ody%aeape@mzPA@s{KFRehpy-rf@> zdpYBT-G@pv>Bf%xIpdPu8u#H=W2dioYie`D<3fx&8laEkB9tswybaBQ>jVK$Wk`Sp9!97=D#D|7K8p#ir}{|LTW{62z@SmFbu(N9I%0 zh2A-T6#xIxw&*xBN1n}Ig{>;9`e<98ijC@(Y`KpA|7eTqk(!k(viSe1F2BDw)FZ}9 ziai|vpH90c*IrAT`!OcEHVvY~@zRq4%A5T1Q6(CnX%M~2q_r;*@*&fpa(lGtA7i3x z)1Zb*Ii&W#Kew4g{Vz*k8%zPF08@Y|z!YE#Fa?+bOaZ0&nFvt8BQ76^S2DcSRX-|YHWikz!YE#Fa?+bOaZ0A%a>hto;+%d%DN>u8P-+412t^3b{(0)WzfoQ9rXHrZZH*HQYR())2*n(`3u&FP2& zWr-SU#go%WB_v0rMYXIU7U(u<>$}l${YUQW==_Va<4SYCY+sH~pxP$(G}IHJQQ3{E z08@Y|z!YE#Fa?+b zOaZ3AA1FXu;u`Lcqxn^gD5$pGl~{yIzkqvr??G8gS=WXV@2@;oA+_=QCkd$W|4d~i z)^eRVAHy}?&Mz_yHq_ux7}gtl@+Zi10MZN`i3eyhTr^B3eFKrQUqFGO3E#%xW0=h1 z1C}YsHkblT0j2;`fGNNfU<&*>1=`93GSxUsYMdn@!J3vdBtAVY(VAq7q0z8N6sv^v zq@?sA6tHBHC|8t_LS(6lL((&(NL_L;tZrFJNyJXq*mx4GYm6y1J;9P9#lo_XXxC}+ zmLNNcQf*62&rG#s+MbL}NeOz^nr7|!Y*5cOK_fzm^-RkMtBn{V=UBqyE$t%PwvCNT z46`I8T0(;3f)gS`+eSuMBE#Ym;^W%24YP!XTF6-y1BRK}r)8z2j3rUGGBVRsGwf0^ z?3UDwlvq0{l4;DCg!K5<2^L#?rd2NX*s;VwYHaq9xKVZ~pjUWsFgf>?tk^WWb+m=V z?2^s3GG*B;nL~)l5!M7LH6|jmRd{5p;PSsgV+YYETlsKV83`n}grHcvDKt1VA}F{m z`R^MN);=_{eQ4X(ZNefV!Uvk@eB4JwJ&R4E(az-PXJakxP&)v*-c}Wxa4}yW{J&=PaYDV zWlv8`q&H&2Xys@fE+dxQ7_rigLZYo@2ieo@u_-~xR=XVDoY1_L&O5KcNr#mJWLPq3 ziKsa)zLvaUSA_m22HERrKgcQcT`4tkYpez#ggVm?bhU^yCNvTl4wmMm)kaq#AP3qYEwK~g>k5)2erB2p9oXwt|A(ym@s!rsLGc;~lDMl?FyX83b zY3%Pl+4_eia%o&d#*;687giF z$Wo%3X19>I`NVa+VwxOFUm0JfoEsNP&XE|JhX);6FUvx;M(*H;thtj&_#_&?bn0PB zk>}aVk@S^awq>y^D0rh-qdKURt3hkrjEl2FHR694Rr7qu}Dx(bX z%e!hC9sH-6scFjSekD$6p;t(Alb+F6!M^-#KgkWBEYy;>bX$^tC9zC(B~M8eB~4!; z0AOXB=#`#D{wt@-c}u5r^z%mA(sB*B9R3RbE`N$I)XV^#4;<_xM+UluWsf4vE*^?Ulf6`nX#hjwzW99Zp%h_fp zAnAc7LnA59;-M3OX}4l{buB?m6b~RRBhx&cKNBIW~9c3q~_g;>%Z~@ z!UQ!%?pu*?WsZOcM|&=$B+AG=26F#^n&MNbA3#mjNvU$-)$`Oo0gh`zD@9(lTr?Lz z#{Y{A1qK5@*l@=%ka+%w^4$#KhK+`ahEx1OG8bSnf5wo;cO^ao0fq*WX8>E60!#s> z08@Y|z!YE#Fa?+bOaZ0FSd33$f$i7UKw^QXYczM3L zdZsecFZZ$1G;gJKOZqD-<)L^Oc$Hf#^F*j{cDaWEwMP90Xcw+(UkwZ3QNjL0-ULd! zS|0hYK&iE-{U&N1em|n`rG_x5FskpRdh5%bLJyav;++2S(SE9>%BpSk0Qr8Rsg=k3 zX%DYjF2y^b008@Y|z!do3Q-HQX-8fAMo#w8# zR#`m2e<~iJ^>=X$Nn}Dff}z^Kp^o}{a~cv)uq-U0(zih(FwzbJCDvfGvYS*x;{Wg9 z@9;bLQvNuZ0YDEl;7ff!^&6~z$8Q8b*{_Qq$NL!y^bYTA{Uz^Q9{Y9k2=KV$UhKZs zJ;%Mjd$7Bg`)RiwZgbp{-KgUa>B~@D9pck%WMU4ns>vZQUhO463bJU*IK`Rd5gb>> zDUZT*$yK8nxa5pLPEo|AJ;f)&C8xOKxa5>M7A`qoU_I!=AUT)z6pt5|oZ_|Ul2fLG zx#Sf0B$u4rqsFLvQz&;;1qqApC#@A2I z6|PU;AaolyS7@49jB8GqC@#96FN9wG37L7d7* zZ)RF>O4t&y;dv*{o;VH#M1O@cKJ!H<4m84vbKiH2|Dc^91kG_IIYaU3dtsZH*;c$y7`ZHPoY!$ovw!dPK*hy^#?l`5ECd<(b8tA|gf zE=F#1JwLmV0w6lL^VfodL~kKUT^i?{jvq2k%O z=%n*58Z_^)I-G$O>W?@cP zeS9l+3o07k>l*( zx!Ah=H8iS5Eob1qdE&_?NKDDOk8TGCI%{t4C;By5Ee=5OIDcR~D%sov-#KCsE-kC! z2-p;Dc267?wP5s6T<7w>4rorCP;=@^p-t>h$k?Yy2v{=#HxZgROZuL|v-aLb3-J^* zZ`w(`sNDu3b9E=|UHvqU=)pM`AAeW$d)kQoH?0sxytNm7b68`3Id(o0->HTRXU)UW zi+s&*cAtz+eLqs@=Xn$LzPKMvip|C`hgajsw`Mq+8Ec`gp5yR=n3vJET^rDEKfQ$e z1@1$lv99^m2~$w=st|Oz>kq=}>S1_gVGXn}V0zSz{9<$~a~1yJl?5oP#S3_S_&ccZ zR1sd;-_yA*!QcE$=63Od`6t}(etXm>^PQ-EOWWd|&uz!23bx`pyIVQ;jlGO^#_G`0 zx4#yKex^4Mwr@vyoios@+b)RTXr|=%KE46%+^7?F=gtxW&wVIX|FVg5@Gc(B>|1C0 zi~d9KxBAw&W$z96L6>A@1Uh$o_UUg(DSHHFwWwS)8ZkeYbu3%7B7|B9r30fi;!@B4vHA|03SJZ7nQzTUoif_p?mrD@tM#M@rKrk=*H!csFiaz3YVJH zLj{Q%pYgI6l(rv#q7?>=L>RScrTkz9H+7{27JjHF73qEXBJ!*Tcq6 z9ngX&zek-D4hY^$?uk#_=A1)UWD5Pf@=@HbF6Qc<*YJ+7-$CygXQDG_pGLFR+R(ny z-=g4k?l|zh2F^~Y+i~gBV{lkRFluqC8a|xsiDxEF#b@_y#W7DmKws|ajK3_=I8SHo z7IWtn<85ybMuuOOBWLOzv}9Z-aY`RIbN0B`(c0{_;;-E|;hYEK>CYQISIG@j`gKD; z|1})_eWBrOWF{(E{%DSQ=tDDkL+DIilAIQsO6Q;}lc`PZL&j1s?LU~|9v0d@yiM!y z@W?iiVFOLm>5O7;*#p}alxVY;dGgBzBQGJwj+}8~(XoA;arzsR5{;){dWyxU zBa@YP7arz}6F%XLd589q2TaZ0z3Y&{IPp-4Cf(Sv(gUYj7$+Y(R@lNgZP#wjIA!<# z_8Jf0YR!Yf+6p9-ep&X6)Zx}7OMF~JcxZ^xp!&j2QHt@=vJNFI8bW&7KN$^y#Vn9- zE*7&umO=SxS~>i|e_+gl5E2hV{$y-r3MHHBXJac<7}=9zBeXJwldX!;5lB3dG~&8H zM2e6QF;-MlC22|mNYv5io}*kv|3UU2veS)l$SzR zBQva0$kc$}S!-3s528&7UoIolni@-{hz+-lBD2Jr%h$$V`FzgEB#|t#N#KvFKB%SG z0c0Vqhnn(+5?h1p-=+I47rz zRo{nASqq4@fLIHNwSZU)=s&v!gd9KY#nDMiJ7k{m!_A%ryli724ybq6vF-hdD5c3t$E10CP)^rSe2RWEN#7-O zsoc+2qpEKb87%7`X8p}XkFx&t@_uvH-(0mW|C9aAM2c@#uZjMtMlrVcO+!b~B;j1> zOJZQLQJl5?0UCC;FG|~$FZi}zEewt<5VqXan%AJZ&X5}C@~{7PMBH$DDej`p!S%n{ zj$@8~9JLWG75r~}EH0k00jI4Z{iydjoMP3PC!X6XPDyAh^bBc&d(Uqyv`(q#&^4Nj z54)Suq*`~8{V8{R^1^}qdgI1L6^x$eSeg@uzCUyTzYuxQk@0B)iaYooZW-4CzkK>6 z{^i{w+;iO)H0EeybK+rt@w?QMD0{~*`7^$pfonfFmA|Q$4d3}yXa4CxV|=n<2wIi! ziE!;&lF;0@2-ltP9-8x2b+ccio_I5_$2PB-IQ+v=__rO)@rm=jaNegsic#nn{K}GI zd?qUweU{wVJTB)Xu92LJH`f0E<@{(8jS;o+x=}vPQz_4htDcO<^L{MABT70(<)d$e zdCi{_KKO*F*aIVvK{@#G^%KzxpGS&GXP-h|IXS4#&I>rt<|E9-_wjT5S#e3jH}cms zaTC5;d{=Osh!IjOnH~cAEP5C^Jiac9suPEn9(o3M4B3U6 z&#Z}&sT;2T=23Bd@7IKFO%hQyIw?Hfr6n$@{j#9{>L$Lo%-?xFtOc4qJ`Y)SQE04v z2{LwVj0Y~9id#?o2$y_vLhx_C0{vpS>d60jGdg3tj3yMHbLbZN;nOWAI(EjFip$6JFpUZXd;kXsTm2RH+i+FdWYk3ym9`E72)wmV9 zu_?*iZDf5hsoHF0-SCZYpg05H=$ng%b$sCXwstN)+HfgOzgS)Tw)a9|#?-FDh+Va? z`1SzQzvX!J%&Jy6tJy#lHnb3x+&qt0ZW|yjPM?HEUdX{8J)e*FH*ic3db$(3+AT-? zb$}j?@@<2R{>#L>!4J@Ux1aHAh5&T&a=WOo1&OF>_Pq{$FQwt)x4g{1JuwZ>yE9#U zsgswv`_7U0`4!7hL`o>u9ckof(C-Djweul-{N^guw!}|(TmK?Hu(JRi-8Bu>buU6w z=gt(OKb$D8-`7H19T8{kcS8|{O>x6B%Z2(Et2uLmuHbJLyJ7Pz2O4&!F#pAtR($OF*O0B_ z`?#+2YaDlK3Ql|OfY_(;OuQzFkNWhS9`(4lMyNe-ipV((!bDy`SNlJUM|XM5*<*tj zns&Gu>K^JZ*c;svT3oQ;^K0HgvtR5gGzgo9dpy1+%CBI%7_-I?M_MN1o3phdH_zAk zd0)K{HQeOro z4$vS=Lh;tEb@!l>t9CK>-b#G%+l^@b^KNF{+`7(ntG^Y09eoJ*d@Vs7^?3=}c?jXf zMVnBH=Urh|TMb%V{ZsM8$vJ4Io5aq!NEp#`1|Bi}Y3%JN z5_T=vD0JEXfe_nggJVG*Cz@S)5Z8LV0q*!@OO(<-7jH8@gJu=JiN<|rMR`{{n`?;m z@p}bERQrV#q^mmxUA;d+Ts5%-y?Z%7YVo#Xj+K4;2}Rp;@f!v6qV^m)Dx7#WPuO3$ zPT*+2fb;`O%N3J8>*HjO|1=WwKaw}{r6dx-B149u8=32WM_;U8tIyH**9Ysp^r!uH z_|5T4_KWtb?I*SWm-}Y>_VEq$<$RC(Z1$PvGt?*2$LLe)UEsaQJHxx1cYyaDuVSyY zUO8U>2G@+w+=7p~rHMY>z%3fgYU4are#c)7*!;N4gu`OWg|G z7P)1(b#n`FyQ3@Ct<~k|`s;#qUb@rT9ojkCWNoyzw)UE)P_tZeD3BWh3FLYJx$Z#j86ejU$UP0@x&pbUfLs?K z*BQul0&*RJTr`j~133ibB#$PtxcvTz0&*RI9Che%ov$+9|ETgkL9-*T8WJhCkof;s z^!_h9$TpY)OaZ0U{x(C0WhPe3~P zziIs6t9<i;P76SxE9eg<+s0lC{i?iP@{3FK}7x$8jg8j!mR}|hxy3;4%|GU7zk$SPxdvRR;lIWIpVr#-Ygzk$5oKNW6TAWB z76Q2iKyE&en+N3bfZSXlHwVbg26D52+)N;s3*=@1x#>Xebs#ql$V~-uQ-IuSKyEUS zn*`)00=Wr5E(ges2XfdIe#Fd%0Ia>+n03CJY^ISY_W0CMp_E)K}W0=c0;ZU~SY4CDp@ zxtD=l43HZL}AlDDby$Iy`0=XA}-19*0IUv^u$n^$ty@1@aK&~f{>jC7t z1G#5_TsI*1G?42Ggjw-2ZNTe?IU(e*S-1wOnkJwg2x>&}IAoXCU_zkh=}!ZUMQQK<);RyAI^8 z0lBL{?h25*4CF2WxgUYtMId(p$dv-Q^FZzgAoo3xI|t;>0=Y9l?mHlN8pwSMTJ+%X_`6v%xE7koz3S6$7~rx&1)y6Ck$_$n6Dkdw|?-Ah!$16#zLWki$Su1ab}_mk;DV268)r+zuf3 z5s=#sItF@dey=b}a++cm8;flVO;R{1|(gP4~n57RV zy#buz3;qlJC4<9&^{Wjd^cVTj{2apwLw$n>|E_+pK8~-?=dqrGG9}ptQ-CSJ6krN4 z1(*U%0j2;`fGP05r~qwa`EfdHn%$DZC1u8@B}mOOn(oVKRD z*O${t)~T%ux#~F$Ezj?c5Nk@#YQ<>2ZX{nWT(VC0NsZ!@t(LS3*(yi8Ic-)NM-P_s zKAc?742#`rFHa>ZQc$iz#dJ+vN^DxZtge(?maCSgQ&}n;@u4*>%T~QgI-27{_Q_Qm zZb`Iea*{e_)=Ayz`81CT={ib3RC+&7OH&@=y*V9Gpe#{C%i_stq}<4nN;y&s50%@S z(-5vKyK-e|ITRaD%9f-T4Of@jLE``KU{3%j(}Zm>1(*U%0j2;`fGNNfU^`jNj`+(qv% zbQ^V(bua6}b$+^YT8DPNHbvV-TVH!i^Qq=t&8wQenpPS&%}MSVz1V%NJ1vIP zn-JFXtH@XJop5PSnatsmQznYH`pLP%_30agZsX<(O*4ye%?T65MfdZC(5pWo^N|&T&GZ(|OMD$Q z+t(3g44xrAwl5n`ym<jOTCVpo^K^qfAts&yT>8p^-|&cH~f(0%cY`svtkrb zC){}_c_+HH+D#}O^tkiQObbp4TOu|*@5I>?$Dx4euTaKkzUah(MmTZq`;PG+v=fA& zIgTV}C_a5J47CY;MZA5aX@_frW@6ptq3GAP_wkS)CZHQR^U$f=ZfM)oJR#;-FR}Ki zY@GJ?Jv`t>+SuLxhAG@1ubv!SpJk*H_5i0(`nD{Kz2 z;D)XT;A6K&509gPF*RqiTw!~`xFTQYbM|( zLKA06-&1(j-rHy)o`U90JBb&y+aP4F?u5OopT-eAIOpQy?}~m;8?pbU6~c(O_M&eN zYs@dl&PU=q)o|ggc{qBJulddHlhLW~M+*HsZ=&88_oGR%**NC#Y8?623`a9#E!5R> z96k{9GTOFl1N!Z!mvFzpeMmIcHNQGx3QArTf)026L0DZq46iJ#fffc#kGhdxjBaJF z!XLb{0A;m!0k02#2Nj+w!YliGI=3bGo1e+tE?zMIg!|oZkNRZ36V-2NTfFnR?f6u| zR$OOyE9btkm(k8x9a{SK*TT@x^yb0#?I^Ev26}bd1@Rlrl>FYuH=vyxb;9o4Swi5s z55?+VHgOK##iN;h>r8*qe+d3o-x{~{{Ux= zxYT}=`+o62$~s~Hi5xMbKj%Dp>MUAWzowaQuteDX9q-IrzZczD+ej?z`4L`pZTGZu?DE~gcZW({5(|Wv+L;BK_xi8_S4AO^DX>-Odu-my#?t?H%7(l zz7vEEKIVYW*Na5~hK5_SGD3VXbki1zhdig#q0@v1pR__BWh%H8{GhYmU8@zLTU{Qm7~=6V-m z(d_S5py{*TLQD4YV#7c)Uf8rF-qd3e63)*-5yKwfBd6}7(wFNC#veFzFTXxM6Z#?E z&^i&_xEvC-a?VEKQj?nKfS4!LySfk$zq!SsbVQ zidyct+we+1`LYDyxk8#yi?!02W?d`$H@XK=KOud7ajO!#$>EmY39``z0o4r>2wfiQV z^I$x3{IC~CCoS!edBzVnduA*WbG*$vr%uK9_t!Ts7`|O-7I_JIoo|G+IhW9(7te?- zoh$LOje$6z-d)GG_a~y1CMzA2=Iuc_T|+Tf`CEZ}F?_RnP4rJSim|HVP{7toN_|C67^G^pF4k-w&i zoAA}*yMp6HjM$=cBRp&0Ak-!Nbrg7gG@evYD6D$M;V6n;ichXz-tP9DuTf8DJ8_>+ zI=U5T#rGSGLY~`e2$R0}0G}&5hI3LUx1V2UqR{uiYBZgETCF~^SzH^UMMvCv3TLNm z6RvL@j(>KqhCb}G3jZ*Bpm3@B+i2zfS9~_!3U)+ooQV>aY!LSNN8n>gHPKVwx9afH zgaZ7-#;H*y*TT^_?XP&{>jUw?FTQfb54?iU^bpWz(ZkT;@pVyDojA1g&@;GW$S%}; zW=)Jt-Ej3ckBaMizb0&Jl8Ca=N#XG>EpbWhmj(S-H}Sn?{?7AZEzs=odC01ZLSyYq zkg;oHJaFMu+hHr!e#TocU-&{1T;{(UHwR7>&hD&k!#p>d>y%!2IrgjxZ?5c&u zw+Ep9Eyts0R<*)e%?6^dp@pdA=6Sqw+W>KK`Xn^+LJt1u`Fy;;fn$2m)1A=OZaLzw z1N3N=ZyRLvUnbrSet_n?{fu8T1fYwT+eL*fNJLGu?{(;VDGe9DyyC4R!&`WNwmodxLVu4$;Qdl8yC zccu{i;Y4x$z82!@z*yY)^CwV1r&VI^n3m|P&>CM*YW#9 z8{np%K`6R+9R4`F8;U4wiW{C;F4VtR&6yK)1%I>H4Vz~<(6BRw`7gG#;$zRhhHM?* z$90`woELhXT5M9yIlCh`Kh+W%QRy31qE9vi&S zw8PC%_fUVq-sqOl;(`UAU-J%{{bE<4LD)3hoF?og@4h@{ZUqZ-!&WOuf0LCJDVe^;xlV zfCgC-inng9y9bqAwTrp;R^o%-Zba*!cQfne)^)C1{jK=x=tH>YYYF10&r8tGLkKS} z+JsU(?+UZpYS7~9pNc0=&Oz(m6NUIgY2v)k-$ze`-i{i+&K(6U)*umKp=F@Iv*o*f z&fk8XfNv)+7k)0Sg}N_~!oCB}iv!vWbqwp+&YTyF9V@@sBzDe4!ib(T@QCS8V{b>1 zuxr6aq09acgxEeC91H3=(d^QLxYpwhaK|TGqLlu*c$@JVG^_AUH10bq%DdXxTtlpn z-zzYp+ApLaUEL|@>ir4gs);4&-OKq=i?m;y`zrT|lbDZmu?-&3G$3?`D+{ZW0^Wn5bz z*9OQ%06Ez&8PNA02DBFn&30=YUst~QXPAB3*&dnNFn%laz*0gvL8k10|(Er+;H-iP^4-iv&$`32}xyn7jX z@(uV>-%tGp>)-Jk!B6(<;>YoRh626AJ6nIrJD10P-8=$3s0Tofdw=&}cQ5zTZaduO zxFx%leWJ5bdZKO>qyB7^zG_|AC_NjcXQT9Nl%9>!vr+p0fKht-^G44lq{nBaTGH&c zAZvoHD!)Y=ZHiXs60Me$1Y1yo)n?DM#%0;9>1kt3u}KzF`_NXV1dA;`)0!bAn%al7 zGFcPk_FTJpIyB zEJht?oUyy`FlU_b31`eZv~NFWoV|P3A%k(^p%P8Hu_M`EvRmUm+-mIf_3jqN$%l>= zwlGfHwVN|e*}cEL#>2N-^PsS{0?DRdmOUeNxHZWV9~TiG8e%jkKM=VoNUQk)x@?@MB%9wBsalc zw@MFiC}ofURlAA*?HUPH0wGkTw;TY=GVAq3S#|AQcGs3oFrr+5J15i=;=qv`w+cl$ zRDx4`y6J4y zopBkg$71X!s#mzEdg5BM(Qtb^717@jrDCtxZw7xN@X=tF^B@_J*dgv9t@$4rT(AIz`($rbbHfOKpU#XBHa7I$Q`EshXTfh zGH3HqBH}KX9Q%D4+pab3YSS(*emk>HOJVyw3qsEgYe3SBHY;9?Ruj$+}Y7%;f3p%`W%Z~=ss7@3kNW9(lB)cXV*^? z95t4&-9^`#OvVaCBGxs+OrGAA6quN}UJ5hDV1O=_3|&bXPBW&DE7mZ-nmI>iP2sb9 zKkxryEEq-h-zILP<1Vmp2;*xSg^o1f1Y)?N?OtNC@0>pH$cHe2lPM7Urvt4805!(H zgjp{f!OIk2q}vsb`!&_DEPMP`llgcPCiRdhUmr28vLp>7Rnz2XQ$K}e;M~e6C}hxC zmEne(eY3goP_Hm~v%Xf|%PUX8KfWb!P^lVxG<3()!iCeIOTi?zF>v0GzN><`>{>f91jzmHz? z7jb=#Vi*+@m_)}d)Hs~sVub&X*_0amrx^I04%Kts<25I^!Jn4-PAOr{jLVyq;1PB2_fDj-A2mwNX z5Lh6fNzHJ61k!~d_Soj?!aJHgE#p(6;#2X{Z3ys=rc1-ARmKlT@M?2?T~2{R9(c96 zzVeQyfCF(mqmF;z1WlwW)sIz}ck~Gsob1RPDrwoCK1ynHedQf=dxB|CRF1+RsQ=jn&I9jnZ3wukegM?B45i{>FRXu1)`FqT1VcFo_9CWV z!7IuhfDgU?LD_;iI|Awds5L*A?Vp!pHryzUKIfqr{su~oD@AWvf?;21&PH}l12Bw^ zSFj#0}*ZPxq{D!wbv!BKu?~a9+hB0tk zYsiiM&*-r>uK{6pUZG{SY<Qp90m<^MBQerdtXSD{iJBOn3j^tdN@8byf{6(FjW zRlB03kpK5CVh%AwUQa0)zk|KnM^5guvND;JT`EcK-a`IL8}*c%zH2 ZpRY5viA~gOe)yN#_}M&fKdb%&e*$0Q%=Q2P delta 2441 zcmcImZEO@p7@pZH$M$Y^-@Wp6mP(g$w57Ip-MjYgdSJY&5rXD=MiLPNRRY9TXlbl) z6bMH`j6o!YF&!nb<(Hx%0lR@Cg0?CeKfn@=4Izq6O$-u#GzR%qz}dZ7QVi;{gWIs%g;3F+Cqq6B<;PrMHFJ;F4tupUuCTVJh~)}leSCW} zP|{6~_@!Q;NBb7{*AiVTLZdJW7u1v*h95n%NUkJ{yMP`K*3gftC%X&Ky60QkC=U7k z(-Gpn5gNFB${MF161M+2qM1bhO{RY=okRU^9iW5rUSfyaT(UX8s$9Y)=E~~FFb>f7 zY96PVkV>WCqj)-97#zhCtqP@Rrg?s0p$KytJrSx)PZKlw7K@OC5=ixa3n}=}d(rDx zUQhyXMeXrclC|Cqkn*bDchq6Blblo^CdKM5=v7c%686(OS>%|p#E?d_k&6bR67Cetj@FuqXJTJ?ZADM8#-DLx33_p5#Lx;Ri$cVNy}sbu>D4@_IDO^% zA{?PZhaO}kiAl~sf)5nl41ioB4&_=d;Tji>EuV{hR+-sY6V~<)5|fNw2CU?85?=?O zD!#UA%{iux=6p#!#EB)x@5BI@aHJT?Aj28h59=Wbd3a1+|KXn5@rkd}zE(zFaeV>p z=&NYd#j4}f<|JQp-htcRN2owp-L^7v@^Vq5Z^nJjDQ8y^U%O~yU#a1&5@%Id=TQ}j zxx-q`#QNpC%J9H)!GCjM(7-oQe~wLkUSoUsZm|{|3sAr3R_gbA?o9rFr2fX-zO}A3 zH@I)a6LpbbxFH&g#o~2&;<{|^>rTd27!x6`6c8WLr3Yv{mF}N^(fZc3nlVs3Ddzy9i;SF@G zU31plm=9$rzAChQw4i}2p7*>zq+%f zlP8dW8k3MS=PT^pA$&i}mP%{b?g6Yh zwm9|FN~GlP@G&d((|1LQiLUa!GRZQfQk#uX%t0t1s%0S*L}xidt2igi;T)vllDdPV zR0M<20|_{*UQ@rw%|3&VE9~M=cvx`RVZ+O97BiS0l4en<=~dQs$Rb{ob~G({w5X*% z!?yZ(NP-M|^2#oGfAwa_3Ep~py(@U@EmzL**1Okky+VCDJQy=BGXbpG4bZIGe9Fc9 zeOdC@6#{&%rbn~IWfjCYi!P|5#n?^Y^bDdv7n U$Nxfkwhu|$e76@Q|D6QsPgkof6951J diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/indexes/381abe91-f939-4b91-92f2-01a24c2e8e3d/lock.mdb index c99608b77f76ea52301126c16a6f78adc622d786..5fa5e6b49e4f2cba646080a3185fb08f08ff8957 100644 GIT binary patch delta 115 zcmZo@WNBz*5x;lf{XV7zof;wx5WohcHY77nwAY;|qcJf-WMYE=D?>q40pr9#Z7_R- ZeIcA(5R#9T{b4FXe8V;{dlOT`KLC!+BKQCR delta 104 zcmZo@WNBz*5xjTc{XV7yeH}pt2;iG&X*|)zhwXscXNUXM69cs-F$uChyDtrAe~Wj9 Yv$x%Mg0rv5BgDV`w*rfAVruvY0CVam9smFU diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/data.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/data.mdb index 226be2332b67c1f376e37de03c5ccebfaf80bf6b..f2bcb1b8b36f5e17f93dd60d4304fc39907f1138 100644 GIT binary patch literal 225280 zcmeHwdypK(nP+JvjYc!m-ICDzftyAG0qUmSj|LWjWEmD=$uMwU8*Zk$t6FN*(>?Bo zKuBY>-tY!qW3NRJUVJ9bEJ2)E(4JWZdjSp%!fPv7PV40lhiy3QP=N8W=3%YbJ;UPg z-Itk_T~#xc{n7|M$P!FfRc2*X{qj-y<@bHRPY~!ffOmdz_&3kYqNnKZcGy`U2m^wj z;_s16v%S1=dJX>dbNPED!#Q5Qg!ij^zyHgx{Am2~_R2^{!lDBZO&tVbKnUiU{1d-`}W~?pZdq4=(N|i z&bj|5gVAZnE;;a}ugs55``Mj``foxS@re7qaN)i4N9IMR-S@*oZ~j?-blP2?Zp6Ml zH#)6;vz1(e?iF!AC4b|dN0FX9B2C;kd*D+@hL1?QX!AFP8q&K*q{aXCHwO+vl0fzE zg3u@E!YhLt5)+B{h6+Ph58pHV_|T7re-VELB>h8)MX}`2D}%|n6Tdn3AXMPN_yh5K z;``zySQXeBzdXJnZ1|6g<{L+VBft^h2yg^A0vrL307u|MML>kR^$9~lvFx~aq-%A% zUL*JSjZyu*a0(J9@J7mrd1Z{-6BEVq4wB?w>+gkAkl=wgQp!RhXV>#P(}iNWSlj73 zX}eC6{p3T)@DsV^&o$geM+KehJxEh=7%qum zUAJqy+G0&h(7Y18XBzb0Z5I?pQhZSSJ(5D-M)ryz&ob9CoOesEGn(5&5_}l_y>LpJ9Qx1E zL%mXyZtq>@2;I~uT6aJjd53<4WCwLg+XIro(MGOb>a~Y@C5;ujv?18gL_zZr9md$v4p3!bxdE3b)7b$;k~VJjSDo>J>oclerNz4&nTy z{h*{;BcFF&$Avpda7lg7gp2Rvc;#Q^IK_IgQr=d*X~MDVF3E=PcT>^9S$DaGV)gs} zldq;vZf;27h5;E<*>a;)+K{3h_^Ns;o03IYON$2lHzFxnRnAIUMweC1GCz~TPme}( zl#^UbhamrqTdh{AG@176tM|||0oW*9(_hDjbdwdP)SVHa zw2-JfVc14-wWcs?aNPv7TD!VubeG$F?s2!ivjT;s2;}tHYmhr%Dc7OaM+?OgR7FZs ztek4=M%pqIJ*{eD9=f8XrVUM$TwPOC)fM3?`b@Rae5FwaK%vP%m#f@9N}ecRg$@Ii zoXXa#4R>;K@*Tf6SJB!ug0(5@t?N?Eic&_^MN^tml~XI-%P1Kt&Ka!Z$gG+s>M|)TNRPlbd>AvR2i*BiSqBZYIQeIMZHEqjsF0I;)*$7&}ms~gqvXN0N zQPkAd-Ja%@V1#N@`6R0j4NC5^-{0ug>mZ)g=!XkFcHS8@PP^kd*Kypc_qE%xqcTcQ z;qPLlQO%{R0&`@UUA$KoUOqi?7}=UU9Z~ZTA@-MhmtNu)0pc)oY{JP zdaIt^EyD#>cXzRdht`y<=3UD)3_Dj)T&LhlVor2S8QQ;VDmf>ggF)PtWpo=lV2`-t z&=}BHH??;!oXj|GEnh88puu`_G6f@b+fj+WvyVd5SE%gXK1y?*Md-lHs5S~U=8~^l z!7kO@HlMyp#UBBe(-9EbeV`X8k-b1l%gTr(Wfe2485t36mSh;w9brmlN!Bb$ky)R> zskwDSF)Xdi?y#xk+SRK{(0XB#f*=7Ah_*B61Y+Bkgxk&LCYXE4tkav$kdGQPN8vpm zzhx9ZKrDct+!{YUE6W*80q`t~F^)6=A$c{)5_MC7qw=k8|O1|0hM^Gw@-eWUA{wgJZ`_cLt+elf_j4Y=#N!SyWL z0BzUNKVsSj+_Y|G)Rp zHGj#{|F7Tn^e{{Rf8=v(-)8Cm|3v*iOPKoq-#vU8!g3m(tFnp{f5&F&|0nwv=UDpx zkN>wL-#|lCM49pb`A!@)C4XV@6v_YS8Td98fd~9AO7M#u5bx>C?F^sHo!F8MoxjnU z9$L@A-|Wyr4O+k(~c8+=_!+vY41$t zC9HRD&J4($t=DPTWWsNuOX-{kW;ua3KV8@Dn9ZRH&52F1CQE@e%_S5YSbypPn@-zG z0TxS7ctNuhU_=#GI%M4tVQGRP#s*--xeddxK9icP4*M-;F-1i@tx@T)Cuvzt$>@qC zf@Ev4R(6`GkldOjDu!+%V1Db5A?J9wNLC1!>0^jw!Ib_~k0TFath!_&ED~q|O?%&PNw`l2HJi94;dk)&@F55sWJ4k!-bhKk=>Mtf zF)3l1yaT=gZ+>Oy7)$!4 zi?=;dWXk_fJ!`V${~xaX>H94CztVr%GN$~0?YbgU{(tn{MNIks-EE&*0JSy^>($QM zWt;wWAw&LO^US{uS6TA^;B9->GUfl5U*5=)|G$4;Ji(Ixub1Pmv*iEu?0k(S|L-dv z>SN0P&t3fnQ~v+ueK)h@|IehqGs2SpZ;>wEh_IZ7CzAiaaP$aE{{Q?!>kBOT|JpBa zzXWxK_=p5;8hbzlEkL5BH>_toxmMM!m)o0f+N1sKWI!u(QZ3EVY6_LP zv(R1XvKLw{Iak>1r6QT#PhEk+>%le)r104wug?X!d>+Kxn=cH4|I)A!6XHSw@+<`B zj>W<t`_L?pfSPEfdl$M8d}F7TOS;~7k@9F z#4CH8|BvFre>nmi0geDifFr;W;0SO8I077jQ;onhouT#G|6~5YNd3RZ|Hsh(WBxye{vY%IG4%hK|1VPi@A3aJ^#7RukEQ=_$NzU2SM&nG zdFf->*VlgHsb?>IWBFhG{8?*ns@wp7vg&B15cH`*W{!5<&e=70v{t+m+_=EI+0eln zs8K`SH{mo1jOGk5p(=oXC?pnZ^=dKKAf8L$Y~y)oV_#jk*pBd^_7($VNynyV4eYVq zA}cH|Sr%F$Iqtoex$>CJ^OlLZ+?ZXP0G3GQmqK^1Vb|wZMlo z`n~KO2o_{b)%0uw8!<3vgZG8gWqTab(ZXA!>g~fCOaus0TGB@(J*#S2NzaI~sDlxr zhTX4P;NZEIs-|w@l_HD-`qL*Qg&Tx-Vq1pq9(-ZoaQ{1V9-VzdGqv-bt(XG=_raj` zYrq={S_3sW3V6j&v5=HCLoyW+EF|#))D+Y0DVcjUzJv!M4)uFe<6!oOYbG_a(@lCl z>PhfXPZrAsw~AaJlf=w6Y1fi<$m5p5iqK6uMJHLV)RV9}=Opd&9@v4_45DY(nkGUINsRIGt7U4CaKUm3p?V^%xAqg$b- z>ls3xM4E!IE1zc#eTw|l^QvpvdcjG<$S0>&MFBTuS=H0J>EzwKT>u+lR0B;%#7-Fs z41Y~GXg}@uD`Yhs-LHbSx4JCL;9AV~(hO$q)vUMKuLzGt%NEeaOOu#7d#lixf+C8E zV2TQY<*9kZ$1rGX9l#K5k6U0!>}qw%lw`@$@uEB|j!z$(!gjSi8R2#VA+ovM6f_%# z;f$C;k0PU3Si6VM39W>{DQcmM$Td2&gl<<6AtPMGudz3U2dfUOtcm$EJnUyRbAxJb z5aFMM|DrwePtxfQq~8k8w}0?!G-yBGWsPcWkL~}&Oyo0m^oLK`e22rudoGU^* z*K9X5H#k06-IBSq@7w|Fd)m1J=J%&>=k_`^>c%?)wYPwJ9E4a{py#)#J=M&pvaVU6 zLT=ZOjV#Knh(H(w)E#Y{R#Y#yBx<@r7z;Y5;`vC_p72e-?gkNQ9pv4~Kf&CFL*%9|oM3+m~D2%JZ%W;*GdV~goL zFzGmXkdigghN`Cz<#V2O@YO{A4`k!<|0oit{+IjzqxCOL`H88i7?h3N|DQOQ;vt7} z9<(;2f{{GP*~I<-{YB*_G~EB+VAS|v;RJi{?_y5P{vHrqjaJ`KT z(Vfr?bN~MnGBpkfOK1)U#<2c0vBdVo#)OziCKe=w#PRsC z_|f>&@nHNv+#&fNjsQo1Bft^h2yg^A0vrL307rl$z!5m35jc#Wp47qee3oM`D`l?f|Q!bVKt0>cr{<@&E6eJAm!@X)6Ea2yg^A0%sorJpLaFj~xz; zMFVC)$X(EP`+Cjp?Z$chKN9P;=NKd1n5*8{D*j&}UhTP_?WY3PtGH5Xm+|_fEBIdg6XDiMA~k) z3EFqDZ*jfGt4$gFzx>*iqdl3LG+Hu*ZUDA5)nK_gOwHLfFIzV)x1AGMTTAKA9KeQr z-_#&GUI4XdACUkStccB1VhYjN&Rui2HBk$?q+W}L=;Rn4Ovkdxt8MTv^`dzup}`f3 zI~r9SF#>s+BHvVM3rze^i%vEDX-EO{Ksn#-1B^VGC zZUS*W!R-^gPxCv$&omGIVYQ@PPr@lTDZyP!jq!3)QlRlRT~CQKsaN7C_A#9)SJQ~1 z&nVZ@PQHUAj1{$I{}6*;r#^bouRHPo|KE7ByFi#KGy|2>kpBQcP8-qZijNxT92{>S6@#qWsU65kkK z8y|?j6?-xEMC|_9zSvl7b8LNVICgya$9_R)pNPQUh$Fxe;0SO8I0762jsQpCj6k3V z`+sU+PE>R7mZXd>#9nE#K#@rU{U zB02uHeSrD@7#x2)8t2AdVsQLn{yzrCALjpyeH}>_Ni|8M2nM?U!pR{Xz#-~anVtoVPA{mp?V zSn>Z3zx&ibvf}@3opb+BqT>He;1)9=eAa6@GzcYa*0fR5mUW4j?;`J)=;H<9Lhm@` z{hsIj(r$v<0NPN!p$!|?!*3ED0K@;S_*cOE{b-k11_+>wCmZGGzXl%;2)B6W2fW|4 z-tYO|Z_@ic*ZW2OFleLUI_Y~-zkhm+Bt)q$i_#%lJ?X^>j#|5H?B zh<(+=>OM@eQfXB4h!l2bV+Ro1?$~0((T3DG@XD5u5r~WQ+Icq(dckt7P^pfCLhw>x ztG814)o}2`Q1#CVwOr&YbR92WU87OU#W%u?`niIYL zS%MCX#Yb500O=ed1>< z|Nl3?GIWf=|Nq)^Lx%@gYXe{WLt~7!HejAV`!H*5;6J{Vev!2{uzSt+2U%+aU%c&! zB5Q3xd+J$}wKnj>wLg8IwKh=czib(6ZD8cubw$S7zy*&z`tBm8{QvH@PqF0xWt;wW zAw&LO^US{uS6TA^;B9+wF!^b9yXvwz2VZ`9BTN4O{(12POa8xJj=vry|1V#aJ(ixG zuOT^j`u(y$+gCi)hh*%Cv=`Ptcl8@cE{{mN@5nFjySYC)?XJ(HzcYds@*=LQ-y&VS z5uJ}nQ=BgxJ%S9EBGSaqFSNdZCNU9d7hU`1?U$gTDIzWY_xJquWsuZd7x4A}FD6XT z|9@EWKY9kfaRfL590861M}Q;15#R`L1ULd5fmR4ilWDy8e=HgQd*ic?vt<1J4=-H6 zlJVE=`rUDsjQ^{nKbm04_+0-VA$P8JP8I|E$MWy{~s2x=|!5@{E~X46)On-9!bHJ z5U@)vylJcH!b^jKOO3ugOa=wd9;^dcX%W<3oIXVlHG@?Bc%x-dBbm|M9+Kcg;O~V~ z(&Ui(zt9*OIJ#gs=X5}+D5{6cm?NN=v+XXXaxJnuKpT07euN~D5s`ELL;HIqg7 zDrIC5A|GoaG5wrwBswLtq-lnML)g+WC}eae=_8V!^$M01Q8FZJMh!)QxTP{uDY;(yN-_%JK>uvd3VR>^GpSDER1nxR;wy{mcO6Ll?F6AU$lh)?d*QmR==h{2mlh|up+_HzR8}N7R^J)#u+WmPD%f`J~YcLOzEQlH| zTT+`ZB-8_%G)akL&2l=N4*VLYC+D6s-3nKMM{I0 zvzxX|OH131uDha>GZotoxY2d|EvD-%Nd8|g+>p2-@krv&6R&j5|9AGU1MoNG2yg^A z0vrL307rl$z!BgGa0JdY1c-)sLG!VfZ~K7z{{tG+a9Qne|NkglR%KiI|J(KTxNQ+U z{n&m(En@qf3(Wgyn}e#oz`5WdFc170<_m*hyfiGtz>|RZ z5X^zI!E0vYCrQBHg~B;aonf*Vd?`~QJ zlXvRg{+|{^(?UmFE|9{ZSx(}ouY{$^0{Xaq$ zZM4-;{d5PIP;jXeck@j=g?JZ5P`_{b6kXH|wG`*}|4nasf?}`s|Fl}E&pxe_F6{ql z5wvc+meGU#KP3k6N|-78|7K$W{r@M02Tq9p{{c!?Ksplt%MsuRa0EC490861M}Q+R zg9u#g`%tW**7#P?fLk^rt#|1PBgCiI#0=QW8y3TSdumQiB@Q=KE((~1Psys8;6>Bj zvSk~peyQ!E50*POyCt`d;~s=#K9Kci@Qxb7{<&BNj@H@8(cYYa7i2|jniB5=@WnYb zl{h=JMh#^(HATTy;h8#v+ofBpArFyk>RJ?{B{_yQC}Y{=)wYwQ>s85orCccPXjENd<9gn*|d3&+GGg)-9$uW5>liXBkl$>O_Qcq6U)tZ|e%Z%=J^Yu!#mI((0 zftx^_k75bDq?7rbcGb>HW zBd6+!v^Rf!_}h;lfh!^n$Nyt+1LF993~oRi|Bt~9h~xi7as#f}hU5P+xB+qeKL-Cl zj{nEt|HtwFBKiMUJ2?Iyga7{`6#oy&dDC!N?R;e8{~`YWOAdVLD=hy1I}i2Wg!Jaq zFa!Pnh4;=MVe$X}@X(uo#^V3~bR+g{w2&83X8mR>xdfe$NK^7R?s*gqToGyFzS#qx zV)6fP{-#hvpNWX;;(z;_0|z0g8JCdQ|9=Kz|9vKLIPq$tZ|=ZUE(4UEZyW)R07u{~ zLV(BrhDB826eZp%S<%cG1~8mVb@wFK28A52v}(0d z?Z|dvW^9(wT|=@WeI1ZD5d9Qlv!Sw4R9rJGM$V+Ynr&+Z*T{+Kyl#uYp-@oL;HPe< z3x;S|j;m;*ZosqAXX@B*lgs~(>jWy5%l~Je{9hTr6t9KU&W~JL0Ir*s)=)DuqJ*PF z7GO;z)b41<{ZIyzlV$vx`shW!K8|Ow3j}lJGY8(ct~+q=Z)ZKUHS^Ug_NK~>@tj*7 ztrSN8cL#ZB{R5o~5TH2rh7^jXh-B_9-Y=5QeT)5-O+Nr+wT~?Rr`qku1!|kWSXc%& z{40c&!YW}6Pzj>-2rA8^646FI2f}^<#B*Q?q^|ZX{O90R0RK4z{2Yq}w4Ow^KDbZc z+-qW+hvp3ap&!Zr*;%*s&q=%yKN^2Leqa2K_$~2`@wM@R_*=0TV;^f1s+h8BRa#ndrFve@8MjxBo}- zea)@cL2p?jzGUc3lvH1KZy%*OQ68#K#cZfyWhgGR(x@Ub+|q{cz@ zS%R}Qpka1_EjJBX({im)sg8rX^ipIfdVR56+?5)G1fZyOwDf zcCMhfPQjJLoamT939GrLl5_Go%TQcd4pJDW&sLUeaAytdpf+hwFGf?jZJuz?#;j zv<-ymF-C0|N&iMa#_||9%uk}kI3T{suPzKwdhU)}$ z3(!bKGzKCPS2?3fs-o(^`#EJ0B$o#VI`cz(0kwtL=&u;8&q?H=>J1R-$K7X;>0QDAu4T&#iiXuTvRfkg^(>E<-}Cx zks}VR9+cUVB~#Lj79$SL4Hj*ZVXC^WN?n=)(y=H?sI0nqx43(Xm}TCV%tkHz}6}Pr?H+ch_0PhHPcDw9MBBqVczHDB@+}v(LA2yEn!Ak zxqXy8R=x^BKX6;FR~s%giG<%GVzfns<{?rmW^JJKg;*5|Q16OBmMs(7<){{c2%Vdm zEk#i+8P6<&-Lp9l31(J+MqJZO6;E+GCvQwlhjaVU^)OAbu#U2(Kx)KRbcuRMX5`^gVv3s08$!~1&IV#ts|+e zezRQy3smGlzc+O;jJ|ehVkhcz#MCxlX_V_ok~&FE;Y1#(hl>C2e!UOTdJhwQv*x8< zRZ3Y2rkX(WECK01^D7a`EXjtZ>*SNts@_S_z#8Q5N~SKD}}=K(DO}E z1ZnT&^zF!d^-xp4_iJj9)>OzoLu$pxZDd6;D_a?`yoWEes82P}m@*)f%(5s+rYP(9 z3*D-waM3alcN-9M0Z-0ms-}Y7Vcfoh^v&&V75CIs*0P||CTukS@~3Tgj8q`z)iuRB z)U2d^C_S~;N9&;-{6D`P%=g+s$gu+s=l4P1pC(k`FleB3BF#5D>1*CtUoDM!iPuD|Jn7rAa z(G7TQ0o;20C=Jm*J~W((W+9o%kRkAb49#JROcfdtAf6b>tg0f!uxcNDjO2zGEMUK+ z&FF-#hlpPw_ZDj?O2;-FrwmW^yf9XQq|gL06ujf)w3`qAMS5srG~OhV|F;THbmag4 zM(k+p@z{N_J7TxQHpbS*24Zgwzc~B^_x|@RDfkaZfFr;W;0SO8I0762j=-rwfbwo2* znr|EdjsQo1Bft^h2yg^A0vrL3z|12sO{UrQ0p|aUJkRBSF8|{u7+C)R{y#4N1MN|> zYt91y|K}6$CEiLLOZ+(TeB!~x*An{^?fw6o1@Ik?z!{4G_x}fm;VF(4Z4G8;(siP3 z!fOxv;y9#FQVrE|LBrX5F7+ydB|Yo^$X2NdW3W)UexGrVaPpl4oPMi?3$UC-)~fs*gH3M0IYJzG`xXN`-%^kNZ@qU#nCEW@UxAne zRlmelgwqd~zBIrY7^lw#SqJD!2(n=zTOZtUcVag(+~6BWfFr;W;0SO8I0762jsQo1 zBft^h2yg^A0yBWXI4)y};NCIe0x>d>$J(WI0qDFMRkyY`MPhQ^2yr=K)d!J#0nw}r z>{fYLt?(FKOO+k98!8P%1?!!0onoDGrvbasmPR>GC{w(2h!a}3i{;vuVhQ-Lyvrt$ ze+${T|35{7|9$vS-fZ;x}d{kEUCn;QarxkpYTth0tHNsC?PXLRK9b zl#E&2W{PUc%UX>AHo-V7Jg*g+aQ|r+YoaWS2iA9b( zB`Xl6DnW++AV2%HNI#hz!<=Sg+2qx>lcc>l3G@0waYv*2(TV>TVdH=NyzLV|WBLET z`IVt#EdT%Kh7J#~{QtlBhsGGo|KB`+_FCKzj)gdMW+A% zQ_q?#|NkGZ{ptHG|Nlz=Wy_fU|JSZ7w)Fph^xZ{F|NnQleTwP-zv*8WGW`G7Jo9hE zRhIw%;B9->GX4Kwet9Fy|Nr~v#S<+5|Lf)W>n#8O^z3|%<^R90c&Lx*|Nq?8Z!rD; ze|g`{EdT${q`x!5^8dd@x_BdM*VEees;uIC;ph>T|NrL~T3=xK|6lv%?U%6p|Ns7; zzrGAfd8qA^?L__mX*v9tBft^h2yg^A0vrL307rl$aB2_`;kJFkkkAAFADZ6djg)c9 z`2XniTnRtV0s${sAWp+--F4D-ouuQ?0eIKnBPp~X$t#Gw7iI0EOCB8*gam1{Z6H50 z!2fqixTHbv-F88>kre-3{5_IFq$NSP$SXpeEk*zZ?OM07rl$z!BgGa0EC4rwD-)kpHP*>9O0NGWnmn=OVQs-pD(3 zFaOhmC~bew@;@~gp0XlAR}#ot3yxe(9@rPz{XLRGAB8SScv$x$|I@_?@>bJysM|hx z?R(?1jx%)mKiL29!UYUn{^+`0zdIhO%is1v>Q_g9G{Mm2?`WJGdx@dT-}L#rfBG;( zmw%<|-gZ}{E`QCo%l~10c^%T_PkX=B&i3(t`TAT2kN-ti9l7C6r00*g?xKx{~!4Mzdyv%|3CIO2XKJ7Y3~61 z|A*gw>K|G9|E+WG{|QU~f60L_eTAj}zw=Q4O)UNYh4;=MVd?*Wc<9YPW9k1t-H3gg zrT@R#N-ja~hHO{_VXlCqjr-I9{n*i;k^(FjRtal_e%;H9%!0^u7@e;C|H*Zzpg!Ol zcyfM!{GGuk=f9GuCqA9HJdx~x|BuN3_)+*DjsQo1Bft^h2yg^A0vv&phQP&Gn!@wF zHPmY<6eeDdjtw3#I@iREp{j<3jm}$Ivg4eZs^bviEj(Ai=pAL%Ov%LN@tu3FY$~~S z^{P?{(e^oR$pxkMXpo|-tLgk z|3~e~v+xM%|1b8>uej9TkFCS|f{^rnk^Xz+WCS`E@|7I0(`+p3@a&G_6?f-F0Kwc1JTfQoL z->I_yull|Mi<)5sQDDG1;7rgj%o7HL`NANOp$rQ#Q1mauVFgzU3(unuf5QI%PYyr8 zIRKhy@*R!#9hW-B;`hV>I&(Qy4|9^)5AN&7D>i<3e ze}?`a`~S1_|J?sSf)`;`_GhQc|9@JW2mb&3je8zt@&E6eJ@6?+q>;iO7<_d||BpBf zyf=)acIE94&{Yd^9Xj79EEk@hNBsYvo_F=|J;RR={b=|X@mFH^#||YH#gap>3?}1F z{N~t$K;ZBMoZS~2i*1g5H2MFE)$jWrc&n)g9{Ak(v1Ev=3Z+)Pst%55vM6h5G2ny8 ziY7y@4Z7NSrB$mHh~4<1dBLC?g&d&h>zr*aP(1!0kN?Nx|6x}~aH5k!eZJPn=Uq2E z-=@yFGvW7mgF8{O^Dgz0{8Bu6)Xp!KciW|+GfD!Z z*GB7mCfwBIhvEm${r?e#28xK>>PsSKMJc1|qKPR&T6{>njFO?YB@@9}H5eefjDmZ* zeMe4f|Nrxm=#Muf|BobgCmuuB*qfk6B`pEZ~-nz2#MqIWAUT$r{f3X z55(_@?~9k>w;(!4zHtON0vrL307rl$z!BgGa0EC49088N8IHhV{QL_9=cSKjUtjx) zr=Gp=jpcv!^JlHSsd8gH=T=86g%D9EEW+D$J7?G2(OU6NSi6^1&Cp?0x=}-vp5Zh} zkwin@kSY}25+oLD^=dKKXfk-hx`TukdC;moTx>^ZP`%{@S<>|OUgpYY4!mz&ci`UN&U$ET=BrmQ-U}Xb%CpO|B$=QcnDSn5`xMo)Bd&knEWZz% zdhZHk2jE{Zgs(tm{?m>8SHPXn zHU~`qgV)T)*CgQYLg5@?EzX1f^a)#qCx?HTcs_9uH~_l$|9_HO|2MJZI~)Oy07rl$ zz!BgGa0EC49088NOd>#JCu+qk2;h(ZCb$2Ouw`HEaQlB~c+=X;s_e1TG5%k~b9Q9^ zA93A9o4+a4kcnVKTKsQ+bKoHAkCX=x$^RD%|7+H-;zwi83_m^e(BS`=|KPx%&&%}p z&Hc&jm;3(LtS<;b`+s!p%xvp@E5?{-#V9GT+~NmxC{^-c{d1@5TvaCvL@t=9es*ul zria^M&#jixS-7s2uNEipBGRoJQutb&KUJuvZk@ap7L>p|rH&&-)GKwn^vT?aUE76> zUP(#=C;2w7cnt~}9Fg;GyHu=3_QYF+)pKK4t|ls6`Ko0%FuE)%q56 ze6w9@xHS|sV{htWFsrpo6FY6NKD2irBU@rK3s$kG1wtak5|T5TVi~GsYV9EMa;pZi z#g%k`NFXbGxa|*7jG&4G7}X%=P=4p8M!ixfTo09JilTU$yu6f^)Qlv_hNkPLflP~A zv=5QwmPJulH5qbGe#~zlgY?~D4L0!Z!S*q3-$7c&cDIV#Nh)hu0K|m7qgeI~ruU{G z*9ht_*A!9bk+jpLorqc4%E%(D#)`6z!%nq=%FAtPhNR12f;K56eXQntSb^De=TJ$w z(X_-0wX4&lVTn+4VDhcYma1a=?ABnR%%)hJO!|Vak4T}DY;b? zpa%^zsyLH%UR$90~Et8t>Zo-WY9 zmrlyalBOB$fgyQiMU)K50<+Ia;e9;Gz@vD-hYXma23lNgPq|XwGmcwTuF`=20uAVN z!9dMwQpQq6MUf3+BFpS{&^)}{hG`j=ictuB%jSA0fSXDXg?eQI^)uXiz-0#XQf25J zr@@yT-bcwwR>qPPMUyP8^~bE4Teft~lF`e@{k}2Bdv{Q~1>T+dhETg-zyQAUcv9v% z!ILKlhrO;g&A8aTHBp%q0*H4rlY*Y6v%J!HZYXEgCc zBWrl@M6+jY0**O{qRsJX{Mp0_iHih!)Tz~AVX;I;hEA?W_f6YC%k8DPXS~k5k4Y@Y*~6I z?F8jE6~)pd3C&vqV~F4(6P^yAxyWc)NzaJjSgx9)*?L9?Z3gECvkOI3KvV)`h$R9V zND%-3g2Ya+@)r{WU zto0#JgQZ5i>+gkAh{_Dkkuo^{KY<7Sa{hk|1?T_Yd`9#C6MOmr;ZMAVO*{NG_;^6L z#XFB|=h3#7{0P)vKIt7JE=sf^-bu7kXZRR71k!!7-Hm?+gwJ}%3%p;&`<1-k4c_l1 z-tR@;FL_=;xX?SMynW|+$Fy6LIDi7k@?=YRztlK?$U8=U1Zbn>3BdoqR5+G+FY#95 z7+ln?{7*{be>nmi0geDifFr;W;0SO8I0762j=<@I0Fm>k^o9KfUVCon@BoAV{|8_E zLt~7=|37M;Kl^Yb|3AbA{Eu&?Uu5wA@7TR&`-2Sr|4m=K?TI3T|Nlzusb|ed{{J=G zF8|@$pT5uF|KDEezib(c|9|A#b;U(cJ=1Vm?OgEKqwg+S7@f9m-@DsBg;0#RZtb#7 z|GE&#oe^nkp82=oDw2_>rLDT`)xq2LtVMEjMB1E#FTcDI$*|MYvTuL?ym$i1wGnBr zT`$L9AB;{rmY$ujA^AMwx}WVU9_mA~e?-~~>z}*&jd{^&_t|wKsgvubdg>!7d%AD` zxgdF?4Tb6pdg=!m18Aw;`v24Gpx_mkqMML6ic!0Nl8(;z2`h!~4*oj+lh_Z2UmW`G zU~B&W$7A=!?ugwI+Ze-Dh6>;tM}Q;15#R`L1ULd50geDi;3Oh2-?#lp+7dc-|18)Lm&T)~Yq|!{n14v%HF10} zO_f;wuT!#uqpX71?FM{AR#HU5Zd08if-3Vw{H419rNd^@V+4W8z<3di^%LkBry(Y- zy~%0qD{@xVGZOepGd%#pS;0+Mx3HHqwJ$*O|5jmj{5WX;UrZeCivNFW{POsQ_-bnU za1zISUNlF5Bft^h2yg^A0vrL307rn00I}Wg$^IXy_wYu_IA!+#C}cg_sF|?mkFY?% zAxe=?10tA8+jWu+i+y<4-y;4&b0a*9XunWMt|46%lHQTW6pJ5k(b^i>z0Id6G*acwSf23W&Y6t87kzK$5e&xWs z11})e{WEL>E+ClTziW`4d*FDrk0(C6%Eu5Nuk>-l$18lS@bPjVAAG#b#{?fQ^>M(* zOZ<1o$BX^<#mDFP?}(2VA)5yOz3{n(zAXbjUVzZRd*pAzw^_hv%Kg}Zv`xkKj^%%@|8L-U^<4k&$$DJ>uZcQh zN96kd5lI2UQDiFM>;IdJZ=nBQDwGpt(EYy(3n8uI|Gg1A8VlP055(RYesP$`|D&Tl z|K$j91ULd50geDifFr;W;0SO8W)=aW#|-GRz4(8T=U;efz}HuYu6gZydp>u!ue%PV z{b2v0k3aA0u|sL2>-LSV_jTH#wA8PTP44&g+krID|2KXy_>!;d4yA3n>$<`9zTP{O zcBQuK=pXqy@KD<2|8VEEm-zbdKpN=(%MZN1)YpxN(k>dg>Yw(9_2i-S3m$v;dz!B^ z52dZ!_x1(XqchX+UhS-1cEvp_d|i6zx|Q$z#eZJo>(v8k;Qv4WyRVmh9eXHk&cVXR zf9~tsgK62fpZmKn|Ff@q52d}f^|>u2Uk@KjJGSA#5!u(ththtw_s}(e>Fei1X)mnb z_Vlo?s}H5!cjR+x-}d$Pp|rdHMBRcr!5omagMR)i1(I(ToSzMd&K3HFd0??IUl;`T z(qSP6tbz&1vrt$BHVsRKWuU2F0p9wnfpw5F2ZVV6D09FPpzB06jOq#1l+Q;X;Cn6- znr1w3@8!Y`i5n7+B>p_{N>}#(_y+tBM}Q;15#R`L1ULd50geDifFr;W;0T=22pq;w zzd&$a`dIe$wV!zE*$dxT{#QSL*4mpYH^y^rb+l3#r9S>7@TOx!%t^cEj@BUNZz?OR znxR9my++M-MyI4fAUe^2fQ^NsTY|)5jf8i{EE0QD$c_Wi59F(^4Nfkjb{&G0L!dh? zEmCiG)guFtB^|=(!c8XN4z02RxrAjQhJ@qZdzmYrIq<%9-GO_5JL{pXnXg{KcrO%H za>{$j5OEp;kxqNB^|)lRF5=7ELroc<@c%6qmVoSzr1(_dM3>3kD@3cFrJl_0|ZUB>^T zmitI$iZ@b5kM{plSOj{o|0l{j-@(0S|9@H#ks~NJ^oeK{bl{Od3%W3j5G|<(|9_+^ z#v8%98~=YK4WaD<^5da1?6V_4b0bMUAA-LZPFWi&V@T-2OM_y99M17Z-=uSX23iDl zJfUs3i<+Tr7OBDmPcbCWt~r|9LzCM5{y%S<(uca@b!hUzk$0IRbi(LBYZU41(MGQA z7s${-&iOUy?~xSReUVllZRF~uUVErl()b0^ zw@c{)ERi&-^m%Aj{7c>fuScBYuL4m0f2NtYPxxbD!wK;JH4EZ990861M}Q;15#R`L z1ULd50geDi;G`kI`Tw)3oJFAmPA>mH(P`tqoc~V{&I@wk=W;l z|9EKW;QoOx%)70BWUe{qk7j>;*8XO`ncd;63{k8M58(AA+|!oR_#I$t5{Omdg7J;x zxK&tu-t3*4K)ZFamOc0V2}fM+`AV}W*oPcuNEgzoS4bU)a2v|uK3s7dL3S_ zb`v)HA=W~80s5w*gMsNRe1>dhWh0|llA>ZOBD(T>)pUa+l2unsLq?n;@8HgW>uDLS zCnZ!*)1&Ql8e^u`$mdu?Fg{K#K_H*1@n$ z)f5exmX!iy+7gP5)Dt$Hww03`k0|h}q6%SVWZe)UfJ}Q}aBjmeEI_3??6;W36cx}L z_RwNJEL2(tBx<6fWi=(EgDE2nhL+Y26_Q)CM8(ie1k7*!G2|Q%7bzUaZ2A}?Sc<*1Oi{5kfPppa zhjT+)wlKd~0J8bfeSFpxr^2gWEy;n{ep zlDA9XmH_>qxEi=HRsHPVluZw}!=76$qq8tGX$mW;TQ{WewK#vOP)*%BdFu&&e@{?& zk* zUycArfFp30AaLCOvIe-vwg=2kIW?SS{+Ft{lSL7mL2^%BMNI zUSsU0@^7579t!Kg-WgP7bbT1A^h{ndZre2JJJabXc#@VK_l_wIvh3bIN^_n?=sHFC^E)&7%J`*t z(o;J>s$>wo>9X!goCSpk2v5gyf>WT)c-B0X0dPf=9iUqVz!T;FlbW;mK4G!&)x=j5 zKgRwz-2cA`4Bz1ha0Jd;1i1WkIT#;7@8>tOMR%l}B! zqf$MW|5Mpc?dYAP>INSWE=%HrTlK1e- zprUfq|DRnaG^I%kN73k^UX$VH=*<70*68`(#pM2-`v23bDfIxkx>x^yTCLPao??OQ zr18QI^|xnx@6#ea7RG|*|8L@Ka|Nt5@tyV0X}dgt$OKVP07NFZh;-4p0$!o?OXps-xA*#UmG8Y zzZH8i_C)Od*uL0UY%>bjz&DP-8IHg^c;v(5$12ZuNCp$r!1^uN^3lOKLb*$sNMS7* z87@uNe^7NX%_xD%EW^YpoK++7OS_Q+=yLAqvT{#UC9RdfZCCDH36|)=%06w&axSgf zl9tZthMRUAMR5y?WoV`d$Wy_z1!LoGE8CIgZSE*Prn_A~`b@!9W8sX$Q71FDl}v-uOgO0rtv3+RJQF@JiDfRn}l zuQcVhKH-zX1H->eJfAq2_*$Z#_;lj(L^9D{{$C#(jvXKV@$i$w51;^zeB%gk1ULd5 z0geDifFr;W;0T-|1gO4_=q_PNzTPnzJXIumn%q;S|3@L_(RRZ6e_9Y#+Y@Yh(EqO@ z4%-rE1gYR7(q8-C z_^jjT(Ie7+u>awO3lKgLX`}0Q{qFdR=(NFcLmw`SXp#<{VVmPMy+`uyEL zeHh7z5!YR*y0_i6Bs%T#e^_5$hv?v^UANlVKK?IXpLQ{Z^GA?@P{eij{qWG6f7TzJcGssHv2UY}9C2O!W-GY_osURU@;B~z z6fImvq>1}x4}1#s>4>z8Hh)v7A+=^iTKsQ+bKoE(2`c)7H>Cgn^}PQ0JA+To|MfiX z|4(t@zZ?OMz}brc*Z*_J+>=3F(N<+=gG;e%H5Xm+|_ zh=%333K(}8@N86H+p*Qs#eQwBp_*)9jr40%2E8_Y`6zoe8K@gGhN>DU9CoNKr^eX! zasuOITZ1){RWl_MtH(R1Et#&W53`hn>NJXv1k0UmUG=&CpX>j@RG}LxoRi@M&h`J? z`V~_-H7StVen(E0^=s4l2JHVA3tvk-4Z43c*GKYyc2+0$|5X2f_DTp*Z&s`Z=2OwO zWbNluY-=(;pF$*N5&SJD%LIyBK+Ti|6pr^0MU}+u|FfzN^!c3skBn~3k00m%6J;$e zQhVTVjFF~2FaeGFaz@e~asPkr|IhjVnh|K)EGmR0yWSTSIR77>s+|^8vTfxkhx7k= zt8zjA66!F@`Tv>(hQN{=9w26h|F7NBPdC2%Iu=Z)u>~w@x)HF*1bw!Eg`^Brkk9}3 z-cVua@x+6$60jfQ0gNTKCpIQT@C8_q5E94Z$AAyu>G;9;1Mz#}`{E^F1=t$DJidWa zE<_2-{2WJsBft^h2yg^A0vrL307rl$z!8`s1p0kjm>_pkASw=xd_(2jP~e-u$qy^C zY-e$))m}LyCI@^(R#NMra4gDB5s?B>?uZ>Z{=3@u!NA9>e7_5Pywdlzz{e|m{|bD( z-1n)#$IE;_3VgiO_npAUOMHI`e7xBAk-*32AioG?ukTsAFG4;IzCQ%cv(Wctz{d-G z{{?&;@L9meaoRi99}oI|2l#lt?`wdM2Ymkmd^~TS&@=zv aQsDDjff@bI?STJpv9JV;{SjAS|NjRwAVj(V delta 6380 zcmds5eQZ=&6~Fh+bUGiMd9R(X*Uy=jwiH_0X^ZW&(+;$<3T;@ryRg~NmMscoi$aUb z8UmC~T;r|@gtlkn!_5jcM%QJHiwZ!Xs4*tZii`K&_wJjS z^0}tFZhU|A_RYJW=lss^{LYPCbH-xMkytiC+EcH$9r00lKpK;~aQ5h9HQ70HK6ld3 zdnCx32_@P`@M#hsmoH_iSAKDCwU0_cghs2AQLF8E$F1gBeE@aJgQ z7+9>9T_wqR3gMY$i*n8S45AS;y49msNf^duZ^3Q-K_<}&5#&KDTq|6Q!9BhRMrB|8 z@z^Y5sB>l)xU!;1K+Nc*Xg&ODSf0Xx0Rkm zB6}Qfmgwm#u%eD$#lFQRB-6vWP!8?lW&a|SfwuK;{$l^!!@~790&)iL!Tt~wMSGnf z2dfQ8?6Rvf1voo?VD+;^w%o7h#Z0)eHG*xRbUyVuhWNO6*hR%ftsKB6T*O zK0zjjl7LwihmzWL7RTUxOEa$E7!rMRD2A%xc&|Htuy+S>XQu2S2jxk55_4=N1v7DF zd?=?m@WcDpndaXg$Fk%l3fds;lg>#Sq@$A0953kL~7{%AF^Ri~^ldqdFO8DLIlMXG#ddWvfb+-3 z6;OhuhNHisJNmWW-(lPgI|jTq51)ykJPxanITcoP`G){krMVrgV1TMK)|u8|VwwvR zBlQ1E(rJQoQ49R6$`N0;Y!6AEyP)kM>yABm^-+bK_{3X2N2|c2 zVh|tryCoQFS&Ai2eKicW`C?F>*Z_4L-Gd!HyEERmdYoV<;F)EHIrOc}yE&5F`Ehfw z-QZvs;FE60pk$SHN>52GQZNx1!GB>MqEF?5W1$ za{aATZv%Q;ptm0N`eowHx1tXH2f?)>pE}`3H~mf8q*k;_?`757YP}8WZNAf<`~V{6$6p*sDYZHPTJT7wot+ms(JPRYe0xeL7^;>xY}biuc8`WFe! zBldS+-{Bk@dTqu5r**X>R*+cG!=alpoa?dB>{X9*L5}!%9_zaGPpm-SG9UD`R@uae z0vM%YN9^=YVe`S4>7c^fb?Ta{L}ue}*WOdalo}!?XatZrDF3zC~c8`F1<@3mfU(W=e&VgVA$h3zYyD;!}v)dMAVavOoABLLlPxoWJJN|W&p3v_n)L2YY?EA&I4y);qS zjc8a@PE88HfnM5c0uK_uAhEjuaRD247<=F1>V*5qyQk#2{!Jy#b+xdj ztxU80$(RMc8Do~8h7AWx;b40)JpTMsQ~(`)zB-eiH!U4ow)AiBKZRlW6JF@)N@>vtz-T1p^S+Kvx z$35H;NulthW{JeMAq+$*WV7)H(*pk$+i=xG$Y$+KEw--ZDd#qOe(& zXNbo;Phz)?$RZuz-4Pe&sWP>Tnj;s%nVzD=wh89gV9MD%=oRS|>5??-*y2!?L0n>Z z_LvTaHz)7}#!3ERIVhC{i!D@eBcv5$MQOztLd7XdMbUm$8X+~5s~&a1NYs)TJ507? zr6i?~5ABuqGUBBCmJ%XHS4`7s|b z@0>ayXKo!}PCI-7KYXdNR9fa@wQBK0({kL*!@Hb^_lYi&C1?4NnG?L(PV)@)PO^R> znF;fHd@Dm)n6mVoD6KGj`F|(7?YKhy+b{{3%Td(CV$u04h#V$UyTiUHhDT8G)9n8t4f5kXO+3;Ak z&W%;|`XpImGcFgH+FPi8ywYF@pS%C{gPStkhcND)<~wnohO_yum3eanUX4gl9%&?2 zt=6WuNR))M$Hi2-`u3lQ46~6YyHKchqbgTly-c*Syn>eHk+Gl^FXWZEn88rh$!ZAt zhEB$V)z`2h3K302XsLiLoK7~MAkAHDVc^xXO(2Ars zi9RO@5g4f%bk}NoqkoxV;|Yo0<$VG!Y2{E=$)=4vHwsSR_hDLo&80WX+J#t3B4oz) zqx3{r$o(P;U3NrvW4J-j=Aew18+l))dH-xcEXItIyw1}INI@xEs7YfCx(v(ixL(D5 z#BB=mGCo;jr3v0|nI6;1CoB4lZhLY2yOO^rD}1fyLg+yizILyX{h~fxn}%+h-MMr< Y@Xkfwgm!dxXRSRnyYoj}_KGt9KafYQZvX%Q diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.ms/tasks/lock.mdb index 6d38eab089708e3c50d476c7a142aea909c6b5c6..b8e0e358dd589f5c229210cdc998a1052c04878c 100644 GIT binary patch delta 78 zcmZp0XmAj}ci{a#rUjiEaSRZ^45UIB)F#^NPL$D@m>@E-L4cK^AgX|IVxTsdz2QU= PoL%q|%--z4ct8#Sgm4!{ delta 65 zcmZp0XmAj`ci{a#rUZQ*Zw3fpnP_P|(Zz@DfZKn^`_&TzwI&~66lS(UFs;GN%?^wQ F Date: Mon, 7 Jul 2025 16:42:50 +0200 Subject: [PATCH 083/312] Fix existing snaps --- .../kefir_settings.snap | 11 +- ...rEnqueuedAt_equal_2025-01-16T16_47_41.snap | 326 ++++++++---------- ...rFinishedAt_equal_2025-01-16T16_47_41.snap | 326 ++++++++---------- ...erStartedAt_equal_2025-01-16T16_47_41.snap | 326 ++++++++---------- ...rEnqueuedAt_equal_2025-01-16T16_47_41.snap | 246 +++++++------ ...rFinishedAt_equal_2025-01-16T16_47_41.snap | 246 +++++++------ ...erStartedAt_equal_2025-01-16T16_47_41.snap | 246 +++++++------ ...ue_once_everything_has_been_processed.snap | 155 ++++++++- ...ue_once_everything_has_been_processed.snap | 120 ++++++- .../tests/upgrade/v1_12/v1_12_0.rs | 58 ++-- 10 files changed, 1117 insertions(+), 943 deletions(-) diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap index af7e82c8b..3c97dbe70 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/kefir_settings.snap @@ -61,7 +61,16 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "pagination": { "maxTotalHits": 15 }, - "embedders": {}, + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "pooling": "forceMean", + "documentTemplate": "{{doc.description}}", + "documentTemplateMaxBytes": 400 + } + }, "searchCutoffMs": 8000, "localizedAttributes": [ { diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index f4edae51b..b56cc5ca3 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -4,7 +4,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 24, + "uid": 30, "progress": null, "details": { "upgradeFrom": "v1.12.0", @@ -26,6 +26,155 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "finishedAt": "[date]", "batchStrategy": "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type." }, + { + "uid": 29, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.067201S", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z", + "batchStrategy": "unspecified" + }, + { + "uid": 28, + "progress": null, + "details": { + "deletedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.012727S", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z", + "batchStrategy": "unspecified" + }, + { + "uid": 27, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.059920S", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z", + "batchStrategy": "unspecified" + }, + { + "uid": 26, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.088879S", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z", + "batchStrategy": "unspecified" + }, + { + "uid": 25, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.312911S", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z", + "batchStrategy": "unspecified" + }, + { + "uid": 24, + "progress": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.247378S", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z", + "batchStrategy": "unspecified" + }, { "uid": 23, "progress": null, @@ -348,179 +497,10 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "2025-01-16T17:01:14.112756687Z", "finishedAt": "2025-01-16T17:01:14.120064527Z", "batchStrategy": "unspecified" - }, - { - "uid": 10, - "progress": null, - "details": { - "faceting": { - "maxValuesPerFacet": 99 - }, - "pagination": { - "maxTotalHits": 15 - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007391353S", - "startedAt": "2025-01-16T17:00:29.201180268Z", - "finishedAt": "2025-01-16T17:00:29.208571621Z", - "batchStrategy": "unspecified" - }, - { - "uid": 9, - "progress": null, - "details": { - "faceting": { - "maxValuesPerFacet": 100 - }, - "pagination": { - "maxTotalHits": 1000 - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007445825S", - "startedAt": "2025-01-16T17:00:15.77629445Z", - "finishedAt": "2025-01-16T17:00:15.783740275Z", - "batchStrategy": "unspecified" - }, - { - "uid": 8, - "progress": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - }, - "disableOnWords": [ - "kefir" - ], - "disableOnAttributes": [ - "surname" - ] - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.012020083S", - "startedAt": "2025-01-16T16:59:42.744086671Z", - "finishedAt": "2025-01-16T16:59:42.756106754Z", - "batchStrategy": "unspecified" - }, - { - "uid": 7, - "progress": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - } - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007440092S", - "startedAt": "2025-01-16T16:58:41.2155771Z", - "finishedAt": "2025-01-16T16:58:41.223017192Z", - "batchStrategy": "unspecified" - }, - { - "uid": 6, - "progress": null, - "details": { - "synonyms": { - "boubou": [ - "kefir" - ] - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007565161S", - "startedAt": "2025-01-16T16:54:51.940332781Z", - "finishedAt": "2025-01-16T16:54:51.947897942Z", - "batchStrategy": "unspecified" - }, - { - "uid": 5, - "progress": null, - "details": { - "stopWords": [ - "le", - "un" - ] - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.016307263S", - "startedAt": "2025-01-16T16:53:19.913351957Z", - "finishedAt": "2025-01-16T16:53:19.92965922Z", - "batchStrategy": "unspecified" } ], - "total": 23, + "total": 29, "limit": 20, - "from": 24, - "next": 4 + "from": 30, + "next": 10 } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index f4edae51b..b56cc5ca3 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -4,7 +4,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 24, + "uid": 30, "progress": null, "details": { "upgradeFrom": "v1.12.0", @@ -26,6 +26,155 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "finishedAt": "[date]", "batchStrategy": "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type." }, + { + "uid": 29, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.067201S", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z", + "batchStrategy": "unspecified" + }, + { + "uid": 28, + "progress": null, + "details": { + "deletedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.012727S", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z", + "batchStrategy": "unspecified" + }, + { + "uid": 27, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.059920S", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z", + "batchStrategy": "unspecified" + }, + { + "uid": 26, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.088879S", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z", + "batchStrategy": "unspecified" + }, + { + "uid": 25, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.312911S", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z", + "batchStrategy": "unspecified" + }, + { + "uid": 24, + "progress": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.247378S", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z", + "batchStrategy": "unspecified" + }, { "uid": 23, "progress": null, @@ -348,179 +497,10 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "2025-01-16T17:01:14.112756687Z", "finishedAt": "2025-01-16T17:01:14.120064527Z", "batchStrategy": "unspecified" - }, - { - "uid": 10, - "progress": null, - "details": { - "faceting": { - "maxValuesPerFacet": 99 - }, - "pagination": { - "maxTotalHits": 15 - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007391353S", - "startedAt": "2025-01-16T17:00:29.201180268Z", - "finishedAt": "2025-01-16T17:00:29.208571621Z", - "batchStrategy": "unspecified" - }, - { - "uid": 9, - "progress": null, - "details": { - "faceting": { - "maxValuesPerFacet": 100 - }, - "pagination": { - "maxTotalHits": 1000 - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007445825S", - "startedAt": "2025-01-16T17:00:15.77629445Z", - "finishedAt": "2025-01-16T17:00:15.783740275Z", - "batchStrategy": "unspecified" - }, - { - "uid": 8, - "progress": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - }, - "disableOnWords": [ - "kefir" - ], - "disableOnAttributes": [ - "surname" - ] - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.012020083S", - "startedAt": "2025-01-16T16:59:42.744086671Z", - "finishedAt": "2025-01-16T16:59:42.756106754Z", - "batchStrategy": "unspecified" - }, - { - "uid": 7, - "progress": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - } - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007440092S", - "startedAt": "2025-01-16T16:58:41.2155771Z", - "finishedAt": "2025-01-16T16:58:41.223017192Z", - "batchStrategy": "unspecified" - }, - { - "uid": 6, - "progress": null, - "details": { - "synonyms": { - "boubou": [ - "kefir" - ] - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007565161S", - "startedAt": "2025-01-16T16:54:51.940332781Z", - "finishedAt": "2025-01-16T16:54:51.947897942Z", - "batchStrategy": "unspecified" - }, - { - "uid": 5, - "progress": null, - "details": { - "stopWords": [ - "le", - "un" - ] - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.016307263S", - "startedAt": "2025-01-16T16:53:19.913351957Z", - "finishedAt": "2025-01-16T16:53:19.92965922Z", - "batchStrategy": "unspecified" } ], - "total": 23, + "total": 29, "limit": 20, - "from": 24, - "next": 4 + "from": 30, + "next": 10 } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index f4edae51b..b56cc5ca3 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -4,7 +4,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 24, + "uid": 30, "progress": null, "details": { "upgradeFrom": "v1.12.0", @@ -26,6 +26,155 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "finishedAt": "[date]", "batchStrategy": "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type." }, + { + "uid": 29, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.067201S", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z", + "batchStrategy": "unspecified" + }, + { + "uid": 28, + "progress": null, + "details": { + "deletedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.012727S", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z", + "batchStrategy": "unspecified" + }, + { + "uid": 27, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.059920S", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z", + "batchStrategy": "unspecified" + }, + { + "uid": 26, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.088879S", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z", + "batchStrategy": "unspecified" + }, + { + "uid": 25, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.312911S", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z", + "batchStrategy": "unspecified" + }, + { + "uid": 24, + "progress": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.247378S", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z", + "batchStrategy": "unspecified" + }, { "uid": 23, "progress": null, @@ -348,179 +497,10 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "2025-01-16T17:01:14.112756687Z", "finishedAt": "2025-01-16T17:01:14.120064527Z", "batchStrategy": "unspecified" - }, - { - "uid": 10, - "progress": null, - "details": { - "faceting": { - "maxValuesPerFacet": 99 - }, - "pagination": { - "maxTotalHits": 15 - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007391353S", - "startedAt": "2025-01-16T17:00:29.201180268Z", - "finishedAt": "2025-01-16T17:00:29.208571621Z", - "batchStrategy": "unspecified" - }, - { - "uid": 9, - "progress": null, - "details": { - "faceting": { - "maxValuesPerFacet": 100 - }, - "pagination": { - "maxTotalHits": 1000 - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007445825S", - "startedAt": "2025-01-16T17:00:15.77629445Z", - "finishedAt": "2025-01-16T17:00:15.783740275Z", - "batchStrategy": "unspecified" - }, - { - "uid": 8, - "progress": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - }, - "disableOnWords": [ - "kefir" - ], - "disableOnAttributes": [ - "surname" - ] - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.012020083S", - "startedAt": "2025-01-16T16:59:42.744086671Z", - "finishedAt": "2025-01-16T16:59:42.756106754Z", - "batchStrategy": "unspecified" - }, - { - "uid": 7, - "progress": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - } - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007440092S", - "startedAt": "2025-01-16T16:58:41.2155771Z", - "finishedAt": "2025-01-16T16:58:41.223017192Z", - "batchStrategy": "unspecified" - }, - { - "uid": 6, - "progress": null, - "details": { - "synonyms": { - "boubou": [ - "kefir" - ] - } - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.007565161S", - "startedAt": "2025-01-16T16:54:51.940332781Z", - "finishedAt": "2025-01-16T16:54:51.947897942Z", - "batchStrategy": "unspecified" - }, - { - "uid": 5, - "progress": null, - "details": { - "stopWords": [ - "le", - "un" - ] - }, - "stats": { - "totalNbTasks": 1, - "status": { - "succeeded": 1 - }, - "types": { - "settingsUpdate": 1 - }, - "indexUids": { - "kefir": 1 - } - }, - "duration": "PT0.016307263S", - "startedAt": "2025-01-16T16:53:19.913351957Z", - "finishedAt": "2025-01-16T16:53:19.92965922Z", - "batchStrategy": "unspecified" } ], - "total": 23, + "total": 29, "limit": 20, - "from": 24, - "next": 4 + "from": 30, + "next": 10 } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index 01d2ea341..a52072f56 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -4,8 +4,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 25, - "batchUid": 24, + "uid": 31, + "batchUid": 30, "indexUid": null, "status": "succeeded", "type": "upgradeDatabase", @@ -20,6 +20,118 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "[date]", "finishedAt": "[date]" }, + { + "uid": 30, + "batchUid": 29, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.067201S", + "enqueuedAt": "2025-07-07T13:43:08.772432Z", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z" + }, + { + "uid": 29, + "batchUid": 28, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 1 + }, + "error": null, + "duration": "PT0.012727S", + "enqueuedAt": "2025-07-07T13:42:50.744793Z", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z" + }, + { + "uid": 28, + "batchUid": 27, + "indexUid": "kefir", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "Index `kefir`: Bad embedder configuration in the document with id: `2`. Could not parse `._vectors.doggo_embedder`: trailing characters at line 1 column 13", + "code": "invalid_vectors_type", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_vectors_type" + }, + "duration": "PT0.059920S", + "enqueuedAt": "2025-07-07T13:42:15.624598Z", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z" + }, + { + "uid": 27, + "batchUid": 26, + "indexUid": "mieli", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.088879S", + "enqueuedAt": "2025-07-07T13:40:01.46081Z", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z" + }, + { + "uid": 26, + "batchUid": 25, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.312911S", + "enqueuedAt": "2025-07-07T13:32:46.13871Z", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z" + }, + { + "uid": 25, + "batchUid": 24, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "error": null, + "duration": "PT0.247378S", + "enqueuedAt": "2025-07-07T13:28:27.390054Z", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z" + }, { "uid": 24, "batchUid": 23, @@ -264,134 +376,10 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "enqueuedAt": "2025-01-16T17:02:52.527382964Z", "startedAt": "2025-01-16T17:02:52.539749853Z", "finishedAt": "2025-01-16T17:02:52.547390016Z" - }, - { - "uid": 11, - "batchUid": 11, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "searchCutoffMs": 8000 - }, - "error": null, - "duration": "PT0.007307840S", - "enqueuedAt": "2025-01-16T17:01:14.100316617Z", - "startedAt": "2025-01-16T17:01:14.112756687Z", - "finishedAt": "2025-01-16T17:01:14.120064527Z" - }, - { - "uid": 10, - "batchUid": 10, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "faceting": { - "maxValuesPerFacet": 99 - }, - "pagination": { - "maxTotalHits": 15 - } - }, - "error": null, - "duration": "PT0.007391353S", - "enqueuedAt": "2025-01-16T17:00:29.188815062Z", - "startedAt": "2025-01-16T17:00:29.201180268Z", - "finishedAt": "2025-01-16T17:00:29.208571621Z" - }, - { - "uid": 9, - "batchUid": 9, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "faceting": { - "maxValuesPerFacet": 100 - }, - "pagination": { - "maxTotalHits": 1000 - } - }, - "error": null, - "duration": "PT0.007445825S", - "enqueuedAt": "2025-01-16T17:00:15.759501709Z", - "startedAt": "2025-01-16T17:00:15.77629445Z", - "finishedAt": "2025-01-16T17:00:15.783740275Z" - }, - { - "uid": 8, - "batchUid": 8, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - }, - "disableOnWords": [ - "kefir" - ], - "disableOnAttributes": [ - "surname" - ] - } - }, - "error": null, - "duration": "PT0.012020083S", - "enqueuedAt": "2025-01-16T16:59:42.727292501Z", - "startedAt": "2025-01-16T16:59:42.744086671Z", - "finishedAt": "2025-01-16T16:59:42.756106754Z" - }, - { - "uid": 7, - "batchUid": 7, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - } - } - }, - "error": null, - "duration": "PT0.007440092S", - "enqueuedAt": "2025-01-16T16:58:41.203145044Z", - "startedAt": "2025-01-16T16:58:41.2155771Z", - "finishedAt": "2025-01-16T16:58:41.223017192Z" - }, - { - "uid": 6, - "batchUid": 6, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "synonyms": { - "boubou": [ - "kefir" - ] - } - }, - "error": null, - "duration": "PT0.007565161S", - "enqueuedAt": "2025-01-16T16:54:51.927866243Z", - "startedAt": "2025-01-16T16:54:51.940332781Z", - "finishedAt": "2025-01-16T16:54:51.947897942Z" } ], - "total": 24, + "total": 30, "limit": 20, - "from": 25, - "next": 5 + "from": 31, + "next": 11 } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index 01d2ea341..a52072f56 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -4,8 +4,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 25, - "batchUid": 24, + "uid": 31, + "batchUid": 30, "indexUid": null, "status": "succeeded", "type": "upgradeDatabase", @@ -20,6 +20,118 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "[date]", "finishedAt": "[date]" }, + { + "uid": 30, + "batchUid": 29, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.067201S", + "enqueuedAt": "2025-07-07T13:43:08.772432Z", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z" + }, + { + "uid": 29, + "batchUid": 28, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 1 + }, + "error": null, + "duration": "PT0.012727S", + "enqueuedAt": "2025-07-07T13:42:50.744793Z", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z" + }, + { + "uid": 28, + "batchUid": 27, + "indexUid": "kefir", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "Index `kefir`: Bad embedder configuration in the document with id: `2`. Could not parse `._vectors.doggo_embedder`: trailing characters at line 1 column 13", + "code": "invalid_vectors_type", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_vectors_type" + }, + "duration": "PT0.059920S", + "enqueuedAt": "2025-07-07T13:42:15.624598Z", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z" + }, + { + "uid": 27, + "batchUid": 26, + "indexUid": "mieli", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.088879S", + "enqueuedAt": "2025-07-07T13:40:01.46081Z", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z" + }, + { + "uid": 26, + "batchUid": 25, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.312911S", + "enqueuedAt": "2025-07-07T13:32:46.13871Z", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z" + }, + { + "uid": 25, + "batchUid": 24, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "error": null, + "duration": "PT0.247378S", + "enqueuedAt": "2025-07-07T13:28:27.390054Z", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z" + }, { "uid": 24, "batchUid": 23, @@ -264,134 +376,10 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "enqueuedAt": "2025-01-16T17:02:52.527382964Z", "startedAt": "2025-01-16T17:02:52.539749853Z", "finishedAt": "2025-01-16T17:02:52.547390016Z" - }, - { - "uid": 11, - "batchUid": 11, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "searchCutoffMs": 8000 - }, - "error": null, - "duration": "PT0.007307840S", - "enqueuedAt": "2025-01-16T17:01:14.100316617Z", - "startedAt": "2025-01-16T17:01:14.112756687Z", - "finishedAt": "2025-01-16T17:01:14.120064527Z" - }, - { - "uid": 10, - "batchUid": 10, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "faceting": { - "maxValuesPerFacet": 99 - }, - "pagination": { - "maxTotalHits": 15 - } - }, - "error": null, - "duration": "PT0.007391353S", - "enqueuedAt": "2025-01-16T17:00:29.188815062Z", - "startedAt": "2025-01-16T17:00:29.201180268Z", - "finishedAt": "2025-01-16T17:00:29.208571621Z" - }, - { - "uid": 9, - "batchUid": 9, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "faceting": { - "maxValuesPerFacet": 100 - }, - "pagination": { - "maxTotalHits": 1000 - } - }, - "error": null, - "duration": "PT0.007445825S", - "enqueuedAt": "2025-01-16T17:00:15.759501709Z", - "startedAt": "2025-01-16T17:00:15.77629445Z", - "finishedAt": "2025-01-16T17:00:15.783740275Z" - }, - { - "uid": 8, - "batchUid": 8, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - }, - "disableOnWords": [ - "kefir" - ], - "disableOnAttributes": [ - "surname" - ] - } - }, - "error": null, - "duration": "PT0.012020083S", - "enqueuedAt": "2025-01-16T16:59:42.727292501Z", - "startedAt": "2025-01-16T16:59:42.744086671Z", - "finishedAt": "2025-01-16T16:59:42.756106754Z" - }, - { - "uid": 7, - "batchUid": 7, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - } - } - }, - "error": null, - "duration": "PT0.007440092S", - "enqueuedAt": "2025-01-16T16:58:41.203145044Z", - "startedAt": "2025-01-16T16:58:41.2155771Z", - "finishedAt": "2025-01-16T16:58:41.223017192Z" - }, - { - "uid": 6, - "batchUid": 6, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "synonyms": { - "boubou": [ - "kefir" - ] - } - }, - "error": null, - "duration": "PT0.007565161S", - "enqueuedAt": "2025-01-16T16:54:51.927866243Z", - "startedAt": "2025-01-16T16:54:51.940332781Z", - "finishedAt": "2025-01-16T16:54:51.947897942Z" } ], - "total": 24, + "total": 30, "limit": 20, - "from": 25, - "next": 5 + "from": 31, + "next": 11 } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index 01d2ea341..a52072f56 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -4,8 +4,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 25, - "batchUid": 24, + "uid": 31, + "batchUid": 30, "indexUid": null, "status": "succeeded", "type": "upgradeDatabase", @@ -20,6 +20,118 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "[date]", "finishedAt": "[date]" }, + { + "uid": 30, + "batchUid": 29, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.067201S", + "enqueuedAt": "2025-07-07T13:43:08.772432Z", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z" + }, + { + "uid": 29, + "batchUid": 28, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 1 + }, + "error": null, + "duration": "PT0.012727S", + "enqueuedAt": "2025-07-07T13:42:50.744793Z", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z" + }, + { + "uid": 28, + "batchUid": 27, + "indexUid": "kefir", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "Index `kefir`: Bad embedder configuration in the document with id: `2`. Could not parse `._vectors.doggo_embedder`: trailing characters at line 1 column 13", + "code": "invalid_vectors_type", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_vectors_type" + }, + "duration": "PT0.059920S", + "enqueuedAt": "2025-07-07T13:42:15.624598Z", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z" + }, + { + "uid": 27, + "batchUid": 26, + "indexUid": "mieli", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.088879S", + "enqueuedAt": "2025-07-07T13:40:01.46081Z", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z" + }, + { + "uid": 26, + "batchUid": 25, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.312911S", + "enqueuedAt": "2025-07-07T13:32:46.13871Z", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z" + }, + { + "uid": 25, + "batchUid": 24, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "error": null, + "duration": "PT0.247378S", + "enqueuedAt": "2025-07-07T13:28:27.390054Z", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z" + }, { "uid": 24, "batchUid": 23, @@ -264,134 +376,10 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "enqueuedAt": "2025-01-16T17:02:52.527382964Z", "startedAt": "2025-01-16T17:02:52.539749853Z", "finishedAt": "2025-01-16T17:02:52.547390016Z" - }, - { - "uid": 11, - "batchUid": 11, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "searchCutoffMs": 8000 - }, - "error": null, - "duration": "PT0.007307840S", - "enqueuedAt": "2025-01-16T17:01:14.100316617Z", - "startedAt": "2025-01-16T17:01:14.112756687Z", - "finishedAt": "2025-01-16T17:01:14.120064527Z" - }, - { - "uid": 10, - "batchUid": 10, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "faceting": { - "maxValuesPerFacet": 99 - }, - "pagination": { - "maxTotalHits": 15 - } - }, - "error": null, - "duration": "PT0.007391353S", - "enqueuedAt": "2025-01-16T17:00:29.188815062Z", - "startedAt": "2025-01-16T17:00:29.201180268Z", - "finishedAt": "2025-01-16T17:00:29.208571621Z" - }, - { - "uid": 9, - "batchUid": 9, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "faceting": { - "maxValuesPerFacet": 100 - }, - "pagination": { - "maxTotalHits": 1000 - } - }, - "error": null, - "duration": "PT0.007445825S", - "enqueuedAt": "2025-01-16T17:00:15.759501709Z", - "startedAt": "2025-01-16T17:00:15.77629445Z", - "finishedAt": "2025-01-16T17:00:15.783740275Z" - }, - { - "uid": 8, - "batchUid": 8, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - }, - "disableOnWords": [ - "kefir" - ], - "disableOnAttributes": [ - "surname" - ] - } - }, - "error": null, - "duration": "PT0.012020083S", - "enqueuedAt": "2025-01-16T16:59:42.727292501Z", - "startedAt": "2025-01-16T16:59:42.744086671Z", - "finishedAt": "2025-01-16T16:59:42.756106754Z" - }, - { - "uid": 7, - "batchUid": 7, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "typoTolerance": { - "minWordSizeForTypos": { - "oneTypo": 4 - } - } - }, - "error": null, - "duration": "PT0.007440092S", - "enqueuedAt": "2025-01-16T16:58:41.203145044Z", - "startedAt": "2025-01-16T16:58:41.2155771Z", - "finishedAt": "2025-01-16T16:58:41.223017192Z" - }, - { - "uid": 6, - "batchUid": 6, - "indexUid": "kefir", - "status": "succeeded", - "type": "settingsUpdate", - "canceledBy": null, - "details": { - "synonyms": { - "boubou": [ - "kefir" - ] - } - }, - "error": null, - "duration": "PT0.007565161S", - "enqueuedAt": "2025-01-16T16:54:51.927866243Z", - "startedAt": "2025-01-16T16:54:51.940332781Z", - "finishedAt": "2025-01-16T16:54:51.947897942Z" } ], - "total": 24, + "total": 30, "limit": 20, - "from": 25, - "next": 5 + "from": 31, + "next": 11 } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap index fb62b35da..81b50fb92 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap @@ -4,7 +4,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 24, + "uid": 30, "progress": null, "details": { "upgradeFrom": "v1.12.0", @@ -26,6 +26,155 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "finishedAt": "[date]", "batchStrategy": "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type." }, + { + "uid": 29, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.067201S", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z", + "batchStrategy": "unspecified" + }, + { + "uid": 28, + "progress": null, + "details": { + "deletedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "indexDeletion": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.012727S", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z", + "batchStrategy": "unspecified" + }, + { + "uid": 27, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "failed": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.059920S", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z", + "batchStrategy": "unspecified" + }, + { + "uid": 26, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "mieli": 1 + } + }, + "duration": "PT0.088879S", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z", + "batchStrategy": "unspecified" + }, + { + "uid": 25, + "progress": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "documentAdditionOrUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.312911S", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z", + "batchStrategy": "unspecified" + }, + { + "uid": 24, + "progress": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "stats": { + "totalNbTasks": 1, + "status": { + "succeeded": 1 + }, + "types": { + "settingsUpdate": 1 + }, + "indexUids": { + "kefir": 1 + } + }, + "duration": "PT0.247378S", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z", + "batchStrategy": "unspecified" + }, { "uid": 23, "progress": null, @@ -642,8 +791,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "batchStrategy": "unspecified" } ], - "total": 25, + "total": 31, "limit": 1000, - "from": 24, + "from": 30, "next": null } diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap index abb4dcdd9..1ec334fed 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap @@ -4,8 +4,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs { "results": [ { - "uid": 25, - "batchUid": 24, + "uid": 31, + "batchUid": 30, "indexUid": null, "status": "succeeded", "type": "upgradeDatabase", @@ -20,6 +20,118 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "startedAt": "[date]", "finishedAt": "[date]" }, + { + "uid": 30, + "batchUid": 29, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.067201S", + "enqueuedAt": "2025-07-07T13:43:08.772432Z", + "startedAt": "2025-07-07T13:43:08.772854Z", + "finishedAt": "2025-07-07T13:43:08.840055Z" + }, + { + "uid": 29, + "batchUid": 28, + "indexUid": "mieli", + "status": "succeeded", + "type": "indexDeletion", + "canceledBy": null, + "details": { + "deletedDocuments": 1 + }, + "error": null, + "duration": "PT0.012727S", + "enqueuedAt": "2025-07-07T13:42:50.744793Z", + "startedAt": "2025-07-07T13:42:50.745461Z", + "finishedAt": "2025-07-07T13:42:50.758188Z" + }, + { + "uid": 28, + "batchUid": 27, + "indexUid": "kefir", + "status": "failed", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 0 + }, + "error": { + "message": "Index `kefir`: Bad embedder configuration in the document with id: `2`. Could not parse `._vectors.doggo_embedder`: trailing characters at line 1 column 13", + "code": "invalid_vectors_type", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_vectors_type" + }, + "duration": "PT0.059920S", + "enqueuedAt": "2025-07-07T13:42:15.624598Z", + "startedAt": "2025-07-07T13:42:15.625413Z", + "finishedAt": "2025-07-07T13:42:15.685333Z" + }, + { + "uid": 27, + "batchUid": 26, + "indexUid": "mieli", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.088879S", + "enqueuedAt": "2025-07-07T13:40:01.46081Z", + "startedAt": "2025-07-07T13:40:01.461741Z", + "finishedAt": "2025-07-07T13:40:01.55062Z" + }, + { + "uid": 26, + "batchUid": 25, + "indexUid": "kefir", + "status": "succeeded", + "type": "documentAdditionOrUpdate", + "canceledBy": null, + "details": { + "receivedDocuments": 1, + "indexedDocuments": 1 + }, + "error": null, + "duration": "PT0.312911S", + "enqueuedAt": "2025-07-07T13:32:46.13871Z", + "startedAt": "2025-07-07T13:32:46.139785Z", + "finishedAt": "2025-07-07T13:32:46.452696Z" + }, + { + "uid": 25, + "batchUid": 24, + "indexUid": "kefir", + "status": "succeeded", + "type": "settingsUpdate", + "canceledBy": null, + "details": { + "embedders": { + "doggo_embedder": { + "source": "huggingFace", + "model": "sentence-transformers/all-MiniLM-L6-v2", + "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e", + "documentTemplate": "{{doc.description}}" + } + } + }, + "error": null, + "duration": "PT0.247378S", + "enqueuedAt": "2025-07-07T13:28:27.390054Z", + "startedAt": "2025-07-07T13:28:27.391344Z", + "finishedAt": "2025-07-07T13:28:27.638722Z" + }, { "uid": 24, "batchUid": 23, @@ -497,8 +609,8 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "finishedAt": "2025-01-16T16:45:16.131303739Z" } ], - "total": 26, + "total": 32, "limit": 1000, - "from": 25, + "from": 31, "next": null } diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs index 1b2ae054c..372e24792 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +++ b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs @@ -114,13 +114,13 @@ async fn check_the_index_scheduler(server: &Server) { // All the indexes are still present let (indexes, _) = server.list_indexes(None, None).await; - snapshot!(indexes, @r#" + snapshot!(indexes, @r###" { "results": [ { "uid": "kefir", "createdAt": "2025-01-16T16:45:16.020663157Z", - "updatedAt": "2025-01-23T11:36:22.634859166Z", + "updatedAt": "2025-07-07T13:43:08.835381Z", "primaryKey": "id" } ], @@ -128,7 +128,7 @@ async fn check_the_index_scheduler(server: &Server) { "limit": 20, "total": 1 } - "#); + "###); // And their metadata are still right let (stats, _) = server.stats().await; assert_json_snapshot!(stats, { @@ -141,21 +141,21 @@ async fn check_the_index_scheduler(server: &Server) { { "databaseSize": "[bytes]", "usedDatabaseSize": "[bytes]", - "lastUpdate": "2025-01-23T11:36:22.634859166Z", + "lastUpdate": "2025-07-07T13:43:08.835381Z", "indexes": { "kefir": { - "numberOfDocuments": 1, + "numberOfDocuments": 2, "rawDocumentDbSize": "[bytes]", "avgDocumentSize": "[bytes]", "isIndexing": false, - "numberOfEmbeddings": 0, - "numberOfEmbeddedDocuments": 0, + "numberOfEmbeddings": 2, + "numberOfEmbeddedDocuments": 2, "fieldDistribution": { - "age": 1, - "description": 1, - "id": 1, - "name": 1, - "surname": 1 + "age": 2, + "description": 2, + "id": 2, + "name": 2, + "surname": 2 } } } @@ -227,21 +227,21 @@ async fn check_the_index_scheduler(server: &Server) { { "databaseSize": "[bytes]", "usedDatabaseSize": "[bytes]", - "lastUpdate": "2025-01-23T11:36:22.634859166Z", + "lastUpdate": "2025-07-07T13:43:08.835381Z", "indexes": { "kefir": { - "numberOfDocuments": 1, + "numberOfDocuments": 2, "rawDocumentDbSize": "[bytes]", "avgDocumentSize": "[bytes]", "isIndexing": false, - "numberOfEmbeddings": 0, - "numberOfEmbeddedDocuments": 0, + "numberOfEmbeddings": 2, + "numberOfEmbeddedDocuments": 2, "fieldDistribution": { - "age": 1, - "description": 1, - "id": 1, - "name": 1, - "surname": 1 + "age": 2, + "description": 2, + "id": 2, + "name": 2, + "surname": 2 } } } @@ -254,18 +254,18 @@ async fn check_the_index_scheduler(server: &Server) { ".avgDocumentSize" => "[bytes]", }), @r###" { - "numberOfDocuments": 1, + "numberOfDocuments": 2, "rawDocumentDbSize": "[bytes]", "avgDocumentSize": "[bytes]", "isIndexing": false, - "numberOfEmbeddings": 0, - "numberOfEmbeddedDocuments": 0, + "numberOfEmbeddings": 2, + "numberOfEmbeddedDocuments": 2, "fieldDistribution": { - "age": 1, - "description": 1, - "id": 1, - "name": 1, - "surname": 1 + "age": 2, + "description": 2, + "id": 2, + "name": 2, + "surname": 2 } } "###); From 5f8f48ec95b7c29ebdf70a1af3246a6fdaf1fc39 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 7 Jul 2025 16:43:05 +0200 Subject: [PATCH 084/312] Add new snapshot checking for regenerativeness --- .../search_with_retrieve_vectors.snap | 40 +++++++++++++++++++ .../tests/upgrade/v1_12/v1_12_0.rs | 4 ++ 2 files changed, 44 insertions(+) create mode 100644 crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/search_with_retrieve_vectors.snap diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/search_with_retrieve_vectors.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/search_with_retrieve_vectors.snap new file mode 100644 index 000000000..5baf8155c --- /dev/null +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_features/search_with_retrieve_vectors.snap @@ -0,0 +1,40 @@ +--- +source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +--- +[ + { + "id": 1, + "name": "kefir", + "surname": [ + "kef", + "kefkef", + "kefirounet", + "boubou" + ], + "age": 1.4, + "description": "kefir est un petit chien blanc très mignon", + "_vectors": { + "doggo_embedder": { + "embeddings": "[vector]", + "regenerate": true + } + } + }, + { + "id": 2, + "name": "intel", + "surname": [ + "untel", + "tétel", + "iouiou" + ], + "age": 11.5, + "description": "intel est un grand beagle très mignon", + "_vectors": { + "doggo_embedder": { + "embeddings": "[vector]", + "regenerate": false + } + } + } +] diff --git a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs index 372e24792..b98f27b2d 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs +++ b/crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs @@ -295,4 +295,8 @@ async fn check_the_index_features(server: &Server) { let (results, _status) = kefir.search_post(json!({ "sort": ["age:asc"], "filter": "surname = kefirounet" })).await; snapshot!(results, name: "search_with_sort_and_filter"); + + // ensuring we can get the vectors and their `regenerate` is still good. + let (results, _status) = kefir.search_post(json!({"retrieveVectors": true})).await; + snapshot!(json_string!(results["hits"], {"[]._vectors.doggo_embedder.embeddings" => "[vector]"}), name: "search_with_retrieve_vectors"); } From 073e9f29675ccf26d733ccefa5f7eaf6e47db5cd Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 7 Jul 2025 16:46:16 +0200 Subject: [PATCH 085/312] Disable similarity check on composite embedders using fragments --- crates/milli/src/vector/composite.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/crates/milli/src/vector/composite.rs b/crates/milli/src/vector/composite.rs index 8314b8649..130e2674a 100644 --- a/crates/milli/src/vector/composite.rs +++ b/crates/milli/src/vector/composite.rs @@ -59,12 +59,24 @@ pub struct EmbedderOptions { impl Embedder { pub fn new( - EmbedderOptions { search, index }: EmbedderOptions, + EmbedderOptions { search: search_options, index: index_options }: EmbedderOptions, cache_cap: usize, ) -> Result { - let search = SubEmbedder::new(search, cache_cap)?; + // don't check similarity if one child is a rest embedder with fragments + // FIXME: skipping the check isn't ideal but we are unsure how to handle fragments in this context + let mut skip_similarity_check = false; + for options in [&search_options, &index_options] { + if let SubEmbedderOptions::Rest(options) = &options { + if !options.search_fragments.is_empty() || !options.indexing_fragments.is_empty() { + skip_similarity_check = true; + break; + } + } + } + + let search = SubEmbedder::new(search_options, cache_cap)?; // cache is only used at search - let index = SubEmbedder::new(index, 0)?; + let index = SubEmbedder::new(index_options, 0)?; // check dimensions if search.dimensions() != index.dimensions() { @@ -73,7 +85,12 @@ impl Embedder { index.dimensions(), )); } + // check similarity + if skip_similarity_check { + return Ok(Self { search, index }); + } + let search_embeddings = search .embed( vec![ From 3261aadcf219705fe5ced715880522bcffb50f8a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 7 Jul 2025 16:50:39 +0200 Subject: [PATCH 086/312] Add composite test --- crates/meilisearch/tests/vector/fragments.rs | 109 ++++++++++++++++++- crates/milli/src/vector/composite.rs | 2 +- 2 files changed, 104 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index db57f9f6e..2626284a0 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -21,9 +21,7 @@ async fn shared_index_for_fragments() -> Index<'static, Shared> { server._index(uid).to_shared() } -pub async fn init_fragments_index() -> (Server, String, crate::common::Value) { - let mock_server = MockServer::start().await; - +async fn fragment_mock_server() -> String { let text_to_embedding: BTreeMap<_, _> = vec![ ("kefir", [0.5, -0.5, 0.0]), ("intel", [1.0, 1.0, 0.0]), @@ -35,10 +33,13 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va .into_iter() .collect(); + let mock_server = MockServer::start().await; + Mock::given(method("POST")) .and(path("/")) .respond_with(move |req: &Request| { let text = String::from_utf8_lossy(&req.body).to_string(); + let mut data = [0.0, 0.0, 0.0]; for (inner_text, inner_data) in &text_to_embedding { if text.contains(inner_text) { @@ -51,8 +52,12 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va }) .mount(&mock_server) .await; - let url = mock_server.uri(); + mock_server.uri() +} + +pub async fn init_fragments_index() -> (Server, String, crate::common::Value) { + let url = fragment_mock_server().await; let server = Server::new().await; let index = server.unique_index(); @@ -104,6 +109,72 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va (server, uid, settings) } +pub async fn init_fragments_index_composite() -> (Server, String, crate::common::Value) { + let url = fragment_mock_server().await; + let server = Server::new().await; + let index = server.unique_index(); + + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + + let (_response, code) = server.set_features(json!({"compositeEmbedders": true})).await; + snapshot!(code, @"200 OK"); + + // Configure the index to use our mock embedder + let settings = json!({ + "embedders": { + "rest": { + "source": "composite", + "searchEmbedder": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "searchFragments": { + "query": {"value": "Some pre-prompt for query {{ q }}"}, + } + }, + "indexingEmbedder": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + } + }, + }, + }, + }); + let (response, code) = index.update_settings(settings.clone()).await; + println!("Update settings response: {:?}", response); + snapshot!(code, @"202 Accepted"); + + server.wait_task(response.uid()).await.succeeded(); + + // Send documents + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel", "breed": "labrador"}, + {"id": 3, "name": "dustin", "breed": "bulldog"}, + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + + index.wait_task(value.uid()).await.succeeded(); + + let uid = index.uid.clone(); + (server, uid, settings) +} + #[actix_rt::test] async fn experimental_feature_not_enabled() { let server = Server::new().await; @@ -247,8 +318,7 @@ async fn replace_document() { let (value, code) = index.add_documents(documents, None).await; snapshot!(code, @"202 Accepted"); - let task = index.wait_task(value.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); + index.wait_task(value.uid()).await.succeeded(); // Make sure kefir now has 2 vectors let (documents, code) = index @@ -2244,3 +2314,30 @@ async fn set_fragments_then_document_template() { } "#); } + +#[actix_rt::test] +async fn composite() { + let (server, uid, _settings) = init_fragments_index_composite().await; + let index = server.index(uid); + + let (value, code) = index.search_post( + json!({"vector": [1.0, 1.0, 1.0], "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, "limit": 1} + )).await; + snapshot!(code, @"200 OK"); + snapshot!(value, @r#" + { + "hits": [ + { + "id": 1, + "name": "echo" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 1, + "offset": 0, + "estimatedTotalHits": 4, + "semanticHitCount": 1 + } + "#); +} diff --git a/crates/milli/src/vector/composite.rs b/crates/milli/src/vector/composite.rs index 130e2674a..2e31da094 100644 --- a/crates/milli/src/vector/composite.rs +++ b/crates/milli/src/vector/composite.rs @@ -85,7 +85,7 @@ impl Embedder { index.dimensions(), )); } - + // check similarity if skip_similarity_check { return Ok(Self { search, index }); From 9c60e9689f806442abd9e9ebe40f98ba17c65db9 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 7 Jul 2025 18:34:24 +0200 Subject: [PATCH 087/312] Support not specifying an embedder in the vector filter --- .../milli/src/search/facet/filter_vector.rs | 95 +++++++++++-------- 1 file changed, 56 insertions(+), 39 deletions(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 701ab561c..ee1c19923 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -6,7 +6,7 @@ use crate::vector::{ArroyStats, ArroyWrapper}; use crate::{Index, Result}; pub(super) struct VectorFilter<'a> { - embedder_name: &'a str, + embedder_name: Option<&'a str>, fragment_name: Option<&'a str>, user_provided: bool, // TODO: not_user_provided: bool, @@ -14,12 +14,14 @@ pub(super) struct VectorFilter<'a> { impl<'a> VectorFilter<'a> { pub(super) fn matches(value: &str, op: &Condition) -> bool { - matches!(op, Condition::Exists) && value.starts_with("_vectors.") + matches!(op, Condition::Exists) && (value.starts_with("_vectors.") || value == "_vectors") } /// Parses a vector filter string. /// /// Valid formats: + /// - `_vectors` + /// - `_vectors.userProvided` /// - `_vectors.{embedder_name}` /// - `_vectors.{embedder_name}.userProvided` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` @@ -33,11 +35,7 @@ impl<'a> VectorFilter<'a> { )))); } - let embedder_name = split.next().ok_or_else(|| { - Error::UserError(UserError::InvalidFilter(String::from( - "Vector filter must contain an embedder name", - ))) - })?; + let embedder_name = split.next(); let mut fragment_name = None; if split.peek() == Some(&"fragments") { @@ -74,44 +72,63 @@ impl<'a> VectorFilter<'a> { let index_embedding_configs = index.embedding_configs(); let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; - let Some(embedder_config) = - embedding_configs.iter().find(|config| config.name == self.embedder_name) - else { - return Ok(RoaringBitmap::new()); - }; - let Some(embedder_info) = - index_embedding_configs.embedder_info(rtxn, self.embedder_name)? - else { - return Ok(RoaringBitmap::new()); - }; - - let arroy_wrapper = ArroyWrapper::new( - index.vector_arroy, - embedder_info.embedder_id, - embedder_config.config.quantized(), - ); - - let mut docids = if let Some(fragment_name) = self.fragment_name { - let Some(fragment_config) = embedder_config - .fragments - .as_slice() - .iter() - .find(|fragment| fragment.name == fragment_name) + let mut embedders = Vec::new(); + if let Some(embedder_name) = self.embedder_name { + let Some(embedder_config) = + embedding_configs.iter().find(|config| config.name == embedder_name) else { return Ok(RoaringBitmap::new()); }; - - arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + let Some(embedder_info) = + index_embedding_configs.embedder_info(rtxn, embedder_name)? + else { + return Ok(RoaringBitmap::new()); + }; + + embedders.push((embedder_config, embedder_info)); } else { - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents + for embedder_config in embedding_configs.iter() { + let Some(embedder_info) = + index_embedding_configs.embedder_info(rtxn, &embedder_config.name)? + else { + continue; + }; + embedders.push((embedder_config, embedder_info)); + } }; + + let mut docids = RoaringBitmap::new(); + for (embedder_config, embedder_info) in embedders { + let arroy_wrapper = ArroyWrapper::new( + index.vector_arroy, + embedder_info.embedder_id, + embedder_config.config.quantized(), + ); - // FIXME: performance - if self.user_provided { - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - docids &= user_provided_docsids; + let mut new_docids = if let Some(fragment_name) = self.fragment_name { + let Some(fragment_config) = embedder_config + .fragments + .as_slice() + .iter() + .find(|fragment| fragment.name == fragment_name) + else { + return Ok(RoaringBitmap::new()); + }; + + arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + } else { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents + }; + + // FIXME: performance + if self.user_provided { + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + new_docids &= user_provided_docsids; + } + + docids |= new_docids; } if let Some(universe) = universe { From 5cced0af02018067d78f85b1348ede9b2672eb86 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 7 Jul 2025 18:41:03 +0200 Subject: [PATCH 088/312] Prevent having both a fragment name and userProvided --- crates/milli/src/search/facet/filter_vector.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index ee1c19923..9a0a50124 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -25,7 +25,6 @@ impl<'a> VectorFilter<'a> { /// - `_vectors.{embedder_name}` /// - `_vectors.{embedder_name}.userProvided` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` - /// - `_vectors.{embedder_name}.fragments.{fragment_name}.userProvided` pub(super) fn parse(s: &'a str) -> Result { let mut split = s.split('.').peekable(); @@ -54,6 +53,12 @@ impl<'a> VectorFilter<'a> { user_provided = true; } + if fragment_name.is_some() && user_provided { + return Err(Error::UserError(UserError::InvalidFilter( + String::from("Vector filter cannot specify both a fragment name and userProvided"), + ))); + } + if let Some(next) = split.next() { return Err(Error::UserError(UserError::InvalidFilter(format!( "Unexpected part in vector filter: '{next}'" From 40e7284d70865600e7832a10d357bad9ae68cec5 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 10:01:35 +0200 Subject: [PATCH 089/312] Add tests --- crates/meilisearch/tests/search/filters.rs | 162 +++++++++++++++++++ crates/meilisearch/tests/vector/fragments.rs | 2 +- crates/meilisearch/tests/vector/mod.rs | 1 + 3 files changed, 164 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index ffa025f5c..361762b6c 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -731,3 +731,165 @@ async fn test_filterable_attributes_priority() { ) .await; } + +#[actix_rt::test] +async fn test_vector_filter() { + let index = crate::vector::shared_index_for_fragments().await; + + let (value, _code) = index.search_post(json!({ + "filter": "_vectors EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [ + { + "id": 0 + }, + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 4 + } + "#); + + let (value, _code) = index.search_post(json!({ + "filter": "_vectors.other EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0 + } + "#); + + // This one is counterintuitive, but it is the same as the previous one. + // It's because userProvided is interpreted as an embedder name + let (value, _code) = index.search_post(json!({ + "filter": "_vectors.userProvided EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0 + } + "#); + + let (value, _code) = index.search_post(json!({ + "filter": "_vectors.rest EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [ + { + "id": 0 + }, + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 4 + } + "#); + + let (value, _code) = index.search_post(json!({ + "filter": "_vectors.rest.userProvided EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [ + { + "id": 1 + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 1 + } + "#); + + let (value, _code) = index.search_post(json!({ + "filter": "_vectors.rest.fragments.withBreed EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [ + { + "id": 2 + }, + { + "id": 3 + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 2 + } + "#); + + let (value, _code) = index.search_post(json!({ + "filter": "_vectors.rest.fragments.basic EXISTS", + "attributesToRetrieve": ["id"] + })).await; + snapshot!(value, @r#" + { + "hits": [ + { + "id": 0 + }, + { + "id": 1 + }, + { + "id": 2 + }, + { + "id": 3 + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 4 + } + "#); +} diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 2626284a0..3ce452c1f 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -10,7 +10,7 @@ use crate::common::{Owned, Shared}; use crate::json; use crate::vector::{GetAllDocumentsOptions, Server}; -async fn shared_index_for_fragments() -> Index<'static, Shared> { +pub async fn shared_index_for_fragments() -> Index<'static, Shared> { static INDEX: OnceCell<(Server, String)> = OnceCell::const_new(); let (server, uid) = INDEX .get_or_init(|| async { diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 7f54489b6..9ba37cae3 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -14,6 +14,7 @@ use meilisearch::option::MaxThreads; use crate::common::index::Index; use crate::common::{default_settings, GetAllDocumentsOptions, Server}; use crate::json; +pub use fragments::shared_index_for_fragments; async fn get_server_vector() -> Server { Server::new().await From 2d45124d9b379fd73dba54e7d56d4f067eeb477c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 10:01:50 +0200 Subject: [PATCH 090/312] Fix parsing --- crates/milli/src/search/facet/filter.rs | 9 +++++++-- crates/milli/src/search/facet/filter_vector.rs | 6 ++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index f80d1681f..c9728966a 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -241,7 +241,7 @@ impl<'a> Filter<'a> { let attribute = fid.value(); if matching_features(attribute, &filterable_attributes_rules) - .is_some_and(|(_, features)| features.is_filterable()) + .is_some_and(|(_, features)| features.is_filterable()) || VectorFilter::matches(attribute) { continue; } @@ -546,7 +546,12 @@ impl<'a> Filter<'a> { } FilterCondition::Condition { fid, op } => { let value = fid.value(); - if VectorFilter::matches(value, op) { + if VectorFilter::matches(value) { + if !matches!(op, Condition::Exists) { + return Err(Error::UserError(UserError::InvalidFilter( + String::from("Vector filter can only be used with the `exists` operator"), + ))); + } let vector_filter = VectorFilter::parse(value)?; return vector_filter.evaluate(rtxn, index, universe); } diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 9a0a50124..473741f14 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -1,4 +1,3 @@ -use filter_parser::Condition; use roaring::RoaringBitmap; use crate::error::{Error, UserError}; @@ -13,15 +12,14 @@ pub(super) struct VectorFilter<'a> { } impl<'a> VectorFilter<'a> { - pub(super) fn matches(value: &str, op: &Condition) -> bool { - matches!(op, Condition::Exists) && (value.starts_with("_vectors.") || value == "_vectors") + pub(super) fn matches(value: &str) -> bool { + value.starts_with("_vectors.") || value == "_vectors" } /// Parses a vector filter string. /// /// Valid formats: /// - `_vectors` - /// - `_vectors.userProvided` /// - `_vectors.{embedder_name}` /// - `_vectors.{embedder_name}.userProvided` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` From 4623691d1fd3e40f3f47f0633798f321d7dc4331 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 8 Jul 2025 10:02:25 +0200 Subject: [PATCH 091/312] Don't make the type-that-shall-not-be-written serializable Following tamo's advice Co-Authored-By: Tamo --- crates/milli/src/update/upgrade/v1_15.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/update/upgrade/v1_15.rs b/crates/milli/src/update/upgrade/v1_15.rs index 9ca25d06b..3457e69ba 100644 --- a/crates/milli/src/update/upgrade/v1_15.rs +++ b/crates/milli/src/update/upgrade/v1_15.rs @@ -1,6 +1,6 @@ use heed::RwTxn; use roaring::RoaringBitmap; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use super::UpgradeIndex; use crate::progress::Progress; @@ -34,7 +34,7 @@ impl UpgradeIndex for Latest_V1_14_To_Latest_V1_15 { /// # Warning /// /// This object should not be rewritten to the DB, only read to get the name and `user_provided` roaring. -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize)] pub struct IndexEmbeddingConfig { pub name: String, pub user_provided: RoaringBitmap, From 0301d8f239f9d860c663bec9b48b4b08477e7885 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 11:39:10 +0200 Subject: [PATCH 092/312] Improve error handling --- crates/filter-parser/src/lib.rs | 113 +++++++++- crates/meilisearch/tests/search/filters.rs | 109 ++++++---- crates/milli/src/search/facet/filter.rs | 11 +- .../milli/src/search/facet/filter_vector.rs | 196 ++++++++++++++---- 4 files changed, 342 insertions(+), 87 deletions(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 938702103..b64477170 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -60,7 +60,7 @@ use nom::combinator::{cut, eof, map, opt}; use nom::multi::{many0, separated_list1}; use nom::number::complete::recognize_float; use nom::sequence::{delimited, preceded, terminated, tuple}; -use nom::Finish; +use nom::{Finish, Slice}; use nom_locate::LocatedSpan; pub(crate) use value::parse_value; use value::word_exact; @@ -121,6 +121,16 @@ impl<'a> Token<'a> { Err(Error::new_from_kind(self.span, ErrorKind::NonFiniteFloat)) } } + + /// Split the token by a delimiter and return an iterator of tokens. + /// Each token in the iterator will have its own span that corresponds to a slice of the original token's span. + pub fn split(&self, delimiter: &'a str) -> impl Iterator> + '_ { + let original_addr = self.value().as_ptr() as usize; + self.value().split(delimiter).map(move |part| { + let offset = part.as_ptr() as usize - original_addr; + Token::new(self.span.slice(offset..offset + part.len()), Some(part.to_string())) + }) + } } impl<'a> From> for Token<'a> { @@ -604,6 +614,8 @@ impl std::fmt::Display for Token<'_> { #[cfg(test)] pub mod tests { + use std::fmt::format; + use FilterCondition as Fc; use super::*; @@ -1043,4 +1055,103 @@ pub mod tests { let token: Token = s.into(); assert_eq!(token.value(), s); } + + #[test] + fn split() { + let s = "test string that should not be parsed\n newline"; + let token: Token = s.into(); + let parts: Vec<_> = token.split(" ").collect(); + insta::assert_snapshot!(format!("{parts:#?}"), @r#" + [ + Token { + span: LocatedSpan { + offset: 0, + line: 1, + fragment: "test", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "test", + ), + }, + Token { + span: LocatedSpan { + offset: 5, + line: 1, + fragment: "string", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "string", + ), + }, + Token { + span: LocatedSpan { + offset: 12, + line: 1, + fragment: "that", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "that", + ), + }, + Token { + span: LocatedSpan { + offset: 17, + line: 1, + fragment: "should", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "should", + ), + }, + Token { + span: LocatedSpan { + offset: 24, + line: 1, + fragment: "not", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "not", + ), + }, + Token { + span: LocatedSpan { + offset: 28, + line: 1, + fragment: "be", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "be", + ), + }, + Token { + span: LocatedSpan { + offset: 31, + line: 1, + fragment: "parsed\n", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "parsed\n", + ), + }, + Token { + span: LocatedSpan { + offset: 39, + line: 2, + fragment: "newline", + extra: "test string that should not be parsed\n newline", + }, + value: Some( + "newline", + ), + }, + ] + "#); + } } diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 361762b6c..384605ca6 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -736,10 +736,12 @@ async fn test_filterable_attributes_priority() { async fn test_vector_filter() { let index = crate::vector::shared_index_for_fragments().await; - let (value, _code) = index.search_post(json!({ - "filter": "_vectors EXISTS", - "attributesToRetrieve": ["id"] - })).await; + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { "hits": [ @@ -764,42 +766,44 @@ async fn test_vector_filter() { } "#); - let (value, _code) = index.search_post(json!({ - "filter": "_vectors.other EXISTS", - "attributesToRetrieve": ["id"] - })).await; + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.other EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { - "hits": [], - "query": "", - "processingTimeMs": "[duration]", - "limit": 20, - "offset": 0, - "estimatedTotalHits": 0 + "message": "Index `[uuid]`: The embedder `other` does not exist. Available embedders are: `rest`.\n10:15 _vectors.other EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" } "#); - + // This one is counterintuitive, but it is the same as the previous one. // It's because userProvided is interpreted as an embedder name - let (value, _code) = index.search_post(json!({ - "filter": "_vectors.userProvided EXISTS", - "attributesToRetrieve": ["id"] - })).await; + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.userProvided EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { - "hits": [], - "query": "", - "processingTimeMs": "[duration]", - "limit": 20, - "offset": 0, - "estimatedTotalHits": 0 + "message": "Index `[uuid]`: The embedder `userProvided` does not exist. Available embedders are: `rest`.\n10:22 _vectors.userProvided EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" } "#); - let (value, _code) = index.search_post(json!({ - "filter": "_vectors.rest EXISTS", - "attributesToRetrieve": ["id"] - })).await; + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { "hits": [ @@ -824,10 +828,12 @@ async fn test_vector_filter() { } "#); - let (value, _code) = index.search_post(json!({ - "filter": "_vectors.rest.userProvided EXISTS", - "attributesToRetrieve": ["id"] - })).await; + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.userProvided EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { "hits": [ @@ -843,10 +849,12 @@ async fn test_vector_filter() { } "#); - let (value, _code) = index.search_post(json!({ - "filter": "_vectors.rest.fragments.withBreed EXISTS", - "attributesToRetrieve": ["id"] - })).await; + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.fragments.withBreed EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { "hits": [ @@ -864,11 +872,13 @@ async fn test_vector_filter() { "estimatedTotalHits": 2 } "#); - - let (value, _code) = index.search_post(json!({ - "filter": "_vectors.rest.fragments.basic EXISTS", - "attributesToRetrieve": ["id"] - })).await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.fragments.basic EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; snapshot!(value, @r#" { "hits": [ @@ -892,4 +902,19 @@ async fn test_vector_filter() { "estimatedTotalHits": 4 } "#); + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.fragments.other EXISTS", + "attributesToRetrieve": ["id"] + })) + .await; + snapshot!(value, @r#" + { + "message": "Index `[uuid]`: The fragment `other` does not exist on embedder `rest`. Available fragments on this embedder are: `basic`, `withBreed`.\n25:30 _vectors.rest.fragments.other EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" + } + "#); } diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index c9728966a..c0419997c 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -241,7 +241,8 @@ impl<'a> Filter<'a> { let attribute = fid.value(); if matching_features(attribute, &filterable_attributes_rules) - .is_some_and(|(_, features)| features.is_filterable()) || VectorFilter::matches(attribute) + .is_some_and(|(_, features)| features.is_filterable()) + || VectorFilter::matches(attribute) { continue; } @@ -548,11 +549,11 @@ impl<'a> Filter<'a> { let value = fid.value(); if VectorFilter::matches(value) { if !matches!(op, Condition::Exists) { - return Err(Error::UserError(UserError::InvalidFilter( - String::from("Vector filter can only be used with the `exists` operator"), - ))); + return Err(Error::UserError(UserError::InvalidFilter(String::from( + "Vector filter can only be used with the `exists` operator", + )))); } - let vector_filter = VectorFilter::parse(value)?; + let vector_filter = VectorFilter::parse(fid)?; return vector_filter.evaluate(rtxn, index, universe); } diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 473741f14..0a7ac313a 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -1,16 +1,118 @@ +use filter_parser::Token; use roaring::RoaringBitmap; use crate::error::{Error, UserError}; use crate::vector::{ArroyStats, ArroyWrapper}; -use crate::{Index, Result}; +use crate::Index; pub(super) struct VectorFilter<'a> { - embedder_name: Option<&'a str>, - fragment_name: Option<&'a str>, + embedder_token: Option>, + fragment_token: Option>, user_provided: bool, // TODO: not_user_provided: bool, } +#[derive(Debug)] +pub enum VectorFilterError<'a> { + EmptyFilter, + InvalidPrefix(Token<'a>), + MissingFragmentName(Token<'a>), + UserProvidedWithFragment(Token<'a>), + LeftoverToken(Token<'a>), + EmbedderDoesNotExist { + embedder: &'a Token<'a>, + available: Vec, + }, + FragmentDoesNotExist { + embedder: &'a Token<'a>, + fragment: &'a Token<'a>, + available: Vec, + }, +} + +use VectorFilterError::*; + +impl std::error::Error for VectorFilterError<'_> {} + +impl std::fmt::Display for VectorFilterError<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + EmptyFilter => { + write!(f, "Vector filter cannot be empty.") + } + InvalidPrefix(prefix) => { + write!( + f, + "Vector filter must start with `_vectors` but found `{}`.", + prefix.value() + ) + } + MissingFragmentName(_token) => { + write!(f, "Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.") + } + UserProvidedWithFragment(_token) => { + write!(f, "Vector filter cannot specify both a fragment name and userProvided.") + } + LeftoverToken(token) => { + write!(f, "Vector filter has leftover token: `{}`.", token.value()) + } + EmbedderDoesNotExist { embedder, available } => { + write!(f, "The embedder `{}` does not exist.", embedder.value())?; + if available.is_empty() { + write!(f, " This index does not have configured embedders.") + } else { + write!(f, " Available embedders are: ")?; + let mut available = available.clone(); + available.sort_unstable(); + for (idx, embedder) in available.iter().enumerate() { + write!(f, "`{embedder}`")?; + if idx != available.len() - 1 { + write!(f, ", ")?; + } + } + write!(f, ".") + } + } + FragmentDoesNotExist { embedder, fragment, available } => { + write!( + f, + "The fragment `{}` does not exist on embedder `{}`.", + fragment.value(), + embedder.value(), + )?; + if available.is_empty() { + write!(f, " This embedder does not have configured fragments.") + } else { + write!(f, " Available fragments on this embedder are: ")?; + let mut available = available.clone(); + available.sort_unstable(); + for (idx, fragment) in available.iter().enumerate() { + write!(f, "`{fragment}`")?; + if idx != available.len() - 1 { + write!(f, ", ")?; + } + } + write!(f, ".") + } + } + } + } +} + +impl<'a> From> for Error { + fn from(err: VectorFilterError<'a>) -> Self { + match &err { + EmptyFilter => Error::UserError(UserError::InvalidFilter(err.to_string())), + InvalidPrefix(token) + | MissingFragmentName(token) + | UserProvidedWithFragment(token) + | LeftoverToken(token) => token.clone().as_external_error(err).into(), + EmbedderDoesNotExist { embedder: token, .. } + | FragmentDoesNotExist { fragment: token, .. } => token.as_external_error(err).into(), + } + } +} + impl<'a> VectorFilter<'a> { pub(super) fn matches(value: &str) -> bool { value.starts_with("_vectors.") || value == "_vectors" @@ -23,71 +125,74 @@ impl<'a> VectorFilter<'a> { /// - `_vectors.{embedder_name}` /// - `_vectors.{embedder_name}.userProvided` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` - pub(super) fn parse(s: &'a str) -> Result { - let mut split = s.split('.').peekable(); + pub(super) fn parse(s: &'a Token<'a>) -> Result> { + let mut split = s.split(".").peekable(); - if split.next() != Some("_vectors") { - return Err(Error::UserError(UserError::InvalidFilter(String::from( - "Vector filter must start with '_vectors'", - )))); + match split.next() { + Some(token) if token.value() == "_vectors" => (), + Some(token) => return Err(InvalidPrefix(token)), + None => return Err(EmptyFilter), } let embedder_name = split.next(); let mut fragment_name = None; - if split.peek() == Some(&"fragments") { - split.next(); + if split.peek().map(|t| t.value()) == Some("fragments") { + let token = split.next().expect("it was peeked before"); - fragment_name = Some(split.next().ok_or_else(|| { - Error::UserError(UserError::InvalidFilter( - String::from("Vector filter is inconsistent: either specify a fragment name or remove the 'fragments' part"), - )) - })?); + fragment_name = Some(split.next().ok_or(MissingFragmentName(token))?); } - let mut user_provided = false; - if split.peek() == Some(&"userProvided") || split.peek() == Some(&"user_provided") { - split.next(); - user_provided = true; + let mut user_provided_token = None; + if split.peek().map(|t| t.value()) == Some("userProvided") + || split.peek().map(|t| t.value()) == Some("user_provided") + { + user_provided_token = split.next(); } - if fragment_name.is_some() && user_provided { - return Err(Error::UserError(UserError::InvalidFilter( - String::from("Vector filter cannot specify both a fragment name and userProvided"), - ))); + if let (Some(_), Some(user_provided_token)) = (&fragment_name, &user_provided_token) { + return Err(UserProvidedWithFragment(user_provided_token.clone()))?; } if let Some(next) = split.next() { - return Err(Error::UserError(UserError::InvalidFilter(format!( - "Unexpected part in vector filter: '{next}'" - )))); + return Err(LeftoverToken(next))?; } - Ok(Self { embedder_name, fragment_name, user_provided }) + Ok(Self { + embedder_token: embedder_name, + fragment_token: fragment_name, + user_provided: user_provided_token.is_some(), + }) } pub(super) fn evaluate( - &self, + self, rtxn: &heed::RoTxn<'_>, index: &Index, universe: Option<&RoaringBitmap>, - ) -> Result { + ) -> crate::Result { let index_embedding_configs = index.embedding_configs(); let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; let mut embedders = Vec::new(); - if let Some(embedder_name) = self.embedder_name { + if let Some(embedder_token) = &self.embedder_token { + let embedder_name = embedder_token.value(); let Some(embedder_config) = embedding_configs.iter().find(|config| config.name == embedder_name) else { - return Ok(RoaringBitmap::new()); + return Err(EmbedderDoesNotExist { + embedder: embedder_token, + available: embedding_configs.iter().map(|c| c.name.clone()).collect(), + })?; }; - let Some(embedder_info) = - index_embedding_configs.embedder_info(rtxn, embedder_name)? + let Some(embedder_info) = index_embedding_configs.embedder_info(rtxn, embedder_name)? else { - return Ok(RoaringBitmap::new()); + return Err(EmbedderDoesNotExist { + embedder: embedder_token, + available: embedding_configs.iter().map(|c| c.name.clone()).collect(), + })?; }; - + embedders.push((embedder_config, embedder_info)); } else { for embedder_config in embedding_configs.iter() { @@ -99,7 +204,7 @@ impl<'a> VectorFilter<'a> { embedders.push((embedder_config, embedder_info)); } }; - + let mut docids = RoaringBitmap::new(); for (embedder_config, embedder_info) in embedders { let arroy_wrapper = ArroyWrapper::new( @@ -108,14 +213,27 @@ impl<'a> VectorFilter<'a> { embedder_config.config.quantized(), ); - let mut new_docids = if let Some(fragment_name) = self.fragment_name { + let mut new_docids = if let Some(fragment_token) = &self.fragment_token { + let fragment_name = fragment_token.value(); let Some(fragment_config) = embedder_config .fragments .as_slice() .iter() .find(|fragment| fragment.name == fragment_name) else { - return Ok(RoaringBitmap::new()); + return Err(FragmentDoesNotExist { + embedder: self + .embedder_token + .as_ref() + .expect("there can't be a fragment without an embedder"), + fragment: fragment_token, + available: embedder_config + .fragments + .as_slice() + .iter() + .map(|f| f.name.clone()) + .collect(), + })?; }; arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? From d43cd40807f87624bb4a7c87c44830b01f887458 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 11:48:23 +0200 Subject: [PATCH 093/312] Split tests --- crates/meilisearch/tests/search/filters.rs | 98 ++++++++++++++++------ 1 file changed, 74 insertions(+), 24 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 384605ca6..1c49fa5e0 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -733,29 +733,29 @@ async fn test_filterable_attributes_priority() { } #[actix_rt::test] -async fn test_vector_filter() { +async fn vector_filter_all_embedders() { let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ "filter": "_vectors EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" { "hits": [ { - "id": 0 + "name": "kefir" }, { - "id": 1 + "name": "echo" }, { - "id": 2 + "name": "intel" }, { - "id": 3 + "name": "dustin" } ], "query": "", @@ -765,11 +765,16 @@ async fn test_vector_filter() { "estimatedTotalHits": 4 } "#); +} + +#[actix_rt::test] +async fn vector_filter_non_existant_embedder() { + let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ "filter": "_vectors.other EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" @@ -780,13 +785,18 @@ async fn test_vector_filter() { "link": "https://docs.meilisearch.com/errors#invalid_search_filter" } "#); +} + +#[actix_rt::test] +async fn vector_filter_all_embedders_user_provided() { + let index = crate::vector::shared_index_for_fragments().await; // This one is counterintuitive, but it is the same as the previous one. // It's because userProvided is interpreted as an embedder name let (value, _code) = index .search_post(json!({ "filter": "_vectors.userProvided EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" @@ -797,27 +807,32 @@ async fn test_vector_filter() { "link": "https://docs.meilisearch.com/errors#invalid_search_filter" } "#); +} + +#[actix_rt::test] +async fn vector_filter_specific_embedder() { + let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ "filter": "_vectors.rest EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" { "hits": [ { - "id": 0 + "name": "kefir" }, { - "id": 1 + "name": "echo" }, { - "id": 2 + "name": "intel" }, { - "id": 3 + "name": "dustin" } ], "query": "", @@ -827,18 +842,23 @@ async fn test_vector_filter() { "estimatedTotalHits": 4 } "#); +} + +#[actix_rt::test] +async fn vector_filter_user_provided() { + let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ "filter": "_vectors.rest.userProvided EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" { "hits": [ { - "id": 1 + "name": "echo" } ], "query": "", @@ -848,21 +868,26 @@ async fn test_vector_filter() { "estimatedTotalHits": 1 } "#); +} + +#[actix_rt::test] +async fn vector_filter_specific_fragment() { + let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ "filter": "_vectors.rest.fragments.withBreed EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" { "hits": [ { - "id": 2 + "name": "intel" }, { - "id": 3 + "name": "dustin" } ], "query": "", @@ -876,23 +901,23 @@ async fn test_vector_filter() { let (value, _code) = index .search_post(json!({ "filter": "_vectors.rest.fragments.basic EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" { "hits": [ { - "id": 0 + "name": "kefir" }, { - "id": 1 + "name": "echo" }, { - "id": 2 + "name": "intel" }, { - "id": 3 + "name": "dustin" } ], "query": "", @@ -902,11 +927,16 @@ async fn test_vector_filter() { "estimatedTotalHits": 4 } "#); +} + +#[actix_rt::test] +async fn vector_filter_non_existant_fragment() { + let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ "filter": "_vectors.rest.fragments.other EXISTS", - "attributesToRetrieve": ["id"] + "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" @@ -918,3 +948,23 @@ async fn test_vector_filter() { } "#); } + +#[actix_rt::test] +async fn vector_filter_specific_fragment_user_provided() { + let index = crate::vector::shared_index_for_fragments().await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.fragments.other.userProvided EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "message": "Index `[uuid]`: Vector filter cannot specify both a fragment name and userProvided.\n31:43 _vectors.rest.fragments.other.userProvided EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" + } + "#); +} From b4cafec8b3941b2add4fe4ce99047b1b28da18bf Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 11:56:19 +0200 Subject: [PATCH 094/312] Add tests for operators along vector filter --- crates/meilisearch/tests/search/filters.rs | 64 ++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 1c49fa5e0..2e71b5435 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -968,3 +968,67 @@ async fn vector_filter_specific_fragment_user_provided() { } "#); } + +#[actix_rt::test] +async fn vector_filter_negation() { + let index = crate::vector::shared_index_for_fragments().await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.userProvided NOT EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "hits": [ + { + "name": "kefir" + }, + { + "name": "intel" + }, + { + "name": "dustin" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3 + } + "#); +} + +#[actix_rt::test] +async fn vector_filter_or_combination() { +let index = crate::vector::shared_index_for_fragments().await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.fragments.withBreed EXISTS OR _vectors.rest.userProvided EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "hits": [ + { + "name": "echo" + }, + { + "name": "intel" + }, + { + "name": "dustin" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3 + } + "#); +} From 29b74424ad6313ea673869a658e10883fca7e4b1 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 12:03:32 +0200 Subject: [PATCH 095/312] Clean code --- crates/meilisearch/tests/search/filters.rs | 2 +- crates/milli/src/index.rs | 2 +- crates/milli/src/search/facet/filter.rs | 4 +--- crates/milli/src/search/facet/filter_vector.rs | 1 - crates/milli/src/update/index_documents/transform.rs | 2 +- 5 files changed, 4 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 2e71b5435..3cc7cbab5 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -1003,7 +1003,7 @@ async fn vector_filter_negation() { #[actix_rt::test] async fn vector_filter_or_combination() { -let index = crate::vector::shared_index_for_fragments().await; + let index = crate::vector::shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index 2751498bf..b2ec992ba 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -1776,7 +1776,7 @@ impl Index { embedder_info.embedder_id, config.config.quantized(), ); - let embeddings = reader.item_vectors(rtxn, docid)?; // MARKER + let embeddings = reader.item_vectors(rtxn, docid)?; res.insert( config.name.to_owned(), (embeddings, embedder_info.embedding_status.must_regenerate(docid)), diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index c0419997c..1afdf87e6 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -234,11 +234,9 @@ impl<'a> Filter<'a> { pub fn evaluate(&self, rtxn: &heed::RoTxn<'_>, index: &Index) -> Result { // to avoid doing this for each recursive call we're going to do it ONCE ahead of time let fields_ids_map = index.fields_ids_map(rtxn)?; - let filterable_attributes_rules = dbg!(index.filterable_attributes_rules(rtxn)?); + let filterable_attributes_rules = index.filterable_attributes_rules(rtxn)?; for fid in self.condition.fids(MAX_FILTER_DEPTH) { - println!("{fid:?}"); - let attribute = fid.value(); if matching_features(attribute, &filterable_attributes_rules) .is_some_and(|(_, features)| features.is_filterable()) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 0a7ac313a..79e35366c 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -9,7 +9,6 @@ pub(super) struct VectorFilter<'a> { embedder_token: Option>, fragment_token: Option>, user_provided: bool, - // TODO: not_user_provided: bool, } #[derive(Debug)] diff --git a/crates/milli/src/update/index_documents/transform.rs b/crates/milli/src/update/index_documents/transform.rs index d69768d4b..e07483aff 100644 --- a/crates/milli/src/update/index_documents/transform.rs +++ b/crates/milli/src/update/index_documents/transform.rs @@ -966,7 +966,7 @@ impl<'a, 'i> Transform<'a, 'i> { // some user provided, remove only the ids that are not user provided let to_delete = arroy.items_in_store(wtxn, *fragment_id, |items| { items - infos.embedding_status.user_provided_docids() - })?; // MARKER + })?; for to_delete in to_delete { arroy.del_item_in_store(wtxn, to_delete, *fragment_id, dimensions)?; From fb73b83abe67532e496b7ac2f56aa5594ec81bdd Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 12:14:34 +0200 Subject: [PATCH 096/312] Fix performance --- .../milli/src/search/facet/filter_vector.rs | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 79e35366c..a0dc52bac 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -235,19 +235,33 @@ impl<'a> VectorFilter<'a> { })?; }; - arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + if let Some(universe) = universe { + arroy_wrapper + .items_in_store(rtxn, fragment_config.id, |bitmap| bitmap & universe)? + } else { + arroy_wrapper + .items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + } } else { + let mut universe = universe.cloned(); + if self.user_provided { + let user_provided_docsids = + embedder_info.embedding_status.user_provided_docids(); + match &mut universe { + Some(universe) => *universe &= user_provided_docsids, + None => universe = Some(user_provided_docsids.clone()), + } + } + let mut stats = ArroyStats::default(); arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents + if let Some(universe) = &universe { + stats.documents & universe + } else { + stats.documents + } }; - // FIXME: performance - if self.user_provided { - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - new_docids &= user_provided_docsids; - } - docids |= new_docids; } From a56c0369947760bf5980d4dfac97bec1951c0781 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 12:18:52 +0200 Subject: [PATCH 097/312] Update crates/meilisearch-types/src/keys.rs Co-authored-by: gui machiavelli --- crates/meilisearch-types/src/keys.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 2911f22a2..b98f2d38d 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -149,7 +149,7 @@ impl Key { let uid = Uuid::new_v4(); Self { name: Some("Default Read-Only Admin API Key".to_string()), - description: Some("Use it to peek into the instance in a read-only mode. Caution: This key gives you access to all the other api keys. Do not expose it on a public frontend".to_string()), + description: Some("Use it to read information across the whole database. Caution! Do not expose this key on a public frontend".to_string()), uid, actions: vec![Action::AllGet, Action::KeysGet], indexes: vec![IndexUidPattern::all()], From 9cee432255f1a66b6b16c6b46bea243902eade2d Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 13:36:26 +0200 Subject: [PATCH 098/312] Fix broken tests --- crates/meilisearch-types/src/keys.rs | 1 + crates/meilisearch/tests/auth/api_keys.rs | 4 ++-- crates/meilisearch/tests/auth/errors.rs | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index c0ec5ae0b..e210f8df3 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -431,6 +431,7 @@ impl Action { DocumentsAdd => false, DocumentsGet => true, DocumentsDelete => false, + Export => true, IndexesAdd => false, IndexesGet => true, IndexesUpdate => false, diff --git a/crates/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs index f717fd53e..0b8a3d2c5 100644 --- a/crates/meilisearch/tests/auth/api_keys.rs +++ b/crates/meilisearch/tests/auth/api_keys.rs @@ -421,7 +421,7 @@ async fn error_add_api_key_invalid_parameters_actions() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r#" { - "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `*.get`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", + "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `*.get`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" @@ -852,7 +852,7 @@ async fn list_api_keys() { }, { "name": "Default Read-Only Admin API Key", - "description": "Use it to peek into the instance in a read-only mode. Caution: This key gives you access to all the other api keys. Do not expose it on a public frontend", + "description": "Use it to read information across the whole database. Caution! Do not expose this key on a public frontend", "key": "[ignored]", "uid": "[ignored]", "actions": [ diff --git a/crates/meilisearch/tests/auth/errors.rs b/crates/meilisearch/tests/auth/errors.rs index 845fe7085..e8d935fde 100644 --- a/crates/meilisearch/tests/auth/errors.rs +++ b/crates/meilisearch/tests/auth/errors.rs @@ -93,7 +93,7 @@ async fn create_api_key_bad_actions() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r#" { - "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", + "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `*.get`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" From 2f1be0ff863fd4e272b6df30809ca8559e1bb12f Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 13:55:07 +0200 Subject: [PATCH 099/312] Ignore faulty test (see #5746) --- crates/meilisearch/tests/vector/fragments.rs | 83 +------------------- 1 file changed, 2 insertions(+), 81 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 2626284a0..345155034 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -2203,6 +2203,7 @@ async fn both_fragments_and_document_template() { "#); } +#[ignore = "failing due to issue #5746"] #[actix_rt::test] async fn set_fragments_then_document_template() { let (server, uid, settings) = init_fragments_index().await; @@ -2232,87 +2233,7 @@ async fn set_fragments_then_document_template() { let (settings, code) = index.settings().await; snapshot!(code, @"200 OK"); - snapshot!(settings, @r#" - { - "displayedAttributes": [ - "*" - ], - "searchableAttributes": [ - "*" - ], - "filterableAttributes": [], - "sortableAttributes": [], - "rankingRules": [ - "words", - "typo", - "proximity", - "attribute", - "sort", - "exactness" - ], - "stopWords": [], - "nonSeparatorTokens": [], - "separatorTokens": [], - "dictionary": [], - "synonyms": {}, - "distinctAttribute": null, - "proximityPrecision": "byWord", - "typoTolerance": { - "enabled": true, - "minWordSizeForTypos": { - "oneTypo": 5, - "twoTypos": 9 - }, - "disableOnWords": [], - "disableOnAttributes": [], - "disableOnNumbers": false - }, - "faceting": { - "maxValuesPerFacet": 100, - "sortFacetValuesBy": { - "*": "alpha" - } - }, - "pagination": { - "maxTotalHits": 1000 - }, - "embedders": { - "rest": { - "source": "rest", - "dimensions": 3, - "url": "http://127.0.0.1:55578", - "indexingFragments": { - "basic": { - "value": "{{ doc.name }} is a dog" - }, - "withBreed": { - "value": "{{ doc.name }} is a {{ doc.breed }}" - } - }, - "searchFragments": { - "justBreed": { - "value": "It's a {{ media.breed }}" - }, - "justName": { - "value": "{{ media.name }} is a dog" - }, - "query": { - "value": "Some pre-prompt for query {{ q }}" - } - }, - "request": "{{fragment}}", - "response": { - "data": "{{embedding}}" - }, - "headers": {} - } - }, - "searchCutoffMs": null, - "localizedAttributes": null, - "facetSearch": true, - "prefixSearch": "indexingTime" - } - "#); + snapshot!(settings, @r#""#); // Should have removed fragments } #[actix_rt::test] From 1ae47bec77c5d89652082fcc2b7b0436a33a2526 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 13:57:07 +0200 Subject: [PATCH 100/312] Improve composite test --- crates/meilisearch/tests/vector/fragments.rs | 22 ++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 345155034..b1fddedf8 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -2261,4 +2261,26 @@ async fn composite() { "semanticHitCount": 1 } "#); + + let (value, code) = index.search_post( + json!({"q": "bulldog", "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, "limit": 1} + )).await; + snapshot!(code, @"200 OK"); + snapshot!(value, @r#" + { + "hits": [ + { + "id": 3, + "name": "dustin", + "breed": "bulldog" + } + ], + "query": "bulldog", + "processingTimeMs": "[duration]", + "limit": 1, + "offset": 0, + "estimatedTotalHits": 4, + "semanticHitCount": 1 + } + "#); } From 3cc5d86598543ac27a7afdad647d23e0892dc7c1 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 13:57:17 +0200 Subject: [PATCH 101/312] Format --- crates/meilisearch/tests/vector/fragments.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index b1fddedf8..15f9f9887 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -2262,9 +2262,12 @@ async fn composite() { } "#); - let (value, code) = index.search_post( - json!({"q": "bulldog", "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, "limit": 1} - )).await; + let (value, code) = index + .search_post( + json!({"q": "bulldog", "hybrid": {"semanticRatio": 1.0, "embedder": "rest"}, "limit": 1} + ), + ) + .await; snapshot!(code, @"200 OK"); snapshot!(value, @r#" { From 0a4f2ef89138b40be98f7d238c0e08236b8df57e Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 15:27:35 +0200 Subject: [PATCH 102/312] Leak mock servers --- .../meilisearch/tests/search/multi/proxy.rs | 6 +-- crates/meilisearch/tests/vector/fragments.rs | 4 +- crates/meilisearch/tests/vector/openai.rs | 20 ++++----- crates/meilisearch/tests/vector/rest.rs | 42 +++++++++---------- 4 files changed, 36 insertions(+), 36 deletions(-) diff --git a/crates/meilisearch/tests/search/multi/proxy.rs b/crates/meilisearch/tests/search/multi/proxy.rs index 55736d058..c1d40ef3b 100644 --- a/crates/meilisearch/tests/search/multi/proxy.rs +++ b/crates/meilisearch/tests/search/multi/proxy.rs @@ -2499,7 +2499,7 @@ pub struct LocalMeiliParams { /// A server that exploits [`MockServer`] to provide an URL for testing network and the network. pub struct LocalMeili { - mock_server: MockServer, + mock_server: &'static MockServer, } impl LocalMeili { @@ -2508,7 +2508,7 @@ impl LocalMeili { } pub async fn with_params(server: Arc, params: LocalMeiliParams) -> Self { - let mock_server = MockServer::start().await; + let mock_server = Box::leak(Box::new(MockServer::start().await)); // tokio won't let us execute asynchronous code from a sync function inside of an async test, // so instead we spawn another thread that will call the service on a brand new tokio runtime @@ -2572,7 +2572,7 @@ impl LocalMeili { response.set_body_json(value) } }) - .mount(&mock_server) + .mount(mock_server) .await; Self { mock_server } } diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 15f9f9887..4fe2bddb6 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -33,7 +33,7 @@ async fn fragment_mock_server() -> String { .into_iter() .collect(); - let mock_server = MockServer::start().await; + let mock_server = Box::leak(Box::new(MockServer::start().await)); Mock::given(method("POST")) .and(path("/")) @@ -50,7 +50,7 @@ async fn fragment_mock_server() -> String { } ResponseTemplate::new(200).set_body_json(json!({ "data": data })) }) - .mount(&mock_server) + .mount(mock_server) .await; mock_server.uri() diff --git a/crates/meilisearch/tests/vector/openai.rs b/crates/meilisearch/tests/vector/openai.rs index 4ae8cb041..e207c3eb6 100644 --- a/crates/meilisearch/tests/vector/openai.rs +++ b/crates/meilisearch/tests/vector/openai.rs @@ -136,7 +136,7 @@ fn long_text() -> &'static str { }) } -async fn create_mock_tokenized() -> (MockServer, Value) { +async fn create_mock_tokenized() -> (&'static MockServer, Value) { create_mock_with_template("{{doc.text}}", ModelDimensions::Large, false, false).await } @@ -145,8 +145,8 @@ async fn create_mock_with_template( model_dimensions: ModelDimensions, fallible: bool, slow: bool, -) -> (MockServer, Value) { - let mock_server = MockServer::start().await; +) -> (&'static MockServer, Value) { + let mock_server = Box::leak(Box::new(MockServer::start().await)); const API_KEY: &str = "my-api-key"; const API_KEY_BEARER: &str = "Bearer my-api-key"; @@ -299,7 +299,7 @@ async fn create_mock_with_template( } })) }) - .mount(&mock_server) + .mount(mock_server) .await; let url = mock_server.uri(); @@ -321,27 +321,27 @@ const DOGGO_TEMPLATE: &str = r#"{%- if doc.gender == "F" -%}Une chienne nommée Un chien nommé {{doc.name}}, né en {{doc.birthyear}} {%- endif %}, de race {{doc.breed}}."#; -async fn create_mock() -> (MockServer, Value) { +async fn create_mock() -> (&'static MockServer, Value) { create_mock_with_template(DOGGO_TEMPLATE, ModelDimensions::Large, false, false).await } -async fn create_mock_dimensions() -> (MockServer, Value) { +async fn create_mock_dimensions() -> (&'static MockServer, Value) { create_mock_with_template(DOGGO_TEMPLATE, ModelDimensions::Large512, false, false).await } -async fn create_mock_small_embedding_model() -> (MockServer, Value) { +async fn create_mock_small_embedding_model() -> (&'static MockServer, Value) { create_mock_with_template(DOGGO_TEMPLATE, ModelDimensions::Small, false, false).await } -async fn create_mock_legacy_embedding_model() -> (MockServer, Value) { +async fn create_mock_legacy_embedding_model() -> (&'static MockServer, Value) { create_mock_with_template(DOGGO_TEMPLATE, ModelDimensions::Ada, false, false).await } -async fn create_fallible_mock() -> (MockServer, Value) { +async fn create_fallible_mock() -> (&'static MockServer, Value) { create_mock_with_template(DOGGO_TEMPLATE, ModelDimensions::Large, true, false).await } -async fn create_slow_mock() -> (MockServer, Value) { +async fn create_slow_mock() -> (&'static MockServer, Value) { create_mock_with_template(DOGGO_TEMPLATE, ModelDimensions::Large, true, true).await } diff --git a/crates/meilisearch/tests/vector/rest.rs b/crates/meilisearch/tests/vector/rest.rs index e03563bcc..974341cd0 100644 --- a/crates/meilisearch/tests/vector/rest.rs +++ b/crates/meilisearch/tests/vector/rest.rs @@ -12,8 +12,8 @@ use crate::common::Value; use crate::json; use crate::vector::{get_server_vector, GetAllDocumentsOptions}; -async fn create_mock() -> (MockServer, Value) { - let mock_server = MockServer::start().await; +async fn create_mock() -> (&'static MockServer, Value) { + let mock_server = Box::leak(Box::new(MockServer::start().await)); let text_to_embedding: BTreeMap<_, _> = vec![ // text -> embedding @@ -32,7 +32,7 @@ async fn create_mock() -> (MockServer, Value) { json!({ "data": text_to_embedding.get(text.as_str()).unwrap_or(&[99., 99., 99.]) }), ) }) - .mount(&mock_server) + .mount(mock_server) .await; let url = mock_server.uri(); @@ -50,8 +50,8 @@ async fn create_mock() -> (MockServer, Value) { (mock_server, embedder_settings) } -async fn create_mock_default_template() -> (MockServer, Value) { - let mock_server = MockServer::start().await; +async fn create_mock_default_template() -> (&'static MockServer, Value) { + let mock_server = Box::leak(Box::new(MockServer::start().await)); let text_to_embedding: BTreeMap<_, _> = vec![ // text -> embedding @@ -73,7 +73,7 @@ async fn create_mock_default_template() -> (MockServer, Value) { .set_body_json(json!({"error": "text not found", "text": text})), } }) - .mount(&mock_server) + .mount(mock_server) .await; let url = mock_server.uri(); @@ -106,8 +106,8 @@ struct SingleResponse { embedding: Vec, } -async fn create_mock_multiple() -> (MockServer, Value) { - let mock_server = MockServer::start().await; +async fn create_mock_multiple() -> (&'static MockServer, Value) { + let mock_server = Box::leak(Box::new(MockServer::start().await)); let text_to_embedding: BTreeMap<_, _> = vec![ // text -> embedding @@ -146,7 +146,7 @@ async fn create_mock_multiple() -> (MockServer, Value) { ResponseTemplate::new(200).set_body_json(response) }) - .mount(&mock_server) + .mount(mock_server) .await; let url = mock_server.uri(); @@ -176,8 +176,8 @@ struct SingleRequest { input: String, } -async fn create_mock_single_response_in_array() -> (MockServer, Value) { - let mock_server = MockServer::start().await; +async fn create_mock_single_response_in_array() -> (&'static MockServer, Value) { + let mock_server = Box::leak(Box::new(MockServer::start().await)); let text_to_embedding: BTreeMap<_, _> = vec![ // text -> embedding @@ -212,7 +212,7 @@ async fn create_mock_single_response_in_array() -> (MockServer, Value) { ResponseTemplate::new(200).set_body_json(response) }) - .mount(&mock_server) + .mount(mock_server) .await; let url = mock_server.uri(); @@ -236,8 +236,8 @@ async fn create_mock_single_response_in_array() -> (MockServer, Value) { (mock_server, embedder_settings) } -async fn create_mock_raw_with_custom_header() -> (MockServer, Value) { - let mock_server = MockServer::start().await; +async fn create_mock_raw_with_custom_header() -> (&'static MockServer, Value) { + let mock_server = Box::leak(Box::new(MockServer::start().await)); let text_to_embedding: BTreeMap<_, _> = vec![ // text -> embedding @@ -277,7 +277,7 @@ async fn create_mock_raw_with_custom_header() -> (MockServer, Value) { ResponseTemplate::new(200).set_body_json(output) }) - .mount(&mock_server) + .mount(mock_server) .await; let url = mock_server.uri(); @@ -293,8 +293,8 @@ async fn create_mock_raw_with_custom_header() -> (MockServer, Value) { (mock_server, embedder_settings) } -async fn create_mock_raw() -> (MockServer, Value) { - let mock_server = MockServer::start().await; +async fn create_mock_raw() -> (&'static MockServer, Value) { + let mock_server = Box::leak(Box::new(MockServer::start().await)); let text_to_embedding: BTreeMap<_, _> = vec![ // text -> embedding @@ -321,7 +321,7 @@ async fn create_mock_raw() -> (MockServer, Value) { ResponseTemplate::new(200).set_body_json(output) }) - .mount(&mock_server) + .mount(mock_server) .await; let url = mock_server.uri(); @@ -337,8 +337,8 @@ async fn create_mock_raw() -> (MockServer, Value) { (mock_server, embedder_settings) } -async fn create_faulty_mock_raw(sender: mpsc::Sender<()>) -> (MockServer, Value) { - let mock_server = MockServer::start().await; +async fn create_faulty_mock_raw(sender: mpsc::Sender<()>) -> (&'static MockServer, Value) { + let mock_server = Box::leak(Box::new(MockServer::start().await)); let count = AtomicUsize::new(0); Mock::given(method("POST")) @@ -355,7 +355,7 @@ async fn create_faulty_mock_raw(sender: mpsc::Sender<()>) -> (MockServer, Value) ResponseTemplate::new(500).set_body_string("Service Unavailable") } }) - .mount(&mock_server) + .mount(mock_server) .await; let url = mock_server.uri(); From 9e98a25e45ab68540e2aba398a0e3eadd9fc1b9d Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 15:56:09 +0200 Subject: [PATCH 103/312] Fix clippy --- crates/filter-parser/src/lib.rs | 2 -- crates/milli/src/search/facet/filter_vector.rs | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index b64477170..02f338673 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -614,8 +614,6 @@ impl std::fmt::Display for Token<'_> { #[cfg(test)] pub mod tests { - use std::fmt::format; - use FilterCondition as Fc; use super::*; diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index a0dc52bac..0b9cad702 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -212,7 +212,7 @@ impl<'a> VectorFilter<'a> { embedder_config.config.quantized(), ); - let mut new_docids = if let Some(fragment_token) = &self.fragment_token { + docids |= if let Some(fragment_token) = &self.fragment_token { let fragment_name = fragment_token.value(); let Some(fragment_config) = embedder_config .fragments @@ -261,8 +261,6 @@ impl<'a> VectorFilter<'a> { stats.documents } }; - - docids |= new_docids; } if let Some(universe) = universe { From 881c37393fb79208f114c9a16696f4841096daa0 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 16:06:27 +0200 Subject: [PATCH 104/312] Add telemetry --- .../src/routes/indexes/documents.rs | 25 +++++++++++++++++-- .../src/routes/indexes/search_analytics.rs | 6 +++++ 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index bc5539081..173b3ecc8 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -135,6 +135,7 @@ pub struct DocumentsFetchAggregator { per_document_id: bool, // if a filter was used per_filter: bool, + with_vector_filter: bool, #[serde(rename = "vector.retrieve_vectors")] retrieve_vectors: bool, @@ -153,8 +154,17 @@ pub struct DocumentsFetchAggregator { #[derive(Copy, Clone, Debug, PartialEq, Eq)] pub enum DocumentFetchKind { - PerDocumentId { retrieve_vectors: bool }, - Normal { with_filter: bool, limit: usize, offset: usize, retrieve_vectors: bool, ids: usize }, + PerDocumentId { + retrieve_vectors: bool, + }, + Normal { + with_filter: bool, + with_vector_filter: bool, + limit: usize, + offset: usize, + retrieve_vectors: bool, + ids: usize, + }, } impl DocumentsFetchAggregator { @@ -174,6 +184,7 @@ impl DocumentsFetchAggregator { Self { per_document_id: matches!(query, DocumentFetchKind::PerDocumentId { .. }), per_filter: matches!(query, DocumentFetchKind::Normal { with_filter, .. } if *with_filter), + with_vector_filter: matches!(query, DocumentFetchKind::Normal { with_vector_filter, .. } if *with_vector_filter), max_limit: limit, max_offset: offset, retrieve_vectors, @@ -193,6 +204,7 @@ impl Aggregate for DocumentsFetchAggregator { Box::new(Self { per_document_id: self.per_document_id | new.per_document_id, per_filter: self.per_filter | new.per_filter, + with_vector_filter: self.with_vector_filter | new.with_vector_filter, retrieve_vectors: self.retrieve_vectors | new.retrieve_vectors, max_limit: self.max_limit.max(new.max_limit), max_offset: self.max_offset.max(new.max_offset), @@ -276,6 +288,7 @@ pub async fn get_document( retrieve_vectors: param_retrieve_vectors.0, per_document_id: true, per_filter: false, + with_vector_filter: false, max_limit: 0, max_offset: 0, max_document_ids: 0, @@ -495,6 +508,10 @@ pub async fn documents_by_query_post( analytics.publish( DocumentsFetchAggregator:: { per_filter: body.filter.is_some(), + with_vector_filter: body + .filter + .as_ref() + .is_some_and(|f| f.to_string().contains("_vectors")), retrieve_vectors: body.retrieve_vectors, max_limit: body.limit, max_offset: body.offset, @@ -596,6 +613,10 @@ pub async fn get_documents( analytics.publish( DocumentsFetchAggregator:: { per_filter: query.filter.is_some(), + with_vector_filter: query + .filter + .as_ref() + .is_some_and(|f| f.to_string().contains("_vectors")), retrieve_vectors: query.retrieve_vectors, max_limit: query.limit, max_offset: query.offset, diff --git a/crates/meilisearch/src/routes/indexes/search_analytics.rs b/crates/meilisearch/src/routes/indexes/search_analytics.rs index 07f79eba7..6b3b7ea46 100644 --- a/crates/meilisearch/src/routes/indexes/search_analytics.rs +++ b/crates/meilisearch/src/routes/indexes/search_analytics.rs @@ -40,6 +40,7 @@ pub struct SearchAggregator { // filter filter_with_geo_radius: bool, filter_with_geo_bounding_box: bool, + filter_on_vectors: bool, // every time a request has a filter, this field must be incremented by the number of terms it contains filter_sum_of_criteria_terms: usize, // every time a request has a filter, this field must be incremented by one @@ -163,6 +164,7 @@ impl SearchAggregator { let stringified_filters = filter.to_string(); ret.filter_with_geo_radius = stringified_filters.contains("_geoRadius("); ret.filter_with_geo_bounding_box = stringified_filters.contains("_geoBoundingBox("); + ret.filter_on_vectors = stringified_filters.contains("_vectors"); ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count(); } @@ -260,6 +262,7 @@ impl Aggregate for SearchAggregator { distinct, filter_with_geo_radius, filter_with_geo_bounding_box, + filter_on_vectors, filter_sum_of_criteria_terms, filter_total_number_of_criteria, used_syntax, @@ -314,6 +317,7 @@ impl Aggregate for SearchAggregator { // filter self.filter_with_geo_radius |= filter_with_geo_radius; self.filter_with_geo_bounding_box |= filter_with_geo_bounding_box; + self.filter_on_vectors |= filter_on_vectors; self.filter_sum_of_criteria_terms = self.filter_sum_of_criteria_terms.saturating_add(filter_sum_of_criteria_terms); self.filter_total_number_of_criteria = @@ -388,6 +392,7 @@ impl Aggregate for SearchAggregator { distinct, filter_with_geo_radius, filter_with_geo_bounding_box, + filter_on_vectors, filter_sum_of_criteria_terms, filter_total_number_of_criteria, used_syntax, @@ -445,6 +450,7 @@ impl Aggregate for SearchAggregator { "filter": { "with_geoRadius": filter_with_geo_radius, "with_geoBoundingBox": filter_with_geo_bounding_box, + "on_vectors": filter_on_vectors, "avg_criteria_number": format!("{:.2}", filter_sum_of_criteria_terms as f64 / filter_total_number_of_criteria as f64), "most_used_syntax": used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)), }, From feb53104e51a244660e3eb2d082f7b6be42d226c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 16:19:55 +0200 Subject: [PATCH 105/312] Grammar --- .../src/scheduler/test_document_addition.rs | 2 +- crates/meilisearch/src/routes/tasks.rs | 8 ++++---- crates/meilisearch/tests/documents/errors.rs | 2 +- crates/meilisearch/tests/search/errors.rs | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/test_document_addition.rs b/crates/index-scheduler/src/scheduler/test_document_addition.rs index b642f5604..7ca72da95 100644 --- a/crates/index-scheduler/src/scheduler/test_document_addition.rs +++ b/crates/index-scheduler/src/scheduler/test_document_addition.rs @@ -736,7 +736,7 @@ fn test_document_addition_mixed_rights_with_index() { #[test] fn test_document_addition_mixed_right_without_index_starts_with_cant_create() { // We're going to autobatch multiple document addition. - // - The index does not exists + // - The index does not exist // - The first document addition don't have the right to create an index // - The second do. They should not batch together. // - The second should batch with everything else as it's going to create an index. diff --git a/crates/meilisearch/src/routes/tasks.rs b/crates/meilisearch/src/routes/tasks.rs index 95c105894..fb0f73425 100644 --- a/crates/meilisearch/src/routes/tasks.rs +++ b/crates/meilisearch/src/routes/tasks.rs @@ -336,7 +336,7 @@ impl Aggregate for TaskFilterAnalytics Date: Tue, 8 Jul 2025 16:23:45 +0200 Subject: [PATCH 106/312] Add test --- crates/meilisearch/tests/search/filters.rs | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 3cc7cbab5..3b26ab4ee 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -767,6 +767,26 @@ async fn vector_filter_all_embedders() { "#); } +#[actix_rt::test] +async fn vector_filter_missing_fragment() { + let index = crate::vector::shared_index_for_fragments().await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.fragments EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "message": "Index `[uuid]`: Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.\n15:24 _vectors.rest.fragments EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" + } + "#); +} + #[actix_rt::test] async fn vector_filter_non_existant_embedder() { let index = crate::vector::shared_index_for_fragments().await; From 8adf6141e09cc9dce662ddcc31a65f89fd81fdec Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 8 Jul 2025 16:55:43 +0200 Subject: [PATCH 107/312] Fix old test --- crates/milli/src/test_index.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/crates/milli/src/test_index.rs b/crates/milli/src/test_index.rs index 6bb6b1345..0ec348301 100644 --- a/crates/milli/src/test_index.rs +++ b/crates/milli/src/test_index.rs @@ -19,7 +19,9 @@ use crate::update::{ }; use crate::vector::settings::{EmbedderSource, EmbeddingSettings}; use crate::vector::RuntimeEmbedders; -use crate::{db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult}; +use crate::{ + db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult, UserError, +}; pub(crate) struct TempIndex { pub inner: Index, @@ -1341,8 +1343,8 @@ fn vectors_are_never_indexed_as_searchable_or_filterable() { let results = search .filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()) .execute() - .unwrap(); - assert!(results.candidates.is_empty()); + .unwrap_err(); + assert!(matches!(results, Error::UserError(UserError::InvalidFilter(_)))); index .update_settings(|settings| { @@ -1373,6 +1375,6 @@ fn vectors_are_never_indexed_as_searchable_or_filterable() { let results = search .filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()) .execute() - .unwrap(); - assert!(results.candidates.is_empty()); + .unwrap_err(); + assert!(matches!(results, Error::UserError(UserError::InvalidFilter(_)))); } From 39f808714d6ae7667d914817dcf645c17c027374 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 9 Jul 2025 18:03:32 +0200 Subject: [PATCH 108/312] Implement a documentTemplate filter --- crates/meilisearch/tests/search/filters.rs | 79 +++++++++++++++++++ crates/meilisearch/tests/vector/mod.rs | 4 +- crates/meilisearch/tests/vector/rest.rs | 2 +- .../milli/src/search/facet/filter_vector.rs | 39 +++++++-- 4 files changed, 115 insertions(+), 9 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 3b26ab4ee..d0f388220 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -989,6 +989,85 @@ async fn vector_filter_specific_fragment_user_provided() { "#); } +#[actix_rt::test] +async fn vector_filter_document_template_but_fragments_used() { + let index = crate::vector::shared_index_for_fragments().await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.documentTemplate EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "hits": [], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0 + } + "#); +} + +#[actix_rt::test] +async fn vector_filter_document_template() { + let (_mock, setting) = crate::vector::create_mock().await; + let server = crate::vector::get_server_vector().await; + let index = server.index("doggo"); + + let (response, code) = index + .update_settings(json!({ + "embedders": { + "rest": setting, + }, + })) + .await; + snapshot!(code, @"202 Accepted"); + server.wait_task(response.uid()).await.succeeded(); + + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel"}, + {"id": 3, "name": "iko" } + ]); + let (value, code) = index.add_documents(documents, None).await; + snapshot!(code, @"202 Accepted"); + index.wait_task(value.uid()).await.succeeded(); + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors.rest.documentTemplate EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "hits": [ + { + "name": "kefir" + }, + { + "name": "echo" + }, + { + "name": "intel" + }, + { + "name": "iko" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 4 + } + "#); +} + #[actix_rt::test] async fn vector_filter_negation() { let index = crate::vector::shared_index_for_fragments().await; diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 9ba37cae3..8851d029e 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -14,9 +14,9 @@ use meilisearch::option::MaxThreads; use crate::common::index::Index; use crate::common::{default_settings, GetAllDocumentsOptions, Server}; use crate::json; -pub use fragments::shared_index_for_fragments; +pub use {fragments::shared_index_for_fragments, rest::create_mock}; -async fn get_server_vector() -> Server { +pub async fn get_server_vector() -> Server { Server::new().await } diff --git a/crates/meilisearch/tests/vector/rest.rs b/crates/meilisearch/tests/vector/rest.rs index 974341cd0..dae9e9139 100644 --- a/crates/meilisearch/tests/vector/rest.rs +++ b/crates/meilisearch/tests/vector/rest.rs @@ -12,7 +12,7 @@ use crate::common::Value; use crate::json; use crate::vector::{get_server_vector, GetAllDocumentsOptions}; -async fn create_mock() -> (&'static MockServer, Value) { +pub async fn create_mock() -> (&'static MockServer, Value) { let mock_server = Box::leak(Box::new(MockServer::start().await)); let text_to_embedding: BTreeMap<_, _> = vec![ diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 0b9cad702..e3ec698f5 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -8,6 +8,7 @@ use crate::Index; pub(super) struct VectorFilter<'a> { embedder_token: Option>, fragment_token: Option>, + document_template: bool, user_provided: bool, } @@ -17,6 +18,7 @@ pub enum VectorFilterError<'a> { InvalidPrefix(Token<'a>), MissingFragmentName(Token<'a>), UserProvidedWithFragment(Token<'a>), + DocumentTemplateWithFragment(Token<'a>), LeftoverToken(Token<'a>), EmbedderDoesNotExist { embedder: &'a Token<'a>, @@ -52,6 +54,9 @@ impl std::fmt::Display for VectorFilterError<'_> { UserProvidedWithFragment(_token) => { write!(f, "Vector filter cannot specify both a fragment name and userProvided.") } + DocumentTemplateWithFragment(_token) => { + write!(f, "Vector filter cannot specify both a fragment name and documentTemplate.") + } LeftoverToken(token) => { write!(f, "Vector filter has leftover token: `{}`.", token.value()) } @@ -105,6 +110,7 @@ impl<'a> From> for Error { InvalidPrefix(token) | MissingFragmentName(token) | UserProvidedWithFragment(token) + | DocumentTemplateWithFragment(token) | LeftoverToken(token) => token.clone().as_external_error(err).into(), EmbedderDoesNotExist { embedder: token, .. } | FragmentDoesNotExist { fragment: token, .. } => token.as_external_error(err).into(), @@ -123,6 +129,8 @@ impl<'a> VectorFilter<'a> { /// - `_vectors` /// - `_vectors.{embedder_name}` /// - `_vectors.{embedder_name}.userProvided` + /// - `_vectors.{embedder_name}.documentTemplate` + /// - `_vectors.{embedder_name}.documentTemplate.userProvided` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` pub(super) fn parse(s: &'a Token<'a>) -> Result> { let mut split = s.split(".").peekable(); @@ -149,10 +157,22 @@ impl<'a> VectorFilter<'a> { user_provided_token = split.next(); } + let mut document_template_token = None; + if split.peek().map(|t| t.value()) == Some("documentTemplate") + || split.peek().map(|t| t.value()) == Some("document_template") + { + document_template_token = split.next(); + } + if let (Some(_), Some(user_provided_token)) = (&fragment_name, &user_provided_token) { return Err(UserProvidedWithFragment(user_provided_token.clone()))?; } + if let (Some(_), Some(document_template_token)) = (&fragment_name, &document_template_token) + { + return Err(DocumentTemplateWithFragment(document_template_token.clone()))?; + } + if let Some(next) = split.next() { return Err(LeftoverToken(next))?; } @@ -161,6 +181,7 @@ impl<'a> VectorFilter<'a> { embedder_token: embedder_name, fragment_token: fragment_name, user_provided: user_provided_token.is_some(), + document_template: document_template_token.is_some(), }) } @@ -176,7 +197,8 @@ impl<'a> VectorFilter<'a> { let mut embedders = Vec::new(); if let Some(embedder_token) = &self.embedder_token { let embedder_name = embedder_token.value(); - let Some(embedder_config) = + + let Some(embedding_config) = embedding_configs.iter().find(|config| config.name == embedder_name) else { return Err(EmbedderDoesNotExist { @@ -184,6 +206,7 @@ impl<'a> VectorFilter<'a> { available: embedding_configs.iter().map(|c| c.name.clone()).collect(), })?; }; + let Some(embedder_info) = index_embedding_configs.embedder_info(rtxn, embedder_name)? else { return Err(EmbedderDoesNotExist { @@ -192,7 +215,11 @@ impl<'a> VectorFilter<'a> { })?; }; - embedders.push((embedder_config, embedder_info)); + if self.document_template && !embedding_config.fragments.as_slice().is_empty() { + return Ok(RoaringBitmap::new()); + } + + embedders.push((embedding_config, embedder_info)); } else { for embedder_config in embedding_configs.iter() { let Some(embedder_info) = @@ -205,16 +232,16 @@ impl<'a> VectorFilter<'a> { }; let mut docids = RoaringBitmap::new(); - for (embedder_config, embedder_info) in embedders { + for (embedding_config, embedder_info) in embedders { let arroy_wrapper = ArroyWrapper::new( index.vector_arroy, embedder_info.embedder_id, - embedder_config.config.quantized(), + embedding_config.config.quantized(), ); docids |= if let Some(fragment_token) = &self.fragment_token { let fragment_name = fragment_token.value(); - let Some(fragment_config) = embedder_config + let Some(fragment_config) = embedding_config .fragments .as_slice() .iter() @@ -226,7 +253,7 @@ impl<'a> VectorFilter<'a> { .as_ref() .expect("there can't be a fragment without an embedder"), fragment: fragment_token, - available: embedder_config + available: embedding_config .fragments .as_slice() .iter() From a3b8c2b71fc1aa2766a94b6daaf7c3872c466d01 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 9 Jul 2025 18:21:52 +0200 Subject: [PATCH 109/312] Gate behind multimodal experimental feature --- crates/filter-parser/src/lib.rs | 19 +++++++++++++++++++ crates/meilisearch/src/search/mod.rs | 14 +++++++++++++- crates/meilisearch/tests/search/filters.rs | 20 ++++++++++++++++++++ crates/milli/src/search/facet/filter.rs | 4 ++++ 4 files changed, 56 insertions(+), 1 deletion(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 02f338673..1590b08fd 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -189,6 +189,25 @@ impl<'a> FilterCondition<'a> { } } + pub fn use_vector_filter(&self) -> Option<&Token> { + match self { + FilterCondition::Condition { fid, op: _ } => { + if fid.value().starts_with("_vectors.") || fid.value() == "_vectors" { + Some(fid) + } else { + None + } + } + FilterCondition::Not(this) => this.use_vector_filter(), + FilterCondition::Or(seq) | FilterCondition::And(seq) => { + seq.iter().find_map(|filter| filter.use_vector_filter()) + } + FilterCondition::GeoLowerThan { .. } + | FilterCondition::GeoBoundingBox { .. } + | FilterCondition::In { .. } => None, + } + } + pub fn fids(&self, depth: usize) -> Box + '_> { if depth == 0 { return Box::new(std::iter::empty()); diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 1c987a70c..e82f4dff7 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -2077,7 +2077,7 @@ pub(crate) fn parse_filter( })?; if let Some(ref filter) = filter { - // If the contains operator is used while the contains filter features is not enabled, errors out + // If the contains operator is used while the contains filter feature is not enabled, errors out if let Some((token, error)) = filter.use_contains_operator().zip(features.check_contains_filter().err()) { @@ -2088,6 +2088,18 @@ pub(crate) fn parse_filter( } } + if let Some(ref filter) = filter { + // If a vector filter is used while the multi modal feature is not enabled, errors out + if let Some((token, error)) = + filter.use_vector_filter().zip(features.check_multimodal("using a vector filter").err()) + { + return Err(ResponseError::from_msg( + token.as_external_error(error).to_string(), + Code::FeatureNotEnabled, + )); + } + } + Ok(filter) } diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index d0f388220..ff6a0cb17 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -1068,6 +1068,26 @@ async fn vector_filter_document_template() { "#); } +#[actix_rt::test] +async fn vector_filter_feature_gate() { + let index = shared_index_with_documents().await; + + let (value, _code) = index + .search_post(json!({ + "filter": "_vectors EXISTS", + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "message": "using a vector filter requires enabling the `multimodal` experimental feature. See https://github.com/orgs/meilisearch/discussions/846\n1:9 _vectors EXISTS", + "code": "feature_not_enabled", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#feature_not_enabled" + } + "#); +} + #[actix_rt::test] async fn vector_filter_negation() { let index = crate::vector::shared_index_for_fragments().await; diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index 1afdf87e6..21a552965 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -228,6 +228,10 @@ impl<'a> Filter<'a> { pub fn use_contains_operator(&self) -> Option<&Token> { self.condition.use_contains_operator() } + + pub fn use_vector_filter(&self) -> Option<&Token> { + self.condition.use_vector_filter() + } } impl<'a> Filter<'a> { From a9309774602667c47a63ab04684c72bb73376338 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 10 Jul 2025 09:37:58 +0200 Subject: [PATCH 110/312] Fix test --- crates/meilisearch/tests/search/filters.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index ff6a0cb17..aa2b06e76 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -1017,6 +1017,9 @@ async fn vector_filter_document_template() { let server = crate::vector::get_server_vector().await; let index = server.index("doggo"); + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + snapshot!(code, @"200 OK"); + let (response, code) = index .update_settings(json!({ "embedders": { From 126aefc2073b0b24c64a9444be7cfa0802ce2b89 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Thu, 10 Jul 2025 16:39:58 +0300 Subject: [PATCH 111/312] Fix more tests Signed-off-by: Martin Tzvetanov Grigorov --- crates/meilisearch/tests/batches/mod.rs | 129 ++++++++++++---------- crates/meilisearch/tests/common/server.rs | 13 --- 2 files changed, 70 insertions(+), 72 deletions(-) diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index d5374a144..5b67dc50c 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -1,11 +1,30 @@ mod errors; +use insta::internals::{Content, ContentPath}; +use once_cell::sync::Lazy; +use regex::Regex; use meili_snap::insta::assert_json_snapshot; use meili_snap::{json_string, snapshot}; use crate::common::Server; use crate::json; + +static TASK_WITH_ID_RE: Lazy = Lazy::new(|| { + Regex::new(r"task with id (\d+) of type") + .unwrap() +}); + +fn task_with_id_redaction(value: Content, _path: ContentPath) -> Content { + match value { + Content::String(s) => { + let replaced = TASK_WITH_ID_RE.replace_all(&s, "task with id X of type"); + Content::String(replaced.to_string()) + } + _ => value.clone(), + } +} + #[actix_rt::test] async fn error_get_unexisting_batch_status() { let server = Server::new_shared(); @@ -30,7 +49,7 @@ async fn get_batch_status() { let server = Server::new_shared(); let index = server.unique_index(); let (task, _status_code) = index.create(None).await; - index.wait_task(task.uid()).await.succeeded(); + let task = index.wait_task(task.uid()).await.succeeded(); let (_response, code) = index.get_batch(task.batch_uid()).await; assert_eq!(code, 200); } @@ -272,7 +291,7 @@ async fn test_summarized_document_addition_or_update() { let index = server.unique_index(); let (task, _status_code) = index.add_documents(json!({ "id": 42, "content": "doggos & fluff" }), None).await; - index.wait_task(task.uid()).await.succeeded(); + let task = index.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -285,7 +304,6 @@ async fn test_summarized_document_addition_or_update() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { @@ -311,13 +329,13 @@ async fn test_summarized_document_addition_or_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" + "batchStrategy": "batched all enqueued tasks" } "###); let (task, _status_code) = index.add_documents(json!({ "id": 42, "content": "doggos & fluff" }), Some("id")).await; - index.wait_task(task.uid()).await.succeeded(); + let task = index.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -330,7 +348,6 @@ async fn test_summarized_document_addition_or_update() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { @@ -355,7 +372,7 @@ async fn test_summarized_document_addition_or_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" + "batchStrategy": "batched all enqueued tasks" } "###); } @@ -368,7 +385,7 @@ async fn test_summarized_delete_documents_by_batch() { let task_uid_2 = (u32::MAX - 2) as u64; let task_uid_3 = (u32::MAX - 3) as u64; let (task, _status_code) = index.delete_batch(vec![task_uid_1, task_uid_2, task_uid_3]).await; - index.wait_task(task.uid()).await.failed(); + let task = index.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -380,7 +397,6 @@ async fn test_summarized_delete_documents_by_batch() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { @@ -404,13 +420,13 @@ async fn test_summarized_delete_documents_by_batch() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" + "batchStrategy": "batched all enqueued tasks" } "###); index.create(None).await; let (task, _status_code) = index.delete_batch(vec![42]).await; - index.wait_task(task.uid()).await.succeeded(); + let task = index.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -458,7 +474,7 @@ async fn test_summarized_delete_documents_by_filter() { let (task, _status_code) = index.delete_document_by_filter(json!({ "filter": "doggo = bernese" })).await; - index.wait_task(task.uid()).await.failed(); + let task = index.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -470,7 +486,6 @@ async fn test_summarized_delete_documents_by_filter() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { @@ -495,14 +510,14 @@ async fn test_summarized_delete_documents_by_filter() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" + "batchStrategy": "batched all enqueued tasks" } "###); index.create(None).await; let (task, _status_code) = index.delete_document_by_filter(json!({ "filter": "doggo = bernese" })).await; - index.wait_task(task.uid()).await.failed(); + let task = index.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -515,7 +530,6 @@ async fn test_summarized_delete_documents_by_filter() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { @@ -540,14 +554,14 @@ async fn test_summarized_delete_documents_by_filter() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" + "batchStrategy": "batched all enqueued tasks" } "###); index.update_settings(json!({ "filterableAttributes": ["doggo"] })).await; let (task, _status_code) = index.delete_document_by_filter(json!({ "filter": "doggo = bernese" })).await; - index.wait_task(task.uid()).await.succeeded(); + let task = index.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -595,7 +609,7 @@ async fn test_summarized_delete_document_by_id() { let server = Server::new_shared(); let index = server.unique_index(); let (task, _status_code) = index.delete_document(1).await; - index.wait_task(task.uid()).await.failed(); + let task = index.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -607,7 +621,6 @@ async fn test_summarized_delete_document_by_id() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { @@ -631,13 +644,13 @@ async fn test_summarized_delete_document_by_id() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" + "batchStrategy": "batched all enqueued tasks" } "###); index.create(None).await; let (task, _status_code) = index.delete_document(42).await; - index.wait_task(task.uid()).await.succeeded(); + let task = index.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -650,7 +663,6 @@ async fn test_summarized_delete_document_by_id() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" }, @r###" { @@ -674,7 +686,7 @@ async fn test_summarized_delete_document_by_id() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" + "batchStrategy": "batched all enqueued tasks" } "###); } @@ -696,7 +708,7 @@ async fn test_summarized_settings_update() { "###); let (task,_status_code) = index.update_settings(json!({ "displayedAttributes": ["doggos", "name"], "filterableAttributes": ["age", "nb_paw_pads"], "sortableAttributes": ["iq"] })).await; - index.wait_task(task.uid()).await.succeeded(); + let task = index.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -709,8 +721,6 @@ async fn test_summarized_settings_update() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" - }, @r###" { @@ -743,7 +753,7 @@ async fn test_summarized_settings_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" + "batchStrategy": "batched all enqueued tasks" } "###); } @@ -753,7 +763,7 @@ async fn test_summarized_index_creation() { let server = Server::new_shared(); let index = server.unique_index(); let (task, _status_code) = index.create(None).await; - index.wait_task(task.uid()).await.succeeded(); + let task = index.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -765,7 +775,7 @@ async fn test_summarized_index_creation() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "task with id X of type `indexCreation` cannot be batched" + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -786,12 +796,12 @@ async fn test_summarized_index_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexCreation` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexCreation` that cannot be batched with any other task." } "###); let (task, _status_code) = index.create(Some("doggos")).await; - index.wait_task(task.uid()).await.failed(); + let task = index.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -803,7 +813,7 @@ async fn test_summarized_index_creation() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "task with id X of type `indexCreation` cannot be batched" + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -826,7 +836,7 @@ async fn test_summarized_index_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexCreation` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexCreation` that cannot be batched with any other task." } "###); } @@ -946,7 +956,7 @@ async fn test_summarized_index_update() { let index = server.unique_index(); // If the index doesn't exist yet, we should get errors with or without the primary key. let (task, _status_code) = index.update(None).await; - index.wait_task(task.uid()).await.failed(); + let task = index.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -958,7 +968,7 @@ async fn test_summarized_index_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "task with id X of type `indexUpdate` cannot be batched" + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -979,12 +989,12 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexUpdate` that cannot be batched with any other task." } "###); let (task, _status_code) = index.update(Some("bones")).await; - index.wait_task(task.uid()).await.failed(); + let task = index.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -996,7 +1006,7 @@ async fn test_summarized_index_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "task with id X of type `indexUpdate` cannot be batched" + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1019,15 +1029,15 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexUpdate` that cannot be batched with any other task." } "###); - // And run the same two tests once the index do exists. + // And run the same two tests once the index does exist. index.create(None).await; let (task, _status_code) = index.update(None).await; - index.wait_task(task.uid()).await.succeeded(); + let task = index.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -1039,7 +1049,7 @@ async fn test_summarized_index_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "task with id X of type `indexUpdate` cannot be batched" + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1060,12 +1070,12 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexUpdate` that cannot be batched with any other task." } "###); let (task, _status_code) = index.update(Some("bones")).await; - index.wait_task(task.uid()).await.succeeded(); + let task = index.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -1077,7 +1087,7 @@ async fn test_summarized_index_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "task with id X of type `indexUpdate` cannot be batched" + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1100,7 +1110,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexUpdate` that cannot be batched with any other task." } "###); } @@ -1124,7 +1134,7 @@ async fn test_summarized_index_swap() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".batchCreationComplete" => "task with id X of type `indexSwap` cannot be batched" + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1154,7 +1164,7 @@ async fn test_summarized_index_swap() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexSwap` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexSwap` that cannot be batched with any other task." } "###); @@ -1179,7 +1189,7 @@ async fn test_summarized_index_swap() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "task with id X of type `indexCreation` cannot be batched" + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1200,11 +1210,12 @@ async fn test_summarized_index_swap() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexCreation` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexCreation` that cannot be batched with any other task." } "###); } + #[actix_web::test] async fn test_summarized_batch_cancelation() { let server = Server::new_shared(); @@ -1213,7 +1224,7 @@ async fn test_summarized_batch_cancelation() { let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = server.cancel_tasks(format!("uids={}", task.uid()).as_str()).await; - index.wait_task(task.uid()).await.succeeded(); + let task = index.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -1224,8 +1235,8 @@ async fn test_summarized_batch_cancelation() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".batchCreationComplete" => "task with id X of type `taskCancelation` cannot be batched", ".details.originalFilter" => "?uids=X", + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1250,7 +1261,7 @@ async fn test_summarized_batch_cancelation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `taskCancelation` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `taskCancelation` that cannot be batched with any other task." } "###); } @@ -1263,7 +1274,7 @@ async fn test_summarized_batch_deletion() { let (task, _status_code) = index.create(None).await; index.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = server.delete_tasks(format!("uids={}", task.uid()).as_str()).await; - index.wait_task(task.uid()).await.succeeded(); + let task = index.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -1308,8 +1319,8 @@ async fn test_summarized_batch_deletion() { async fn test_summarized_dump_creation() { let server = Server::new_shared(); let (task, _status_code) = server.create_dump().await; - server.wait_task(task.uid()).await.succeeded(); - let (batch, _) = server.get_latest_batch().await; + let task = server.wait_task(task.uid()).await.succeeded(); + let (batch, _) = server.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -1320,7 +1331,7 @@ async fn test_summarized_dump_creation() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".batchCreationComplete" => "task with id X of type `dumpCreation` cannot be batched" + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1343,7 +1354,7 @@ async fn test_summarized_dump_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `dumpCreation` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `dumpCreation` that cannot be batched with any other task." } "###); } diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 787cafc9f..431972983 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -429,19 +429,6 @@ impl Server { self.service.get(url).await } - // https://www.meilisearch.com/docs/reference/api/batches#get-batches states: - // "Batches are always returned in descending order of uid. This means that by default, - // the most recently created batch objects appear first." - pub async fn get_latest_batch(&self) -> (Option, StatusCode) { - let url = "/batches?limit=1&offset=0"; - let (value, code) = self.service.get(url).await; - value - .get("results") - .and_then(|results| results.as_array()) - .and_then(|array| array.first()) - .map_or((None, code), |latest| (Some(Value(latest.clone())), code)) - } - pub async fn get_features(&self) -> (Value, StatusCode) { self.service.get("/experimental-features").await } From 30fd546c1227bbfc3e71e3541f41847fca4c4713 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 10 Jul 2025 16:43:10 +0200 Subject: [PATCH 112/312] Format --- crates/milli/src/vector/mod.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index f64223e41..3d77e78d7 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -485,6 +485,8 @@ impl ArroyWrapper { limit: usize, filter: Option<&RoaringBitmap>, ) -> Result, arroy::Error> { + println!("nns_by_vector: quantized={} {:?} limit={} filter={:?}", + self.quantized, vector, limit, filter); if self.quantized { self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) } else { @@ -517,6 +519,8 @@ impl ArroyWrapper { results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); + println!("nns_by_vector: results={:?}", results); + Ok(results) } From f244439b4f7493edb4a7f2387f80ae932cd3229b Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 10 Jul 2025 16:43:45 +0200 Subject: [PATCH 113/312] Revert "Format" This reverts commit 30fd546c1227bbfc3e71e3541f41847fca4c4713. --- crates/milli/src/vector/mod.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index 3d77e78d7..f64223e41 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -485,8 +485,6 @@ impl ArroyWrapper { limit: usize, filter: Option<&RoaringBitmap>, ) -> Result, arroy::Error> { - println!("nns_by_vector: quantized={} {:?} limit={} filter={:?}", - self.quantized, vector, limit, filter); if self.quantized { self._nns_by_vector(rtxn, self.quantized_db(), vector, limit, filter) } else { @@ -519,8 +517,6 @@ impl ArroyWrapper { results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance)); - println!("nns_by_vector: results={:?}", results); - Ok(results) } From 50bc1d55f3128235fa0c5ac3019286ca4082a3da Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Thu, 10 Jul 2025 18:23:46 +0200 Subject: [PATCH 114/312] Add test reproducing the bug --- .../tests/settings/get_settings.rs | 65 +++++++++++++++++++ 1 file changed, 65 insertions(+) diff --git a/crates/meilisearch/tests/settings/get_settings.rs b/crates/meilisearch/tests/settings/get_settings.rs index 47e699380..f50f7f940 100644 --- a/crates/meilisearch/tests/settings/get_settings.rs +++ b/crates/meilisearch/tests/settings/get_settings.rs @@ -692,3 +692,68 @@ async fn granular_filterable_attributes() { ] "###); } + +#[actix_rt::test] +async fn test_searchable_attributes_order() { + let server = Server::new_shared(); + let index = server.unique_index(); + + // 1) Create an index with settings "searchableAttributes": ["title", "overview"] + let (response, code) = index.create(None).await; + assert_eq!(code, 202, "{response}"); + server.wait_task(response.uid()).await.succeeded(); + + let (task, code) = index + .update_settings(json!({ + "searchableAttributes": ["title", "overview"] + })) + .await; + assert_eq!(code, 202, "{task}"); + server.wait_task(task.uid()).await.succeeded(); + + // 2) Add documents in the index + let documents = json!([ + { + "id": 1, + "title": "The Matrix", + "overview": "A computer hacker learns from mysterious rebels about the true nature of his reality." + }, + { + "id": 2, + "title": "Inception", + "overview": "A thief who steals corporate secrets through dream-sharing technology." + } + ]); + + let (response, code) = index.add_documents(documents, None).await; + assert_eq!(code, 202, "{response}"); + server.wait_task(response.uid()).await.succeeded(); + + // 3) Modify the settings "searchableAttributes": ["overview", "title"] (overview is put first) + let (task, code) = index + .update_settings(json!({ + "searchableAttributes": ["overview", "title"] + })) + .await; + assert_eq!(code, 202, "{task}"); + server.wait_task(task.uid()).await.succeeded(); + + // 4) Check if it has been applied + let (response, code) = index.settings().await; + assert_eq!(code, 200, "{response}"); + assert_eq!(response["searchableAttributes"], json!(["overview", "title"])); + + // 5) Re-modify the settings "searchableAttributes": ["title", "overview"] (title is put first) + let (task, code) = index + .update_settings(json!({ + "searchableAttributes": ["title", "overview"] + })) + .await; + assert_eq!(code, 202, "{task}"); + server.wait_task(task.uid()).await.succeeded(); + + // 6) Check if it has been applied + let (response, code) = index.settings().await; + assert_eq!(code, 200, "{response}"); + assert_eq!(response["searchableAttributes"], json!(["title", "overview"])); +} From 3f655ea20ef5238c11f2d23a2a0ad8812730d149 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Thu, 10 Jul 2025 18:24:23 +0200 Subject: [PATCH 115/312] compare user defined searchable fields instead of internal searchable fields --- crates/milli/src/update/settings.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index 911f51865..fdc21797f 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -554,10 +554,10 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { match self.searchable_fields { Setting::Set(ref fields) => { // Check to see if the searchable fields changed before doing anything else - let old_fields = self.index.searchable_fields(self.wtxn)?; + let old_fields = self.index.user_defined_searchable_fields(self.wtxn)?; let did_change = { let new_fields = fields.iter().map(String::as_str).collect::>(); - new_fields != old_fields + old_fields.map(|old| new_fields != old).unwrap_or(true) }; if !did_change { return Ok(false); From 9f89881b0df27b5e155a3ec2e1c63831d0573174 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 11 Jul 2025 10:11:58 +0300 Subject: [PATCH 116/312] More tests fixes Signed-off-by: Martin Tzvetanov Grigorov --- crates/index-scheduler/src/queue/tasks.rs | 7 +- crates/meili-snap/src/lib.rs | 2 +- crates/meilisearch/tests/batches/mod.rs | 79 ++++++++++++----------- 3 files changed, 43 insertions(+), 45 deletions(-) diff --git a/crates/index-scheduler/src/queue/tasks.rs b/crates/index-scheduler/src/queue/tasks.rs index 92789b93f..74192232e 100644 --- a/crates/index-scheduler/src/queue/tasks.rs +++ b/crates/index-scheduler/src/queue/tasks.rs @@ -530,12 +530,7 @@ impl Queue { ..task } } else { - dbg!(&task); - if task.status == Status::Succeeded || task.status == Status::Failed { - Task { batch_uid: Some(batch.uid), ..task } - } else { - task - } + task } }) .collect(), diff --git a/crates/meili-snap/src/lib.rs b/crates/meili-snap/src/lib.rs index 1641a6335..a59732f04 100644 --- a/crates/meili-snap/src/lib.rs +++ b/crates/meili-snap/src/lib.rs @@ -55,7 +55,7 @@ pub fn default_snapshot_settings_for_test<'a>( settings.add_dynamic_redaction(".error.message", |content, _content_path| match &content { Content::String(s) => { - let uuid_replaced = UUID_IN_MESSAGE_RE.replace_all(s, "$before[uuid]$after"); + let uuid_replaced = UUID_IN_MESSAGE_RE.replace_all(s, "[uuid]"); Content::String(uuid_replaced.to_string()) } _ => content, diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index 5b67dc50c..e68eb3fc6 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -1,19 +1,16 @@ mod errors; use insta::internals::{Content, ContentPath}; -use once_cell::sync::Lazy; -use regex::Regex; use meili_snap::insta::assert_json_snapshot; use meili_snap::{json_string, snapshot}; +use once_cell::sync::Lazy; +use regex::Regex; use crate::common::Server; use crate::json; - -static TASK_WITH_ID_RE: Lazy = Lazy::new(|| { - Regex::new(r"task with id (\d+) of type") - .unwrap() -}); +static TASK_WITH_ID_RE: Lazy = + Lazy::new(|| Regex::new(r"task with id (\d+) of type").unwrap()); fn task_with_id_redaction(value: Content, _path: ContentPath) -> Content { match value { @@ -304,6 +301,7 @@ async fn test_summarized_document_addition_or_update() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks", }, @r###" { @@ -329,7 +327,7 @@ async fn test_summarized_document_addition_or_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); @@ -372,7 +370,7 @@ async fn test_summarized_document_addition_or_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); } @@ -397,6 +395,7 @@ async fn test_summarized_delete_documents_by_batch() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks", }, @r###" { @@ -420,7 +419,7 @@ async fn test_summarized_delete_documents_by_batch() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); @@ -486,6 +485,7 @@ async fn test_summarized_delete_documents_by_filter() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks", }, @r###" { @@ -510,7 +510,7 @@ async fn test_summarized_delete_documents_by_filter() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); @@ -530,6 +530,7 @@ async fn test_summarized_delete_documents_by_filter() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks", }, @r###" { @@ -554,7 +555,7 @@ async fn test_summarized_delete_documents_by_filter() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); @@ -574,7 +575,7 @@ async fn test_summarized_delete_documents_by_filter() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks for index `[uuid]`" + ".batchCreationComplete" => "batched all enqueued tasks" }, @r###" { @@ -599,7 +600,7 @@ async fn test_summarized_delete_documents_by_filter() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "batched all enqueued tasks for index `[uuid]`" + "batchCreationComplete": "batched all enqueued tasks" } "###); } @@ -621,6 +622,7 @@ async fn test_summarized_delete_document_by_id() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks", }, @r###" { @@ -644,7 +646,7 @@ async fn test_summarized_delete_document_by_id() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); @@ -663,6 +665,7 @@ async fn test_summarized_delete_document_by_id() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks", }, @r###" { @@ -686,7 +689,7 @@ async fn test_summarized_delete_document_by_id() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); } @@ -721,6 +724,7 @@ async fn test_summarized_settings_update() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks" }, @r###" { @@ -753,7 +757,7 @@ async fn test_summarized_settings_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "batched all enqueued tasks" + "batchCreationComplete": "batched all enqueued tasks" } "###); } @@ -775,7 +779,7 @@ async fn test_summarized_index_creation() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), + ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -796,7 +800,7 @@ async fn test_summarized_index_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "created batch containing only task with id X of type `indexCreation` that cannot be batched with any other task." + "batchCreationComplete": "task with id X of type `indexCreation` cannot be batched" } "###); @@ -813,7 +817,7 @@ async fn test_summarized_index_creation() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), + ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -836,7 +840,7 @@ async fn test_summarized_index_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "created batch containing only task with id X of type `indexCreation` that cannot be batched with any other task." + "batchCreationComplete": "task with id X of type `indexCreation` cannot be batched" } "###); } @@ -968,7 +972,7 @@ async fn test_summarized_index_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), + ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -989,7 +993,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "created batch containing only task with id X of type `indexUpdate` that cannot be batched with any other task." + "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" } "###); @@ -1006,7 +1010,7 @@ async fn test_summarized_index_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), + ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1029,7 +1033,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "created batch containing only task with id X of type `indexUpdate` that cannot be batched with any other task." + "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" } "###); @@ -1049,7 +1053,7 @@ async fn test_summarized_index_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), + ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1070,7 +1074,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "created batch containing only task with id X of type `indexUpdate` that cannot be batched with any other task." + "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" } "###); @@ -1087,7 +1091,7 @@ async fn test_summarized_index_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), + ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1110,7 +1114,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "created batch containing only task with id X of type `indexUpdate` that cannot be batched with any other task." + "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" } "###); } @@ -1134,7 +1138,7 @@ async fn test_summarized_index_swap() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), + ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1164,7 +1168,7 @@ async fn test_summarized_index_swap() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "created batch containing only task with id X of type `indexSwap` that cannot be batched with any other task." + "batchCreationComplete": "task with id X of type `indexSwap` cannot be batched" } "###); @@ -1189,7 +1193,7 @@ async fn test_summarized_index_swap() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), + ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1210,12 +1214,11 @@ async fn test_summarized_index_swap() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "created batch containing only task with id X of type `indexCreation` that cannot be batched with any other task." + "batchCreationComplete": "task with id X of type `indexCreation` cannot be batched" } "###); } - #[actix_web::test] async fn test_summarized_batch_cancelation() { let server = Server::new_shared(); @@ -1236,7 +1239,7 @@ async fn test_summarized_batch_cancelation() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".details.originalFilter" => "?uids=X", - ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), + ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1261,7 +1264,7 @@ async fn test_summarized_batch_cancelation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "created batch containing only task with id X of type `taskCancelation` that cannot be batched with any other task." + "batchCreationComplete": "task with id X of type `taskCancelation` cannot be batched" } "###); } @@ -1331,7 +1334,7 @@ async fn test_summarized_dump_creation() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), + ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1354,7 +1357,7 @@ async fn test_summarized_dump_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchStrategy": "created batch containing only task with id X of type `dumpCreation` that cannot be batched with any other task." + "batchCreationComplete": "task with id X of type `dumpCreation` cannot be batched" } "###); } From 3bef4f4413ccdaa5c769c6d95c5211032d46e22b Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 11 Jul 2025 10:16:25 +0300 Subject: [PATCH 117/312] Use Server::wait_task() instead of Index::wait_task() Signed-off-by: Martin Tzvetanov Grigorov --- crates/meilisearch/tests/batches/mod.rs | 74 ++++++++++++------------- 1 file changed, 37 insertions(+), 37 deletions(-) diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index e68eb3fc6..f763a8fc0 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -27,7 +27,7 @@ async fn error_get_unexisting_batch_status() { let server = Server::new_shared(); let index = server.unique_index(); let (task, _coder) = index.create(None).await; - index.wait_task(task.uid()).await.succeeded(); + server.wait_task(task.uid()).await.succeeded(); let (response, code) = index.get_batch(u32::MAX).await; let expected_response = json!({ @@ -46,7 +46,7 @@ async fn get_batch_status() { let server = Server::new_shared(); let index = server.unique_index(); let (task, _status_code) = index.create(None).await; - let task = index.wait_task(task.uid()).await.succeeded(); + let task = server.wait_task(task.uid()).await.succeeded(); let (_response, code) = index.get_batch(task.batch_uid()).await; assert_eq!(code, 200); } @@ -56,9 +56,9 @@ async fn list_batches() { let server = Server::new_shared(); let index = server.unique_index(); let (task, _status_code) = index.create(None).await; - index.wait_task(task.uid()).await.succeeded(); + server.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = index.create(None).await; - index.wait_task(task.uid()).await.failed(); + server.wait_task(task.uid()).await.failed(); let (response, code) = index.list_batches().await; assert_eq!(code, 200); assert_eq!( @@ -112,10 +112,10 @@ async fn list_batches_with_star_filters() { let server = Server::new().await; let index = server.index("test"); let (task, _code) = index.create(None).await; - index.wait_task(task.uid()).await.succeeded(); + server.wait_task(task.uid()).await.succeeded(); let index = server.index("test"); let (task, _code) = index.create(None).await; - index.wait_task(task.uid()).await.failed(); + server.wait_task(task.uid()).await.failed(); let (response, code) = index.service.get("/batches?indexUids=test").await; assert_eq!(code, 200); @@ -158,9 +158,9 @@ async fn list_batches_status_filtered() { let server = Server::new_shared(); let index = server.unique_index(); let (task, _status_code) = index.create(None).await; - index.wait_task(task.uid()).await.succeeded(); + server.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = index.create(None).await; - index.wait_task(task.uid()).await.failed(); + server.wait_task(task.uid()).await.failed(); let (response, code) = index.filtered_batches(&[], &["succeeded"], &[]).await; assert_eq!(code, 200, "{response}"); @@ -180,9 +180,9 @@ async fn list_batches_type_filtered() { let server = Server::new_shared(); let index = server.unique_index(); let (task, _) = index.create(None).await; - index.wait_task(task.uid()).await.succeeded(); + server.wait_task(task.uid()).await.succeeded(); let (task, _) = index.delete().await; - index.wait_task(task.uid()).await.succeeded(); + server.wait_task(task.uid()).await.succeeded(); let (response, code) = index.filtered_batches(&["indexCreation"], &[], &[]).await; assert_eq!(code, 200, "{response}"); assert_eq!(response["results"].as_array().unwrap().len(), 1); @@ -202,7 +202,7 @@ async fn list_batches_invalid_canceled_by_filter() { let server = Server::new_shared(); let index = server.unique_index(); let (task, _status_code) = index.create(None).await; - index.wait_task(task.uid()).await.succeeded(); + server.wait_task(task.uid()).await.succeeded(); let (response, code) = index.filtered_batches(&[], &[], &["0"]).await; assert_eq!(code, 200, "{response}"); @@ -214,9 +214,9 @@ async fn list_batches_status_and_type_filtered() { let server = Server::new_shared(); let index = server.unique_index(); let (task, _status_code) = index.create(None).await; - index.wait_task(task.uid()).await.succeeded(); + server.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = index.update(Some("id")).await; - index.wait_task(task.uid()).await.succeeded(); + server.wait_task(task.uid()).await.succeeded(); let (response, code) = index.filtered_batches(&["indexCreation"], &["failed"], &[]).await; assert_eq!(code, 200, "{response}"); @@ -288,7 +288,7 @@ async fn test_summarized_document_addition_or_update() { let index = server.unique_index(); let (task, _status_code) = index.add_documents(json!({ "id": 42, "content": "doggos & fluff" }), None).await; - let task = index.wait_task(task.uid()).await.succeeded(); + let task = server.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -333,7 +333,7 @@ async fn test_summarized_document_addition_or_update() { let (task, _status_code) = index.add_documents(json!({ "id": 42, "content": "doggos & fluff" }), Some("id")).await; - let task = index.wait_task(task.uid()).await.succeeded(); + let task = server.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -383,7 +383,7 @@ async fn test_summarized_delete_documents_by_batch() { let task_uid_2 = (u32::MAX - 2) as u64; let task_uid_3 = (u32::MAX - 3) as u64; let (task, _status_code) = index.delete_batch(vec![task_uid_1, task_uid_2, task_uid_3]).await; - let task = index.wait_task(task.uid()).await.failed(); + let task = server.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -425,7 +425,7 @@ async fn test_summarized_delete_documents_by_batch() { index.create(None).await; let (task, _status_code) = index.delete_batch(vec![42]).await; - let task = index.wait_task(task.uid()).await.succeeded(); + let task = server.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -473,7 +473,7 @@ async fn test_summarized_delete_documents_by_filter() { let (task, _status_code) = index.delete_document_by_filter(json!({ "filter": "doggo = bernese" })).await; - let task = index.wait_task(task.uid()).await.failed(); + let task = server.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -517,7 +517,7 @@ async fn test_summarized_delete_documents_by_filter() { index.create(None).await; let (task, _status_code) = index.delete_document_by_filter(json!({ "filter": "doggo = bernese" })).await; - let task = index.wait_task(task.uid()).await.failed(); + let task = server.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -562,7 +562,7 @@ async fn test_summarized_delete_documents_by_filter() { index.update_settings(json!({ "filterableAttributes": ["doggo"] })).await; let (task, _status_code) = index.delete_document_by_filter(json!({ "filter": "doggo = bernese" })).await; - let task = index.wait_task(task.uid()).await.succeeded(); + let task = server.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -610,7 +610,7 @@ async fn test_summarized_delete_document_by_id() { let server = Server::new_shared(); let index = server.unique_index(); let (task, _status_code) = index.delete_document(1).await; - let task = index.wait_task(task.uid()).await.failed(); + let task = server.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -652,7 +652,7 @@ async fn test_summarized_delete_document_by_id() { index.create(None).await; let (task, _status_code) = index.delete_document(42).await; - let task = index.wait_task(task.uid()).await.succeeded(); + let task = server.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -711,7 +711,7 @@ async fn test_summarized_settings_update() { "###); let (task,_status_code) = index.update_settings(json!({ "displayedAttributes": ["doggos", "name"], "filterableAttributes": ["age", "nb_paw_pads"], "sortableAttributes": ["iq"] })).await; - let task = index.wait_task(task.uid()).await.succeeded(); + let task = server.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -767,7 +767,7 @@ async fn test_summarized_index_creation() { let server = Server::new_shared(); let index = server.unique_index(); let (task, _status_code) = index.create(None).await; - let task = index.wait_task(task.uid()).await.succeeded(); + let task = server.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -805,7 +805,7 @@ async fn test_summarized_index_creation() { "###); let (task, _status_code) = index.create(Some("doggos")).await; - let task = index.wait_task(task.uid()).await.failed(); + let task = server.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -850,7 +850,7 @@ async fn test_summarized_index_deletion() { let server = Server::new_shared(); let index = server.unique_index(); let (ret, _code) = index.delete().await; - let batch = index.wait_task(ret.uid()).await.failed(); + let batch = server.wait_task(ret.uid()).await.failed(); snapshot!(batch, @r###" { @@ -881,7 +881,7 @@ async fn test_summarized_index_deletion() { // both batches may get autobatched and the deleted documents count will be wrong. let (ret, _code) = index.add_documents(json!({ "id": 42, "content": "doggos & fluff" }), Some("id")).await; - let batch = index.wait_task(ret.uid()).await.succeeded(); + let batch = server.wait_task(ret.uid()).await.succeeded(); snapshot!(batch, @r###" { @@ -904,7 +904,7 @@ async fn test_summarized_index_deletion() { "###); let (ret, _code) = index.delete().await; - let batch = index.wait_task(ret.uid()).await.succeeded(); + let batch = server.wait_task(ret.uid()).await.succeeded(); snapshot!(batch, @r###" { @@ -927,7 +927,7 @@ async fn test_summarized_index_deletion() { // What happens when you delete an index that doesn't exists. let (ret, _code) = index.delete().await; - let batch = index.wait_task(ret.uid()).await.failed(); + let batch = server.wait_task(ret.uid()).await.failed(); snapshot!(batch, @r###" { @@ -960,7 +960,7 @@ async fn test_summarized_index_update() { let index = server.unique_index(); // If the index doesn't exist yet, we should get errors with or without the primary key. let (task, _status_code) = index.update(None).await; - let task = index.wait_task(task.uid()).await.failed(); + let task = server.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -998,7 +998,7 @@ async fn test_summarized_index_update() { "###); let (task, _status_code) = index.update(Some("bones")).await; - let task = index.wait_task(task.uid()).await.failed(); + let task = server.wait_task(task.uid()).await.failed(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -1041,7 +1041,7 @@ async fn test_summarized_index_update() { index.create(None).await; let (task, _status_code) = index.update(None).await; - let task = index.wait_task(task.uid()).await.succeeded(); + let task = server.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -1079,7 +1079,7 @@ async fn test_summarized_index_update() { "###); let (task, _status_code) = index.update(Some("bones")).await; - let task = index.wait_task(task.uid()).await.succeeded(); + let task = server.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -1225,9 +1225,9 @@ async fn test_summarized_batch_cancelation() { let index = server.unique_index(); // to avoid being flaky we're only going to cancel an already finished batch :( let (task, _status_code) = index.create(None).await; - index.wait_task(task.uid()).await.succeeded(); + server.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = server.cancel_tasks(format!("uids={}", task.uid()).as_str()).await; - let task = index.wait_task(task.uid()).await.succeeded(); + let task = server.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { @@ -1275,9 +1275,9 @@ async fn test_summarized_batch_deletion() { let index = server.unique_index(); // to avoid being flaky we're only going to delete an already finished batch :( let (task, _status_code) = index.create(None).await; - index.wait_task(task.uid()).await.succeeded(); + server.wait_task(task.uid()).await.succeeded(); let (task, _status_code) = server.delete_tasks(format!("uids={}", task.uid()).as_str()).await; - let task = index.wait_task(task.uid()).await.succeeded(); + let task = server.wait_task(task.uid()).await.succeeded(); let (batch, _) = index.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { From eb6ad3ef9c4fab57d80bc127b29e25e18b4491c4 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 11 Jul 2025 10:24:25 +0300 Subject: [PATCH 118/312] Fix batch id detection Signed-off-by: Martin Tzvetanov Grigorov --- crates/meilisearch/tests/batches/mod.rs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index f763a8fc0..268147d02 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -346,6 +346,7 @@ async fn test_summarized_document_addition_or_update() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks", }, @r###" { @@ -438,6 +439,7 @@ async fn test_summarized_delete_documents_by_batch() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, + ".batchCreationComplete" => "batched all enqueued tasks", }, @r###" { @@ -1127,8 +1129,8 @@ async fn test_summarized_index_swap() { { "indexes": ["doggos", "cattos"] } ])) .await; - server.wait_task(task.uid()).await.failed(); - let (batch, _) = server.get_batch(task.uid() as u32).await; + let task = server.wait_task(task.uid()).await.failed(); + let (batch, _) = server.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", @@ -1181,8 +1183,8 @@ async fn test_summarized_index_swap() { { "indexes": [doggos_index.uid, cattos_index.uid] } ])) .await; - server.wait_task(task.uid()).await.succeeded(); - let (batch, _) = server.get_batch(task.uid() as u32).await; + let task = server.wait_task(task.uid()).await.succeeded(); + let (batch, _) = server.get_batch(task.batch_uid()).await; assert_json_snapshot!(batch, { ".uid" => "[uid]", From a39223822af690800ee0ce12806ceb41dce71f68 Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 11 Jul 2025 11:11:46 +0300 Subject: [PATCH 119/312] More tests fixes Signed-off-by: Martin Tzvetanov Grigorov --- crates/meilisearch/tests/batches/mod.rs | 48 ++++++++++++------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index 7a21f1eca..bb8f3b6aa 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -396,7 +396,7 @@ async fn test_summarized_delete_documents_by_batch() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks", + ".batchStrategy" => "batched all enqueued tasks", }, @r###" { @@ -439,7 +439,7 @@ async fn test_summarized_delete_documents_by_batch() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks", + ".batchStrategy" => "batched all enqueued tasks", }, @r###" { @@ -532,7 +532,7 @@ async fn test_summarized_delete_documents_by_filter() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks", + ".batchStrategy" => "batched all enqueued tasks", }, @r###" { @@ -667,7 +667,7 @@ async fn test_summarized_delete_document_by_id() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks", + ".batchStrategy" => "batched all enqueued tasks", }, @r###" { @@ -781,7 +781,7 @@ async fn test_summarized_index_creation() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -802,7 +802,7 @@ async fn test_summarized_index_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexCreation` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexCreation` that cannot be batched with any other task." } "###); @@ -819,7 +819,7 @@ async fn test_summarized_index_creation() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -842,7 +842,7 @@ async fn test_summarized_index_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexCreation` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexCreation` that cannot be batched with any other task." } "###); } @@ -974,7 +974,7 @@ async fn test_summarized_index_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -995,7 +995,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexUpdate` that cannot be batched with any other task." } "###); @@ -1012,7 +1012,7 @@ async fn test_summarized_index_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1035,7 +1035,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexUpdate` that cannot be batched with any other task." } "###); @@ -1055,7 +1055,7 @@ async fn test_summarized_index_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1076,7 +1076,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexUpdate` that cannot be batched with any other task." } "###); @@ -1093,7 +1093,7 @@ async fn test_summarized_index_update() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1116,7 +1116,7 @@ async fn test_summarized_index_update() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexUpdate` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexUpdate` that cannot be batched with any other task." } "###); } @@ -1140,7 +1140,7 @@ async fn test_summarized_index_swap() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1170,7 +1170,7 @@ async fn test_summarized_index_swap() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexSwap` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexSwap` that cannot be batched with any other task." } "###); @@ -1195,7 +1195,7 @@ async fn test_summarized_index_swap() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1216,7 +1216,7 @@ async fn test_summarized_index_swap() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `indexCreation` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `indexCreation` that cannot be batched with any other task." } "###); } @@ -1241,7 +1241,7 @@ async fn test_summarized_batch_cancelation() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".details.originalFilter" => "?uids=X", - ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1266,7 +1266,7 @@ async fn test_summarized_batch_cancelation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `taskCancelation` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `taskCancelation` that cannot be batched with any other task." } "###); } @@ -1336,7 +1336,7 @@ async fn test_summarized_dump_creation() { ".finishedAt" => "[date]", ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", - ".batchCreationComplete" => insta::dynamic_redaction(task_with_id_redaction), + ".batchStrategy" => insta::dynamic_redaction(task_with_id_redaction), }, @r###" { @@ -1359,7 +1359,7 @@ async fn test_summarized_dump_creation() { "duration": "[duration]", "startedAt": "[date]", "finishedAt": "[date]", - "batchCreationComplete": "task with id X of type `dumpCreation` cannot be batched" + "batchStrategy": "created batch containing only task with id X of type `dumpCreation` that cannot be batched with any other task." } "###); } From e3daa907c5fe904951f47d10886cfc7b3cfcdf8c Mon Sep 17 00:00:00 2001 From: Martin Tzvetanov Grigorov Date: Fri, 11 Jul 2025 11:14:39 +0300 Subject: [PATCH 120/312] Update redactions Signed-off-by: Martin Tzvetanov Grigorov --- crates/meilisearch/tests/batches/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch/tests/batches/mod.rs b/crates/meilisearch/tests/batches/mod.rs index bb8f3b6aa..9d6bee7c1 100644 --- a/crates/meilisearch/tests/batches/mod.rs +++ b/crates/meilisearch/tests/batches/mod.rs @@ -301,7 +301,7 @@ async fn test_summarized_document_addition_or_update() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks", + ".batchStrategy" => "batched all enqueued tasks", }, @r###" { @@ -346,7 +346,7 @@ async fn test_summarized_document_addition_or_update() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks", + ".batchStrategy" => "batched all enqueued tasks", }, @r###" { @@ -487,7 +487,7 @@ async fn test_summarized_delete_documents_by_filter() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks", + ".batchStrategy" => "batched all enqueued tasks", }, @r###" { @@ -577,7 +577,7 @@ async fn test_summarized_delete_documents_by_filter() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks" + ".batchStrategy" => "batched all enqueued tasks" }, @r###" { @@ -624,7 +624,7 @@ async fn test_summarized_delete_document_by_id() { ".stats.progressTrace" => "[progressTrace]", ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks", + ".batchStrategy" => "batched all enqueued tasks", }, @r###" { @@ -726,7 +726,7 @@ async fn test_summarized_settings_update() { ".stats.writeChannelCongestion" => "[writeChannelCongestion]", ".stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".stats.indexUids" => r#"{"[uuid]": 1}"#, - ".batchCreationComplete" => "batched all enqueued tasks" + ".batchStrategy" => "batched all enqueued tasks" }, @r###" { From 78d0625a91051155c9f9454514734fdbebaf4b77 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 11 Jul 2025 11:23:20 +0200 Subject: [PATCH 121/312] Decrease default payload size for exports --- crates/index-scheduler/src/scheduler/process_export.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/index-scheduler/src/scheduler/process_export.rs b/crates/index-scheduler/src/scheduler/process_export.rs index 2062e1c28..10d5cc11b 100644 --- a/crates/index-scheduler/src/scheduler/process_export.rs +++ b/crates/index-scheduler/src/scheduler/process_export.rs @@ -167,7 +167,7 @@ impl IndexScheduler { }, ); - let limit = payload_size.map(|ps| ps.as_u64() as usize).unwrap_or(50 * 1024 * 1024); // defaults to 50 MiB + let limit = payload_size.map(|ps| ps.as_u64() as usize).unwrap_or(20 * 1024 * 1024); // defaults to 20 MiB let documents_url = format!("{base_url}/indexes/{uid}/documents"); request_threads() From 9bdfdd395bc8a8ca690857fc9fa19bae0f8f5b6e Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 11 Jul 2025 11:29:47 +0200 Subject: [PATCH 122/312] Fix document step overflowing --- crates/index-scheduler/src/scheduler/process_export.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/index-scheduler/src/scheduler/process_export.rs b/crates/index-scheduler/src/scheduler/process_export.rs index 10d5cc11b..aeb22f441 100644 --- a/crates/index-scheduler/src/scheduler/process_export.rs +++ b/crates/index-scheduler/src/scheduler/process_export.rs @@ -276,7 +276,7 @@ impl IndexScheduler { } buffer.extend_from_slice(&tmp_buffer); - if i % 100 == 0 { + if i > 0 && i % 100 == 0 { step.fetch_add(100, atomic::Ordering::Relaxed); } } From 3f42f1a036ba94ce6f822d03e238013b41ca830e Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 11 Jul 2025 11:32:05 +0200 Subject: [PATCH 123/312] Get rid of bearer --- .../src/scheduler/process_export.rs | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/process_export.rs b/crates/index-scheduler/src/scheduler/process_export.rs index aeb22f441..80ed873b4 100644 --- a/crates/index-scheduler/src/scheduler/process_export.rs +++ b/crates/index-scheduler/src/scheduler/process_export.rs @@ -62,13 +62,14 @@ impl IndexScheduler { let ExportIndexSettings { filter, override_settings } = export_settings; let index = self.index(uid)?; let index_rtxn = index.read_txn()?; + let bearer = api_key.map(|api_key| format!("Bearer {api_key}")); // First, check if the index already exists let url = format!("{base_url}/indexes/{uid}"); let response = retry(&must_stop_processing, || { let mut request = agent.get(&url); - if let Some(api_key) = api_key { - request = request.set("Authorization", &format!("Bearer {api_key}")); + if let Some(bearer) = &bearer { + request = request.set("Authorization", bearer); } request.send_bytes(Default::default()).map_err(into_backoff_error) @@ -90,8 +91,8 @@ impl IndexScheduler { let url = format!("{base_url}/indexes"); retry(&must_stop_processing, || { let mut request = agent.post(&url); - if let Some(api_key) = api_key { - request = request.set("Authorization", &format!("Bearer {api_key}")); + if let Some(bearer) = &bearer { + request = request.set("Authorization", bearer); } let index_param = json!({ "uid": uid, "primaryKey": primary_key }); request.send_json(&index_param).map_err(into_backoff_error) @@ -103,8 +104,8 @@ impl IndexScheduler { let url = format!("{base_url}/indexes/{uid}"); retry(&must_stop_processing, || { let mut request = agent.patch(&url); - if let Some(api_key) = api_key { - request = request.set("Authorization", &format!("Bearer {api_key}")); + if let Some(bearer) = &bearer { + request = request.set("Authorization", bearer); } let index_param = json!({ "primaryKey": primary_key }); request.send_json(&index_param).map_err(into_backoff_error) @@ -122,7 +123,6 @@ impl IndexScheduler { } // Retry logic for sending settings let url = format!("{base_url}/indexes/{uid}/settings"); - let bearer = api_key.map(|api_key| format!("Bearer {api_key}")); retry(&must_stop_processing, || { let mut request = agent.patch(&url); if let Some(bearer) = bearer.as_ref() { @@ -265,9 +265,8 @@ impl IndexScheduler { let mut request = agent.post(&documents_url); request = request.set("Content-Type", "application/x-ndjson"); request = request.set("Content-Encoding", "gzip"); - if let Some(api_key) = api_key { - request = request - .set("Authorization", &(format!("Bearer {api_key}"))); + if let Some(bearer) = &bearer { + request = request.set("Authorization", bearer); } request.send_bytes(&compressed_buffer).map_err(into_backoff_error) })?; @@ -284,8 +283,8 @@ impl IndexScheduler { retry(&must_stop_processing, || { let mut request = agent.post(&documents_url); request = request.set("Content-Type", "application/x-ndjson"); - if let Some(api_key) = api_key { - request = request.set("Authorization", &(format!("Bearer {api_key}"))); + if let Some(bearer) = &bearer { + request = request.set("Authorization", bearer); } request.send_bytes(&buffer).map_err(into_backoff_error) })?; From aa09edb3fb24f0f690da698c3ccc80dd7c444337 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 11 Jul 2025 12:17:40 +0200 Subject: [PATCH 124/312] Fix errors being silently dropped --- crates/index-scheduler/src/scheduler/process_export.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/index-scheduler/src/scheduler/process_export.rs b/crates/index-scheduler/src/scheduler/process_export.rs index 80ed873b4..a951a7ca6 100644 --- a/crates/index-scheduler/src/scheduler/process_export.rs +++ b/crates/index-scheduler/src/scheduler/process_export.rs @@ -170,7 +170,7 @@ impl IndexScheduler { let limit = payload_size.map(|ps| ps.as_u64() as usize).unwrap_or(20 * 1024 * 1024); // defaults to 20 MiB let documents_url = format!("{base_url}/indexes/{uid}/documents"); - request_threads() + let results = request_threads() .broadcast(|ctx| { let index_rtxn = index .read_txn() @@ -297,6 +297,9 @@ impl IndexScheduler { Some(uid.to_string()), ) })?; + for result in results { + result?; + } step.store(total_documents, atomic::Ordering::Relaxed); } From ae26658913846a471173198dbda68b93b07e89bd Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 11 Jul 2025 12:18:24 +0200 Subject: [PATCH 125/312] Use the most appropriate unit in payload_too_large error --- crates/meilisearch/src/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/src/error.rs b/crates/meilisearch/src/error.rs index 91c6c23fa..d8cc41ad1 100644 --- a/crates/meilisearch/src/error.rs +++ b/crates/meilisearch/src/error.rs @@ -49,7 +49,7 @@ pub enum MeilisearchHttpError { TooManySearchRequests(usize), #[error("Internal error: Search limiter is down.")] SearchLimiterIsDown, - #[error("The provided payload reached the size limit. The maximum accepted payload size is {}.", Byte::from_u64(*.0 as u64).get_appropriate_unit(UnitType::Binary))] + #[error("The provided payload reached the size limit. The maximum accepted payload size is {}.", Byte::from_u64(dbg!(*.0 as u64)).get_appropriate_unit(if *.0 % 1024 == 0 { UnitType::Binary } else { UnitType::Decimal }))] PayloadTooLarge(usize), #[error("Two indexes must be given for each swap. The list `[{}]` contains {} indexes.", .0.iter().map(|uid| format!("\"{uid}\"")).collect::>().join(", "), .0.len() From 1ade76ba109fbd99441ad70d6c6876aa7ea86d9d Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 11 Jul 2025 12:23:41 +0200 Subject: [PATCH 126/312] Remove sneaky debug --- crates/meilisearch/src/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/src/error.rs b/crates/meilisearch/src/error.rs index d8cc41ad1..8d4430f07 100644 --- a/crates/meilisearch/src/error.rs +++ b/crates/meilisearch/src/error.rs @@ -49,7 +49,7 @@ pub enum MeilisearchHttpError { TooManySearchRequests(usize), #[error("Internal error: Search limiter is down.")] SearchLimiterIsDown, - #[error("The provided payload reached the size limit. The maximum accepted payload size is {}.", Byte::from_u64(dbg!(*.0 as u64)).get_appropriate_unit(if *.0 % 1024 == 0 { UnitType::Binary } else { UnitType::Decimal }))] + #[error("The provided payload reached the size limit. The maximum accepted payload size is {}.", Byte::from_u64(*.0 as u64).get_appropriate_unit(if *.0 % 1024 == 0 { UnitType::Binary } else { UnitType::Decimal }))] PayloadTooLarge(usize), #[error("Two indexes must be given for each swap. The list `[{}]` contains {} indexes.", .0.iter().map(|uid| format!("\"{uid}\"")).collect::>().join(", "), .0.len() From cfa6ba6c3b21b7317c8227525921a462cc0c68c9 Mon Sep 17 00:00:00 2001 From: kametsun Date: Sat, 12 Jul 2025 00:50:26 +0900 Subject: [PATCH 127/312] Fix stats showing wrong document count after clear all Update database stats after clearing documents to ensure /stats endpoint returns correct numberOfDocuments: 0 instead of stale count. --- crates/milli/src/update/clear_documents.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/milli/src/update/clear_documents.rs b/crates/milli/src/update/clear_documents.rs index 01631e9a3..ea07dfc3b 100644 --- a/crates/milli/src/update/clear_documents.rs +++ b/crates/milli/src/update/clear_documents.rs @@ -2,7 +2,7 @@ use heed::RwTxn; use roaring::RoaringBitmap; use time::OffsetDateTime; -use crate::{FieldDistribution, Index, Result}; +use crate::{database_stats::DatabaseStats, FieldDistribution, Index, Result}; pub struct ClearDocuments<'t, 'i> { wtxn: &'t mut RwTxn<'i>, @@ -92,6 +92,10 @@ impl<'t, 'i> ClearDocuments<'t, 'i> { documents.clear(self.wtxn)?; + // Update the stats of the documents database after clearing all documents. + let stats = DatabaseStats::new(self.index.documents.remap_data_type(), self.wtxn)?; + self.index.put_documents_stats(self.wtxn, stats)?; + Ok(number_of_documents) } } From 9a9be76757d9381d317f68ef4ab0f99629de6700 Mon Sep 17 00:00:00 2001 From: kametsun Date: Sat, 12 Jul 2025 01:09:17 +0900 Subject: [PATCH 128/312] add: verify that the statistics are correctly update assert --- crates/milli/src/update/clear_documents.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/milli/src/update/clear_documents.rs b/crates/milli/src/update/clear_documents.rs index ea07dfc3b..ff1057267 100644 --- a/crates/milli/src/update/clear_documents.rs +++ b/crates/milli/src/update/clear_documents.rs @@ -125,6 +125,9 @@ mod tests { wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); + + // Variables for statistics verification + let stats = index.documents_stats(&rtxn).unwrap().unwrap(); // the value is 7 because there is `[id, name, age, country, _geo, _geo.lng, _geo.lat]` assert_eq!(index.fields_ids_map(&rtxn).unwrap().len(), 7); @@ -146,5 +149,9 @@ mod tests { assert!(index.field_id_docid_facet_f64s.is_empty(&rtxn).unwrap()); assert!(index.field_id_docid_facet_strings.is_empty(&rtxn).unwrap()); assert!(index.documents.is_empty(&rtxn).unwrap()); + + // Verify that the statistics are correctly updated after clearing documents + assert_eq!(index.number_of_documents(&rtxn).unwrap(), 0); + assert_eq!(stats.number_of_entries(), 0); } } From 5cd61b50f944c6aefca040ba5ee1ab1049ffe463 Mon Sep 17 00:00:00 2001 From: kametsun Date: Sat, 12 Jul 2025 18:19:26 +0900 Subject: [PATCH 129/312] Fix formatting --- crates/milli/src/update/clear_documents.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/milli/src/update/clear_documents.rs b/crates/milli/src/update/clear_documents.rs index ff1057267..84eeca7f9 100644 --- a/crates/milli/src/update/clear_documents.rs +++ b/crates/milli/src/update/clear_documents.rs @@ -125,7 +125,7 @@ mod tests { wtxn.commit().unwrap(); let rtxn = index.read_txn().unwrap(); - + // Variables for statistics verification let stats = index.documents_stats(&rtxn).unwrap().unwrap(); From 662c5d98715c824200d74f8006fdf623d011af22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 24 Jun 2025 17:33:24 +0200 Subject: [PATCH 130/312] Introduce filters in the chat completions --- crates/meilisearch-types/src/error.rs | 1 + crates/meilisearch-types/src/features.rs | 3 +++ .../src/routes/chats/chat_completions.rs | 4 ++++ crates/meilisearch/src/routes/chats/settings.rs | 13 +++++++++++-- 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index c57e2d042..fb5bd4f18 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -415,6 +415,7 @@ InvalidChatCompletionPrompts , InvalidRequest , BAD_REQU InvalidChatCompletionSystemPrompt , InvalidRequest , BAD_REQUEST ; InvalidChatCompletionSearchDescriptionPrompt , InvalidRequest , BAD_REQUEST ; InvalidChatCompletionSearchQueryParamPrompt , InvalidRequest , BAD_REQUEST ; +InvalidChatCompletionSearchFilterParamPrompt , InvalidRequest , BAD_REQUEST ; InvalidChatCompletionSearchIndexUidParamPrompt , InvalidRequest , BAD_REQUEST ; InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQUEST } diff --git a/crates/meilisearch-types/src/features.rs b/crates/meilisearch-types/src/features.rs index 3c78035e8..0fabec32f 100644 --- a/crates/meilisearch-types/src/features.rs +++ b/crates/meilisearch-types/src/features.rs @@ -8,6 +8,7 @@ pub const DEFAULT_CHAT_SYSTEM_PROMPT: &str = "You are a highly capable research pub const DEFAULT_CHAT_SEARCH_DESCRIPTION_PROMPT: &str = "Search the database for relevant JSON documents using an optional query."; pub const DEFAULT_CHAT_SEARCH_Q_PARAM_PROMPT: &str = "The search query string used to find relevant documents in the index. This should contain keywords or phrases that best represent what the user is looking for. More specific queries will yield more precise results."; +pub const DEFAULT_CHAT_SEARCH_FILTER_PARAM_PROMPT: &str = "The search filter string used to find relevant documents in the index. It supports parentheses, AND, OR, NOT, EXISTS, IS EMPTY, IS NOT EMPTY. Here is an example: \"price > 100 AND category = 'electronics'\""; pub const DEFAULT_CHAT_SEARCH_INDEX_UID_PARAM_PROMPT: &str = "The name of the index to search within. An index is a collection of documents organized for search. Selecting the right index ensures the most relevant results for the user query."; #[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)] @@ -164,6 +165,7 @@ pub struct ChatCompletionPrompts { pub system: String, pub search_description: String, pub search_q_param: String, + pub search_filter_param: String, pub search_index_uid_param: String, } @@ -173,6 +175,7 @@ impl Default for ChatCompletionPrompts { system: DEFAULT_CHAT_SYSTEM_PROMPT.to_string(), search_description: DEFAULT_CHAT_SEARCH_DESCRIPTION_PROMPT.to_string(), search_q_param: DEFAULT_CHAT_SEARCH_Q_PARAM_PROMPT.to_string(), + search_filter_param: DEFAULT_CHAT_SEARCH_FILTER_PARAM_PROMPT.to_string(), search_index_uid_param: DEFAULT_CHAT_SEARCH_INDEX_UID_PARAM_PROMPT.to_string(), } } diff --git a/crates/meilisearch/src/routes/chats/chat_completions.rs b/crates/meilisearch/src/routes/chats/chat_completions.rs index 4f7087ae8..161a1b851 100644 --- a/crates/meilisearch/src/routes/chats/chat_completions.rs +++ b/crates/meilisearch/src/routes/chats/chat_completions.rs @@ -203,6 +203,10 @@ fn setup_search_tool( // "type": ["string", "null"], "type": "string", "description": prompts.search_q_param, + }, + "filter": { + "type": "string", + "description": prompts.search_filter_param, } }, "required": ["index_uid", "q"], diff --git a/crates/meilisearch/src/routes/chats/settings.rs b/crates/meilisearch/src/routes/chats/settings.rs index 38eb0d3c5..44c099c14 100644 --- a/crates/meilisearch/src/routes/chats/settings.rs +++ b/crates/meilisearch/src/routes/chats/settings.rs @@ -8,8 +8,8 @@ use meilisearch_types::error::{Code, ResponseError}; use meilisearch_types::features::{ ChatCompletionPrompts as DbChatCompletionPrompts, ChatCompletionSettings, ChatCompletionSource as DbChatCompletionSource, DEFAULT_CHAT_SEARCH_DESCRIPTION_PROMPT, - DEFAULT_CHAT_SEARCH_INDEX_UID_PARAM_PROMPT, DEFAULT_CHAT_SEARCH_Q_PARAM_PROMPT, - DEFAULT_CHAT_SYSTEM_PROMPT, + DEFAULT_CHAT_SEARCH_FILTER_PARAM_PROMPT, DEFAULT_CHAT_SEARCH_INDEX_UID_PARAM_PROMPT, + DEFAULT_CHAT_SEARCH_Q_PARAM_PROMPT, DEFAULT_CHAT_SYSTEM_PROMPT, }; use meilisearch_types::keys::actions; use meilisearch_types::milli::update::Setting; @@ -84,6 +84,11 @@ async fn patch_settings( Setting::Reset => DEFAULT_CHAT_SEARCH_Q_PARAM_PROMPT.to_string(), Setting::NotSet => old_settings.prompts.search_q_param, }, + search_filter_param: match new_prompts.search_filter_param { + Setting::Set(new_description) => new_description, + Setting::Reset => DEFAULT_CHAT_SEARCH_FILTER_PARAM_PROMPT.to_string(), + Setting::NotSet => old_settings.prompts.search_filter_param, + }, search_index_uid_param: match new_prompts.search_index_uid_param { Setting::Set(new_description) => new_description, Setting::Reset => DEFAULT_CHAT_SEARCH_INDEX_UID_PARAM_PROMPT.to_string(), @@ -252,6 +257,10 @@ pub struct ChatPrompts { #[schema(value_type = Option, example = json!("This is query parameter..."))] pub search_q_param: Setting, #[serde(default)] + #[deserr(default, error = DeserrJsonError)] + #[schema(value_type = Option, example = json!("This is filter parameter..."))] + pub search_filter_param: Setting, + #[serde(default)] #[deserr(default, error = DeserrJsonError)] #[schema(value_type = Option, example = json!("This is index you want to search in..."))] pub search_index_uid_param: Setting, From 1a9dbd364eff68b17a43df1892bcdb3d27569a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 24 Jun 2025 18:43:09 +0200 Subject: [PATCH 131/312] Fix some issues --- crates/meilisearch-types/src/features.rs | 2 +- .../src/routes/chats/chat_completions.rs | 39 ++++++++++++------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/crates/meilisearch-types/src/features.rs b/crates/meilisearch-types/src/features.rs index 0fabec32f..2fe4f7d43 100644 --- a/crates/meilisearch-types/src/features.rs +++ b/crates/meilisearch-types/src/features.rs @@ -8,7 +8,7 @@ pub const DEFAULT_CHAT_SYSTEM_PROMPT: &str = "You are a highly capable research pub const DEFAULT_CHAT_SEARCH_DESCRIPTION_PROMPT: &str = "Search the database for relevant JSON documents using an optional query."; pub const DEFAULT_CHAT_SEARCH_Q_PARAM_PROMPT: &str = "The search query string used to find relevant documents in the index. This should contain keywords or phrases that best represent what the user is looking for. More specific queries will yield more precise results."; -pub const DEFAULT_CHAT_SEARCH_FILTER_PARAM_PROMPT: &str = "The search filter string used to find relevant documents in the index. It supports parentheses, AND, OR, NOT, EXISTS, IS EMPTY, IS NOT EMPTY. Here is an example: \"price > 100 AND category = 'electronics'\""; +pub const DEFAULT_CHAT_SEARCH_FILTER_PARAM_PROMPT: &str = "The search filter string used to find relevant documents in the index. It supports parentheses, `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox`. Here is an example: \"price > 100 AND category = 'electronics'\""; pub const DEFAULT_CHAT_SEARCH_INDEX_UID_PARAM_PROMPT: &str = "The name of the index to search within. An index is a collection of documents organized for search. Selecting the right index ensures the most relevant results for the user query."; #[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)] diff --git a/crates/meilisearch/src/routes/chats/chat_completions.rs b/crates/meilisearch/src/routes/chats/chat_completions.rs index 161a1b851..830efa844 100644 --- a/crates/meilisearch/src/routes/chats/chat_completions.rs +++ b/crates/meilisearch/src/routes/chats/chat_completions.rs @@ -209,7 +209,7 @@ fn setup_search_tool( "description": prompts.search_filter_param, } }, - "required": ["index_uid", "q"], + "required": ["index_uid", "q", "filter"], "additionalProperties": false, })) .strict(true) @@ -251,11 +251,16 @@ async fn process_search_request( auth_token: &str, index_uid: String, q: Option, + filter: Option, ) -> Result<(Index, Vec, String), ResponseError> { let index = index_scheduler.index(&index_uid)?; let rtxn = index.static_read_txn()?; let ChatConfig { description: _, prompt: _, search_parameters } = index.chat_config(&rtxn)?; - let mut query = SearchQuery { q, ..SearchQuery::from(search_parameters) }; + let mut query = SearchQuery { + q, + filter: filter.map(serde_json::Value::from), + ..SearchQuery::from(search_parameters) + }; let auth_filter = ActionPolicy::<{ actions::SEARCH }>::authenticate( auth_ctrl, auth_token, @@ -399,16 +404,19 @@ async fn non_streamed_chat( for call in meili_calls { let result = match serde_json::from_str(&call.function.arguments) { - Ok(SearchInIndexParameters { index_uid, q }) => process_search_request( - &index_scheduler, - auth_ctrl.clone(), - &search_queue, - auth_token, - index_uid, - q, - ) - .await - .map_err(|e| e.to_string()), + Ok(SearchInIndexParameters { index_uid, q, filter }) => { + process_search_request( + &index_scheduler, + auth_ctrl.clone(), + &search_queue, + auth_token, + index_uid, + q, + filter, + ) + .await + .map_err(|e| e.to_string()) + } Err(err) => Err(err.to_string()), }; @@ -722,14 +730,15 @@ async fn handle_meili_tools( let mut error = None; - let result = match serde_json::from_str(&call.function.arguments) { - Ok(SearchInIndexParameters { index_uid, q }) => match process_search_request( + let answer = match serde_json::from_str(&call.function.arguments) { + Ok(SearchInIndexParameters { index_uid, q, filter }) => match process_search_request( index_scheduler, auth_ctrl.clone(), search_queue, auth_token, index_uid, q, + filter, ) .await { @@ -805,4 +814,6 @@ struct SearchInIndexParameters { index_uid: String, /// The query parameter to use. q: Option, + /// The filter parameter to use. + filter: Option, } From 34f2ab7093874ccf6617c5ff18c7e0a21e66afb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 26 Jun 2025 12:00:09 +0200 Subject: [PATCH 132/312] WIP report search errors to the LLM --- .../src/routes/chats/chat_completions.rs | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/src/routes/chats/chat_completions.rs b/crates/meilisearch/src/routes/chats/chat_completions.rs index 830efa844..1610419ed 100644 --- a/crates/meilisearch/src/routes/chats/chat_completions.rs +++ b/crates/meilisearch/src/routes/chats/chat_completions.rs @@ -289,14 +289,23 @@ async fn process_search_request( let (search, _is_finite_pagination, _max_total_hits, _offset) = prepare_search(&index_cloned, &rtxn, &query, &search_kind, time_budget, features)?; - search_from_kind(index_uid, search_kind, search) - .map(|(search_results, _)| (rtxn, search_results)) - .map_err(ResponseError::from) + match search_from_kind(index_uid, search_kind, search) { + Ok((search_results, _)) => Ok((rtxn, Ok(search_results))), + Err(MeilisearchHttpError::Milli { + error: meilisearch_types::milli::Error::UserError(user_error), + index_name: _, + }) => Ok((rtxn, Err(user_error))), + Err(err) => Err(ResponseError::from(err)), + } }) .await; permit.drop().await; - let output = output?; + let output = match output? { + Ok((rtxn, Ok(search_results))) => Ok((rtxn, search_results)), + Ok((_rtxn, Err(error))) => return Ok((index, Vec::new(), error.to_string())), + Err(err) => Err(ResponseError::from(err)), + }; let mut documents = Vec::new(); if let Ok((ref rtxn, ref search_result)) = output { MEILISEARCH_CHAT_SEARCH_REQUESTS.with_label_values(&["internal"]).inc(); @@ -730,7 +739,7 @@ async fn handle_meili_tools( let mut error = None; - let answer = match serde_json::from_str(&call.function.arguments) { + let result = match serde_json::from_str(&call.function.arguments) { Ok(SearchInIndexParameters { index_uid, q, filter }) => match process_search_request( index_scheduler, auth_ctrl.clone(), From e654f662230448662b4ee14aaf7d75c1550ae85b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Tue, 1 Jul 2025 16:24:02 +0200 Subject: [PATCH 133/312] Support filtering --- crates/meilisearch-types/src/features.rs | 4 +- .../src/routes/chats/chat_completions.rs | 53 +++++++++++++++++-- crates/milli/src/attribute_patterns.rs | 6 +++ 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch-types/src/features.rs b/crates/meilisearch-types/src/features.rs index 2fe4f7d43..006f39d15 100644 --- a/crates/meilisearch-types/src/features.rs +++ b/crates/meilisearch-types/src/features.rs @@ -4,11 +4,11 @@ use serde::{Deserialize, Serialize}; use crate::error::{Code, ResponseError}; -pub const DEFAULT_CHAT_SYSTEM_PROMPT: &str = "You are a highly capable research assistant with access to powerful search tools. IMPORTANT INSTRUCTIONS:1. When answering questions, you MUST make multiple tool calls (at least 2-3) to gather comprehensive information.2. Use different search queries for each tool call - vary keywords, rephrase questions, and explore different semantic angles to ensure broad coverage.3. Always explicitly announce BEFORE making each tool call by saying: \"I'll search for [specific information] now.\"4. Combine information from ALL tool calls to provide complete, nuanced answers rather than relying on a single source.5. For complex topics, break down your research into multiple targeted queries rather than using a single generic search."; +pub const DEFAULT_CHAT_SYSTEM_PROMPT: &str = "You are a highly capable research assistant with access to powerful search tools. IMPORTANT INSTRUCTIONS:1. When answering questions, you MUST make multiple tool calls (at least 2-3) to gather comprehensive information.2. Use different search queries for each tool call - vary keywords, rephrase questions, and explore different semantic angles to ensure broad coverage.3. Always explicitly announce BEFORE making each tool call by saying: \"I'll search for [specific information] now.\"4. Combine information from ALL tool calls to provide complete, nuanced answers rather than relying on a single source.5. For complex topics, break down your research into multiple targeted queries rather than using a single generic search. Meilisearch doesn't use the colon (:) syntax to filter but rather the equal (=) one. Separate filters from query and keep the q parameter empty if needed. Same for the filter parameter: keep it empty if need be. If you need to find documents that CONTAINS keywords simply put the keywords in the q parameter do no use a filter for this purpose. Whenever you get an error, read the error message and fix your error. "; pub const DEFAULT_CHAT_SEARCH_DESCRIPTION_PROMPT: &str = "Search the database for relevant JSON documents using an optional query."; pub const DEFAULT_CHAT_SEARCH_Q_PARAM_PROMPT: &str = "The search query string used to find relevant documents in the index. This should contain keywords or phrases that best represent what the user is looking for. More specific queries will yield more precise results."; -pub const DEFAULT_CHAT_SEARCH_FILTER_PARAM_PROMPT: &str = "The search filter string used to find relevant documents in the index. It supports parentheses, `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox`. Here is an example: \"price > 100 AND category = 'electronics'\""; +pub const DEFAULT_CHAT_SEARCH_FILTER_PARAM_PROMPT: &str = "The search filter string used to find relevant documents in the index. It supports parentheses, `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox`. Here is an example: \"price > 100 AND category = 'electronics'\". The following is a list of fields that can be filtered on: "; pub const DEFAULT_CHAT_SEARCH_INDEX_UID_PARAM_PROMPT: &str = "The name of the index to search within. An index is a collection of documents organized for search. Selecting the right index ensures the most relevant results for the user query."; #[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)] diff --git a/crates/meilisearch/src/routes/chats/chat_completions.rs b/crates/meilisearch/src/routes/chats/chat_completions.rs index 1610419ed..b879f85f8 100644 --- a/crates/meilisearch/src/routes/chats/chat_completions.rs +++ b/crates/meilisearch/src/routes/chats/chat_completions.rs @@ -27,9 +27,10 @@ use meilisearch_types::features::{ ChatCompletionPrompts as DbChatCompletionPrompts, ChatCompletionSource as DbChatCompletionSource, SystemRole, }; +use meilisearch_types::heed::RoTxn; use meilisearch_types::keys::actions; use meilisearch_types::milli::index::ChatConfig; -use meilisearch_types::milli::{all_obkv_to_json, obkv_to_json, TimeBudget}; +use meilisearch_types::milli::{all_obkv_to_json, obkv_to_json, OrderBy, TimeBudget}; use meilisearch_types::{Document, Index}; use serde::Deserialize; use serde_json::json; @@ -169,6 +170,7 @@ fn setup_search_tool( let mut index_uids = Vec::new(); let mut function_description = prompts.search_description.clone(); + let mut filter_description = prompts.search_filter_param.clone(); index_scheduler.try_for_each_index::<_, ()>(|name, index| { // Make sure to skip unauthorized indexes if !filters.is_index_authorized(name) { @@ -180,16 +182,22 @@ fn setup_search_tool( let index_description = chat_config.description; let _ = writeln!(&mut function_description, "\n\n - {name}: {index_description}\n"); index_uids.push(name.to_string()); + let facet_distributions = format_facet_distributions(&index, &rtxn, 10).unwrap(); // TODO do not unwrap + let _ = writeln!(&mut filter_description, "\n## Facet distributions of the {name} index"); + let _ = writeln!(&mut filter_description, "{facet_distributions}"); Ok(()) })?; + tracing::debug!("LLM function description: {function_description}"); + tracing::debug!("LLM filter description: {filter_description}"); + let tool = ChatCompletionToolArgs::default() .r#type(ChatCompletionToolType::Function) .function( FunctionObjectArgs::default() .name(MEILI_SEARCH_IN_INDEX_FUNCTION_NAME) - .description(&function_description) + .description(function_description) .parameters(json!({ "type": "object", "properties": { @@ -206,7 +214,7 @@ fn setup_search_tool( }, "filter": { "type": "string", - "description": prompts.search_filter_param, + "description": filter_description, } }, "required": ["index_uid", "q", "filter"], @@ -261,6 +269,9 @@ async fn process_search_request( filter: filter.map(serde_json::Value::from), ..SearchQuery::from(search_parameters) }; + + tracing::debug!("LLM query: {:?}", query); + let auth_filter = ActionPolicy::<{ actions::SEARCH }>::authenticate( auth_ctrl, auth_token, @@ -826,3 +837,39 @@ struct SearchInIndexParameters { /// The filter parameter to use. filter: Option, } + +fn format_facet_distributions( + index: &Index, + rtxn: &RoTxn, + max_values_per_facet: usize, +) -> meilisearch_types::milli::Result { + let universe = index.documents_ids(&rtxn)?; + let rules = index.filterable_attributes_rules(&rtxn)?; + let fields_ids_map = index.fields_ids_map(&rtxn)?; + let filterable_attributes = fields_ids_map + .names() + .filter(|name| rules.iter().any(|rule| rule.match_str(name).matches())) + .map(|name| (name, OrderBy::Count)); + let facets_distribution = index + .facets_distribution(&rtxn) + .max_values_per_facet(max_values_per_facet) + .candidates(universe) + .facets(filterable_attributes) + .execute()?; + + let mut output = String::new(); + for (facet_name, entries) in facets_distribution { + let _ = write!(&mut output, "{}: ", facet_name); + let total_entries = entries.len(); + for (i, (value, count)) in entries.into_iter().enumerate() { + let _ = if total_entries.saturating_sub(1) == i { + write!(&mut output, "{} ({}).", value, count) + } else { + write!(&mut output, "{} ({}), ", value, count) + }; + } + let _ = writeln!(&mut output); + } + + Ok(output) +} diff --git a/crates/milli/src/attribute_patterns.rs b/crates/milli/src/attribute_patterns.rs index 8da6942a3..d879cb2c3 100644 --- a/crates/milli/src/attribute_patterns.rs +++ b/crates/milli/src/attribute_patterns.rs @@ -130,6 +130,12 @@ pub enum PatternMatch { NoMatch, } +impl PatternMatch { + pub fn matches(&self) -> bool { + matches!(self, PatternMatch::Match) + } +} + #[cfg(test)] mod tests { use super::*; From d76dcc8998738dee3c072e1244a4668294f1cb3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Mon, 7 Jul 2025 11:40:52 +0200 Subject: [PATCH 134/312] Make clippy happy --- .../meilisearch/src/routes/chats/chat_completions.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch/src/routes/chats/chat_completions.rs b/crates/meilisearch/src/routes/chats/chat_completions.rs index b879f85f8..799f4c9f0 100644 --- a/crates/meilisearch/src/routes/chats/chat_completions.rs +++ b/crates/meilisearch/src/routes/chats/chat_completions.rs @@ -182,7 +182,7 @@ fn setup_search_tool( let index_description = chat_config.description; let _ = writeln!(&mut function_description, "\n\n - {name}: {index_description}\n"); index_uids.push(name.to_string()); - let facet_distributions = format_facet_distributions(&index, &rtxn, 10).unwrap(); // TODO do not unwrap + let facet_distributions = format_facet_distributions(index, &rtxn, 10).unwrap(); // TODO do not unwrap let _ = writeln!(&mut filter_description, "\n## Facet distributions of the {name} index"); let _ = writeln!(&mut filter_description, "{facet_distributions}"); @@ -315,7 +315,7 @@ async fn process_search_request( let output = match output? { Ok((rtxn, Ok(search_results))) => Ok((rtxn, search_results)), Ok((_rtxn, Err(error))) => return Ok((index, Vec::new(), error.to_string())), - Err(err) => Err(ResponseError::from(err)), + Err(err) => Err(err), }; let mut documents = Vec::new(); if let Ok((ref rtxn, ref search_result)) = output { @@ -843,15 +843,15 @@ fn format_facet_distributions( rtxn: &RoTxn, max_values_per_facet: usize, ) -> meilisearch_types::milli::Result { - let universe = index.documents_ids(&rtxn)?; - let rules = index.filterable_attributes_rules(&rtxn)?; - let fields_ids_map = index.fields_ids_map(&rtxn)?; + let universe = index.documents_ids(rtxn)?; + let rules = index.filterable_attributes_rules(rtxn)?; + let fields_ids_map = index.fields_ids_map(rtxn)?; let filterable_attributes = fields_ids_map .names() .filter(|name| rules.iter().any(|rule| rule.match_str(name).matches())) .map(|name| (name, OrderBy::Count)); let facets_distribution = index - .facets_distribution(&rtxn) + .facets_distribution(rtxn) .max_values_per_facet(max_values_per_facet) .candidates(universe) .facets(filterable_attributes) From d694e312ff6061b435b37e0c6ac9942076825185 Mon Sep 17 00:00:00 2001 From: Many the fish Date: Tue, 15 Jul 2025 11:54:59 +0200 Subject: [PATCH 135/312] Update crates/milli/src/update/settings.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Clément Renault --- crates/milli/src/update/settings.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/milli/src/update/settings.rs b/crates/milli/src/update/settings.rs index fdc21797f..d2f74da2a 100644 --- a/crates/milli/src/update/settings.rs +++ b/crates/milli/src/update/settings.rs @@ -557,7 +557,7 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> { let old_fields = self.index.user_defined_searchable_fields(self.wtxn)?; let did_change = { let new_fields = fields.iter().map(String::as_str).collect::>(); - old_fields.map(|old| new_fields != old).unwrap_or(true) + old_fields.is_none_or(|old| new_fields != old) }; if !did_change { return Ok(false); From 2a015ac3b821545b3e2fab8c7424a6584d536f17 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 15 Jul 2025 14:50:10 +0200 Subject: [PATCH 136/312] Implement basic few shot prompting to improve the query capabilities --- crates/meilisearch-types/src/features.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch-types/src/features.rs b/crates/meilisearch-types/src/features.rs index 006f39d15..d1a6b61d8 100644 --- a/crates/meilisearch-types/src/features.rs +++ b/crates/meilisearch-types/src/features.rs @@ -6,7 +6,7 @@ use crate::error::{Code, ResponseError}; pub const DEFAULT_CHAT_SYSTEM_PROMPT: &str = "You are a highly capable research assistant with access to powerful search tools. IMPORTANT INSTRUCTIONS:1. When answering questions, you MUST make multiple tool calls (at least 2-3) to gather comprehensive information.2. Use different search queries for each tool call - vary keywords, rephrase questions, and explore different semantic angles to ensure broad coverage.3. Always explicitly announce BEFORE making each tool call by saying: \"I'll search for [specific information] now.\"4. Combine information from ALL tool calls to provide complete, nuanced answers rather than relying on a single source.5. For complex topics, break down your research into multiple targeted queries rather than using a single generic search. Meilisearch doesn't use the colon (:) syntax to filter but rather the equal (=) one. Separate filters from query and keep the q parameter empty if needed. Same for the filter parameter: keep it empty if need be. If you need to find documents that CONTAINS keywords simply put the keywords in the q parameter do no use a filter for this purpose. Whenever you get an error, read the error message and fix your error. "; pub const DEFAULT_CHAT_SEARCH_DESCRIPTION_PROMPT: &str = - "Search the database for relevant JSON documents using an optional query."; + "Query: 'best story about Rust before 2018' with year: 2018 (334), 2020 (212), 2021 (210)\r\nlabel: analysis (500), golang (435), javascript (545)\r\ntype: story (760), link (989)\r\nvote: 300 (1), 298 (1), 278 (3)\r\n: {\"q\": \"\", \"filter\": \"category = Rust AND year < 2018 AND vote > 100\"}\r\nQuery: 'A black or green car that can go fast with red brakes'\r\nmaxspeed_kmh: 200 (100), 150 (300), 130 (330)\r\ncolor: black (300), grey (250), red (200), green (50)\r\nbrand: Toyota (300), Renault (100), Jeep (98), Ferrari (50)\r\n: {\"q\": \"red brakes\", \"filter\": \"maxspeed > 150 AND color IN ['black', green]\"}\r\nQuery: 'Superman movie released in 2018 or after' with year: 2018 (334), 2020 (212), 2021 (210)\r\ngenres: Drama (218), Comedy (220), Adventure (210), Fiction (196)\r\n: {\"q\":\"Superman\",\"filter\":\"genres IN [Adventure, Fiction] AND year >= 2018\"}"; pub const DEFAULT_CHAT_SEARCH_Q_PARAM_PROMPT: &str = "The search query string used to find relevant documents in the index. This should contain keywords or phrases that best represent what the user is looking for. More specific queries will yield more precise results."; pub const DEFAULT_CHAT_SEARCH_FILTER_PARAM_PROMPT: &str = "The search filter string used to find relevant documents in the index. It supports parentheses, `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox`. Here is an example: \"price > 100 AND category = 'electronics'\". The following is a list of fields that can be filtered on: "; pub const DEFAULT_CHAT_SEARCH_INDEX_UID_PARAM_PROMPT: &str = "The name of the index to search within. An index is a collection of documents organized for search. Selecting the right index ensures the most relevant results for the user query."; From 0791506124aa7aeff7f0fc715bab661eabe54fcd Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Tue, 15 Jul 2025 17:10:45 +0200 Subject: [PATCH 137/312] Fix some proposals --- crates/meilisearch-types/src/features.rs | 2 +- .../meilisearch/src/routes/chats/chat_completions.rs | 10 +++++----- crates/milli/src/attribute_patterns.rs | 6 ------ 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/crates/meilisearch-types/src/features.rs b/crates/meilisearch-types/src/features.rs index d1a6b61d8..8878a8281 100644 --- a/crates/meilisearch-types/src/features.rs +++ b/crates/meilisearch-types/src/features.rs @@ -6,7 +6,7 @@ use crate::error::{Code, ResponseError}; pub const DEFAULT_CHAT_SYSTEM_PROMPT: &str = "You are a highly capable research assistant with access to powerful search tools. IMPORTANT INSTRUCTIONS:1. When answering questions, you MUST make multiple tool calls (at least 2-3) to gather comprehensive information.2. Use different search queries for each tool call - vary keywords, rephrase questions, and explore different semantic angles to ensure broad coverage.3. Always explicitly announce BEFORE making each tool call by saying: \"I'll search for [specific information] now.\"4. Combine information from ALL tool calls to provide complete, nuanced answers rather than relying on a single source.5. For complex topics, break down your research into multiple targeted queries rather than using a single generic search. Meilisearch doesn't use the colon (:) syntax to filter but rather the equal (=) one. Separate filters from query and keep the q parameter empty if needed. Same for the filter parameter: keep it empty if need be. If you need to find documents that CONTAINS keywords simply put the keywords in the q parameter do no use a filter for this purpose. Whenever you get an error, read the error message and fix your error. "; pub const DEFAULT_CHAT_SEARCH_DESCRIPTION_PROMPT: &str = - "Query: 'best story about Rust before 2018' with year: 2018 (334), 2020 (212), 2021 (210)\r\nlabel: analysis (500), golang (435), javascript (545)\r\ntype: story (760), link (989)\r\nvote: 300 (1), 298 (1), 278 (3)\r\n: {\"q\": \"\", \"filter\": \"category = Rust AND year < 2018 AND vote > 100\"}\r\nQuery: 'A black or green car that can go fast with red brakes'\r\nmaxspeed_kmh: 200 (100), 150 (300), 130 (330)\r\ncolor: black (300), grey (250), red (200), green (50)\r\nbrand: Toyota (300), Renault (100), Jeep (98), Ferrari (50)\r\n: {\"q\": \"red brakes\", \"filter\": \"maxspeed > 150 AND color IN ['black', green]\"}\r\nQuery: 'Superman movie released in 2018 or after' with year: 2018 (334), 2020 (212), 2021 (210)\r\ngenres: Drama (218), Comedy (220), Adventure (210), Fiction (196)\r\n: {\"q\":\"Superman\",\"filter\":\"genres IN [Adventure, Fiction] AND year >= 2018\"}"; + "Query: 'best story about Rust before 2018' with year: 2018, 2020, 2021\nlabel: analysis, golang, javascript\ntype: story, link\nvote: 300, 298, 278\n: {\"q\": \"\", \"filter\": \"category = Rust AND type = story AND year < 2018 AND vote > 100\"}\nQuery: 'A black or green car that can go fast with red brakes' with maxspeed_kmh: 200, 150, 130\ncolor: black, grey, red, green\nbrand: Toyota, Renault, Jeep, Ferrari\n: {\"q\": \"red brakes\", \"filter\": \"maxspeed_kmh > 150 AND color IN ['black', green]\"}\nQuery: 'Superman movie released in 2018 or after' with year: 2018, 2020, 2021\ngenres: Drama, Comedy, Adventure, Fiction\n: {\"q\":\"Superman\",\"filter\":\"genres IN [Adventure, Fiction] AND year >= 2018\"}"; pub const DEFAULT_CHAT_SEARCH_Q_PARAM_PROMPT: &str = "The search query string used to find relevant documents in the index. This should contain keywords or phrases that best represent what the user is looking for. More specific queries will yield more precise results."; pub const DEFAULT_CHAT_SEARCH_FILTER_PARAM_PROMPT: &str = "The search filter string used to find relevant documents in the index. It supports parentheses, `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox`. Here is an example: \"price > 100 AND category = 'electronics'\". The following is a list of fields that can be filtered on: "; pub const DEFAULT_CHAT_SEARCH_INDEX_UID_PARAM_PROMPT: &str = "The name of the index to search within. An index is a collection of documents organized for search. Selecting the right index ensures the most relevant results for the user query."; diff --git a/crates/meilisearch/src/routes/chats/chat_completions.rs b/crates/meilisearch/src/routes/chats/chat_completions.rs index 799f4c9f0..b636678f5 100644 --- a/crates/meilisearch/src/routes/chats/chat_completions.rs +++ b/crates/meilisearch/src/routes/chats/chat_completions.rs @@ -30,7 +30,7 @@ use meilisearch_types::features::{ use meilisearch_types::heed::RoTxn; use meilisearch_types::keys::actions; use meilisearch_types::milli::index::ChatConfig; -use meilisearch_types::milli::{all_obkv_to_json, obkv_to_json, OrderBy, TimeBudget}; +use meilisearch_types::milli::{all_obkv_to_json, obkv_to_json, OrderBy, PatternMatch, TimeBudget}; use meilisearch_types::{Document, Index}; use serde::Deserialize; use serde_json::json; @@ -848,7 +848,7 @@ fn format_facet_distributions( let fields_ids_map = index.fields_ids_map(rtxn)?; let filterable_attributes = fields_ids_map .names() - .filter(|name| rules.iter().any(|rule| rule.match_str(name).matches())) + .filter(|name| rules.iter().any(|rule| matches!(rule.match_str(name), PatternMatch::Match))) .map(|name| (name, OrderBy::Count)); let facets_distribution = index .facets_distribution(rtxn) @@ -861,11 +861,11 @@ fn format_facet_distributions( for (facet_name, entries) in facets_distribution { let _ = write!(&mut output, "{}: ", facet_name); let total_entries = entries.len(); - for (i, (value, count)) in entries.into_iter().enumerate() { + for (i, (value, _count)) in entries.into_iter().enumerate() { let _ = if total_entries.saturating_sub(1) == i { - write!(&mut output, "{} ({}).", value, count) + write!(&mut output, "{value}.") } else { - write!(&mut output, "{} ({}), ", value, count) + write!(&mut output, "{value}, ") }; } let _ = writeln!(&mut output); diff --git a/crates/milli/src/attribute_patterns.rs b/crates/milli/src/attribute_patterns.rs index d879cb2c3..8da6942a3 100644 --- a/crates/milli/src/attribute_patterns.rs +++ b/crates/milli/src/attribute_patterns.rs @@ -130,12 +130,6 @@ pub enum PatternMatch { NoMatch, } -impl PatternMatch { - pub fn matches(&self) -> bool { - matches!(self, PatternMatch::Match) - } -} - #[cfg(test)] mod tests { use super::*; From 77138a42d61f0a4d31b6b17a00e5b4c0286aaa58 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 15 Jul 2025 17:04:08 +0200 Subject: [PATCH 138/312] Apply review suggestions Add preconditions Fix underflow Remove unwrap Turn methods to associated functions Apply review suggestions --- crates/meilisearch/db.snapshot | Bin 0 -> 174088 bytes .../src/analytics/mock_analytics.rs | 2 - crates/meilisearch/src/analytics/mod.rs | 6 -- .../src/routes/indexes/documents.rs | 55 ++--------- crates/milli/src/documents/sort.rs | 90 ++++++++++-------- 5 files changed, 58 insertions(+), 95 deletions(-) create mode 100644 crates/meilisearch/db.snapshot diff --git a/crates/meilisearch/db.snapshot b/crates/meilisearch/db.snapshot new file mode 100644 index 0000000000000000000000000000000000000000..29377ce4225430d3481206f468db3826b1ccb8ba GIT binary patch literal 174088 zcmb@N<8vL1^Y$Axwr!l`#I|kQIYDEyv2EMNNgCUB)1Dot@eF z?6udD#=w31UwmDBAO=$XcE>}~e(Ek*zj*8UC3vN++(RnZu(`JO6wG2$N)}s{b(v+; zTH(9A+1%Bf4w_CtlVYR6qYOi%Foj2_xI-7S7^KBqy^t3PB#V8f4uv1hG!c>{RfQu> zS_qh})7N}jnJyerx{Y9VF9mffX>N8VxXuYaZQRKqBYX{sCUZiVtpF^<*Goo5Awy>MzC|R25WhJ{$xusSC)SvkWw@u&O71c{_z6!pP zTD`-vD~-s`fM~)JU|I60#h22F12pwL-W7Gz!@dA{&wVe?uqw)*Z)ccar#E;J+nVQ} zH~G)j&(tuxkORI#G!kf{(CGhHY2d3D?RoPn;rc6q@s1dmYq8Jr#1RXYEEtX=^8YG* z5tGlp6JXcZrAxJOyfpq<8lHQI&m+k#3!F;Nse9zC7e4r84^A-K%Qb z-uC$SOyQuO^uJnb;l7)){C+NY=r_lbjQp*@|Kg$*$xXeU=CA)ttuqU&x|Pa`A?@XU zXUA_VYjPInE=!}Ht&*NrgBqYQrfFDNrKog-3zjYNP;?Q{r{^h)^}1PZQJ>@wQ&6j{ zQJm%VZa3_sccg zy9rF0i^$RiNccEWH+6Wc~-Ka0^jdWD0H6=10=lU%A?J2n8An9Iuo-4(!&N) z(=!>kN5dUvLZ}TSA+Wi;Plm5bavSQv)MoWGJ5UjoWwf`~KOG#>a*&BLbv+~^Hn3d< znCQFJmVkYSc(;lMbatrE1h{OYSK!>DaNhZ~{JX3MQMSn2RxQcG_1g-TZl_rKVsh4sy|akgVsa{$9sv?3h#=hN zT+BAaw9H~D6-i$Q`|HBdkF~JST}Bj)R#TsfV*r`_#;GVL``AE3rf3FN$>~kyYp42t zZ&#l@#{_O^=FN)WAKN5kNQ?tQt(~j{7VlZn27+PU|*}vv^-O8~!d0S!hs?r$pWl&NDow%U*u%<9t$**oNA+qzH$UAe# zqyIb$|0%bKH4H^czpLr+$b`fQ8H3*yPY&?mNoQe#Qii z%@Vqd>L3gCy)8`jwHG-Y!XOK&b8*XrH;wvlzc6E#mQ=dMZN<9MD244s7QbtpA`}X0 zD%IeK`$I{|X)9`ezqL(Oo>`dtyNB}sNfi&kSOS*Dt`pHZ8mL!*T!!xNQs()r6 zelPBx$GNCw1m!@G(jAt^rw4Tl{OyMHk`&Sh;3O&}@OKcYl*v`zeqqW(qNjfJ0+keb z*8pS3l9!}J7zqeB!fpZ|ya@*>7|ah|qOAtA-%Br&(xjh283n0DQ0yDl_)y6xP;Oxs z6`&Y%oH(fHtYpvx8`2IAR5l8d$dcP2qYzyC2mv19n&p_(zdwr9W#_moT=elX7FAwE zW97WO5c_0CXFwb5hA1hpgk=9R>^>Q zlb|c?5Wz@$5y59hLv>1&PkgWLUYD z0cR0wHI~+@sTiOcy{4pjrKyM=#|5(~%`H~P*2kwZfc7h)P zuf%@KKPO6TE1}B_ar;E9Qi~O*R#c%ie_BEr6JPZmZrn{j3i17^x zM+op|ZtGBM@7=4Cpz6S-`+MevPC_QzR4|bD2mizQHw|)zctRvo*ba#Gn`9hk##P-S zYZMEaB4k4>l3b&(+iDjBFGlBTDR`)c*9B4QCnLq&29j6U=_Vsd>TuqyVONMk$k1pr zLG?T`^Cx9$zHh@aj7aiGD>%gOsF_w^oodgaOIl?_OWdlL$$5;TkC}EbUAtP?f@Llh zJKMB$Mya97Bh8TLqG8^ON9(C%+9@tPQegD|=uYg{EGVd4N-COEli`Peirjk@G z3drsykT=Rl#7pecF2_=iLV#r-ZoV@mKsg}m^!AX>LaC;uM5Bj;Yn=95H}@_QtOugf z(MQ+OBLSl8!2IBcbfh=E)$l}Tm?}+HtRby1&75?6ZBM*DjO$8Go@gH#U$fOlO)e%o zJp6wX4rG4><{4{C`c>tWtm!#1Oe+_L*J`+=!ySgN*OB7>IVj69_5@te@)0g z))yi0zR5>LcHn&Lij*j33w?HWx_f#?zm}FJa9zXMxTd;ahXc_KSRS=@O-${}1-^vOhrd5t{EkM#qJF_GVrp!r=*iy5MAw zI#g;yGwi}P>O``%F!+m*q~wgtAUJ9fvY^mll(N+1ZO6uwi577Uk})5RVl8Mlz+{ei zyd}tBjM;bk-Z@IJoHt87RSLURfsFaWPJwv+214!iByHSnB0_A zrFJrV6-Rm9ZM@ss0{KrF#SDSpoX2-7Anl0gj_0bAVKM$ zzZukUj)ixo*q#Lw-@B*m!W%d|3-8#5Gz^9Q z8Fu(OUkGPXXn=M-!nx^<9!{79Nijy;ic?n_64P3KRJcoF)r8oX>Xa+E1$)T-v6}&h zY7z={1y4VO1diq;ji$}Vi~TbFV{fn;KS+Cfbl`A;b>mde(J`L#JX#9n8NU~Yy<&yT z8C(sy2RG>x*CRX68tIG3zNRL*f!qQXp{f-)`F;bpZ!(${X4qDgV*i*?lZ@NNK$`Yv zN>{ms@l_uC)ZVD_b__`>Q6XN$aM(#(*##v$cOF3r`@Ph2eLKsml z>wA!s$Ko$?$<#KbL|&{d2Es!APO(L}s%#+QpE@kNcxq(bb_6%e_2~L0;5+3bQeSk9 z`YQ>_Nf-PnzZ*u$nTwn{vzzknX^*M0GZI$8UPX|mrTn|90?ri?Ew`4*E>8Z=T4RYy zlU<>cZtHN2C^}5ELgyZ-Ze?n7V?gHkk?2}nG}Q^s2D34#vQaPEU%X^h=Rd|c|KuSB z>bi325>Xym{5;lJkxwZ0)d;7C+M6?X=7-G9nQp#Qd#O=5lrlS`NXS)a+0W1~hC#~^%eQHZ$DguPUzZ8_6g+4OeF1WYGc z3TpeSve-5Lqs1Dg9bAf3DEC$6^`2vWH)=834tJc>nX$1}{7u14Ji-*V!}g^uZ4b>- zSpT|%=tD*&09k-vV~1l>#+F6V?eT=56k1UYi|}_U7=E$Ytbby;+EzIF!9P~EnL_;2 z3PRWcEmY^Ea!TVO6=2*CWNX)T!_#2py)h$NNHI-%rQ6D(Y|~nV3%pfJ5rHVKte2^? zMj-MZg3oB!0`Sth_Nw#{K1jN`A%F*A@4*`6GRH-<@wc}U2+MT?O>GQhe#4ftONN$^_<1Ch=%-=1TF~c6-f2u2 z8nI-DuyU?#k8Uv{hwv^Lh92P5y60Gba+M;lIhMahF2}A@wg;Tcw3R-3X^?6Se=bsg zoLS^l4v=@i2UDLkyb8auas`Y16#4Za@=Hrb=KBD3nQ>rZtz5083%1jj8qCpUS*6+W za3te3l;cbaKf_JHyuumIZ2-KEkGYi^x8e=>0_hQxdSjCvu|q)R>=VWcrOblZ`)lUr zfO?~KSbZzkq`f1;+)a1@fW-Sy6bvXc*%qn1SWxBSL6}U$hW_K5@EH6-DE!(UhAiXR3UsM$az^{4$x!)o@JWbT0x};5_x`;>Ty)(>|=7eB#*@z2F>)rmyh=NW5hoqy@U{8XH zWoHpezbYd_#-{90>abuLg-bK#$ev%H*9+wR5xfeP$XEaU*;upxhQQB|eAQ%|IQ}z{ zALg0y0pCxR%$LK%pXfuMyzbS@5q+|9RobiREa(0OBsw8nPCI3>kIdK4kmshZ6DTF1^M5op+KPOu&x*Rff{d<%iRScj zPiGwZ>r0L`RnyE=K>fqpWfj-Rcj{n^YD-V{1O5z1j7o}RNMqYjo?SRdp9a2I^Fin9 zQPT_kCscbFW>$}M28^OR__7s%cd70>@Qx2e^Z7%@R_? zyCdr_Wdh%K16m&12b00xCW z{$Vn~eOySVl3zLYg`BL1m|PK8E2}d~>?otS>3&{9nKy;-c)_0Mx|zCjb~)$iXoj?r zr(v4wk1~Duwn*|#2{@E_Jg+xfa`AH(ej^`ZoAF!Dq$1-t6cTCduX8fZ&Y`Yhr%CgA zjWT7FB%(QkVO(gaS?MQ-&-@udKbm!5=hH5d=(x?dt{Jn5q-f~+2Pu(0<4eN4Akm;~LF_*+5N;<156wU`@b0O7w;HYIc zwxB3;vS*aAy`{R5%VfI+ueJ&Z>phIbk-Sp-f>Vh}Ca18UiG5A;-qfD!_wKP^KSgGQ%tSUQgn=JGIOB}mM zw#Elim(&qUep|sA?OVf6cXx)sdYgx2SYv7t#{o!u8#(- z9+P^Au`@zGi3pe<*Ca%Oe}2Zf0>PEJYl2%#2xk_KB0grrWttv3Ih_t|?VUTC4?bx@3A z3%hNAF6)|8Q-Hkx5hae&z_|M7KS4=U zKEF<9LI2gXjC=!_jg>TRLBT%Tvz-Z8p?#$I-!##^zUxteL^85W_LwQ~;Y3*Qy7)w? zlP&~wVoBm%KAPNinxd@B3KQv;vLYyX1s~X4!%V90>wz?hz|uz1d{e{BfT#h?k)$v{ z1(0MseX2V^mgJ^+Ee%L-zb=;C_#JDL%P=_+`{c@ni@^wDSOP0!-y7B@ho$^u&CKR| zhV_V8$~O%S?~3;Tkx@F?k4G> z!n5Y~zqopHQ&KhqBi_j}7pwlyMvrCDBa_w0S;~Bqf~%!;X2IkVeoy6L;$q96NxyTD zFPj)_9!7eFA$PTUZ7s95<-nbSo_Z>;tl%$D1p}_1&Wk-wYC_&E{;CSIyP31;0n(vK|Ro1Lfa84yI2y5Igs9M zBb8&5SXrqZ{K2-8vIUv7n^06WJEZM4f4H)gl9*>{ougBp&_UgC+eN$mwu0|A;m%#< z=DJ&u)Wv|zka<&Y(QC!EX_1WD@wXUZGsES07F5i0d$z-rdAYJFxPfr0tRdhc8Y0h7 zrxWC)WmRUO?}E(4NeL4%CBe~kWd7p&)6SJ@*A;^Lk(|1kTy&d5z50Q5hD#J)oxUHF ztYE+LhQKjmBb&-oi*1t%cJcPx<2TiT%=FlOO|guI{*J+kE>#$hIS7yfFU*aLRefZ- zEDr15dkD`fT64C;9L2!X*e#veK*|L!onez!`n2p+LvuW?TetU@xLL&cX}`04f8Io6 z=LN`ugkHfCu-laq(mQksH%_%bTfOn>;y>#ELfC95(e)k1Bc>dA z9m-HCh<++O$nYU6q2}cIH$9r%&j554V`jC2!J2Oiw|H&7xW(6S7vx${(r)E0x+h^% z*RgyB$633N2S`1}E{+WrxJAWV;)E*pF6EW9rnCSIOMc^F)d!8BjJb61*i;IlS7Rjq zocNj=FJd1jX4|-8fP_ZSrH7Ck53yxv{#4-@r+Dw?G|snHhE#A~*&9^GKVl%X+!la#y@gH-(ycEKZJ#u*a3o#-27Tr+>5g6L=1eW7`tPHJ7d?(6Cv+9QO_=RENzwuLP z(f5-&pCv zcd?}|(8vGjV7jZVC{Ul2?6iEAZ9I5(iTPqmP`GQBIsg3i@?y|uoq2HmUWR9W(-hm8 z!K>{7iz@{v_1HBhXqPkE4xVvun^FBh*z5pH6}8_PO(>>gGzQFKaxr?uFkFA9mZyB$ zxl~uSr8%a8K)|LhTd$_Ze3KSe#0S&MI-NK_@8>H9U>&j;fC&M#cIq|pK*G&zjxhbgfKTGQ?@lYy5g9P!05P9~D zB#;elP_n;=mQ+`!zV%ceT!KzR@vUbo=AZsqQVW}%rOPnv6z2m+x&K5G3K zD4iMn=GR*8;hMfEjt-(*xIs@ZZvUhq_LQ4Le*0ZTl}RObtA-ti;PdGLU}<`lo&~5R z2RGV}QgdaKuJ|I9eVuqq*AZj^K}U7#~O2NL!& z9W7`lDI;XIlf`}6DhZcAW~tEre$YsfM>+-%T0Uc37H*#J*U-)*k^d@0^~O^;Mu*`@ z0 zR;<J?8{scTyOoRPM;N4qqY;9M#byDsx-lQNE*cl|f1twT`%Psn zC;w(uw3P*L9;a=adMl)lgwD(Pzf=&!X3Z4r@T-SNL!L~779B}?ox%L4Y>F3utYGte z6t+&6XeTh7IJm6{qQbW;)>Hcyb@0;uTxk1;V!JJ}?D7+Xo?7V8j+aFHfx+MHcgz}!KgSNv` z$L?pOh+)d5AmwAmuHdS>}FJs%i&~_Mi#2j6&!UdtL^UG&9k2hc|P;=dTYWD z=TVm(j}-|~^s<_+eJy~N$ahh=Xetcfl*MH^d4qNd>0#prnI*G7ncR^=YthNP8~Z29 zdHy^_-`(hX%qK>x6UrN$y10Pj*!~n=yUo2A&jkwaWnjxqM;>zX^knX%)nhdZz##Oz!10Ob|ceV2K$1CQ1&Fc@1mB?_ft;JPjCA6l=wr>dS zB-oWhILI93G-;HTjs$6-)2>=}%Qf>=KMgb&sW>)fTy1uikZY{ET*L1rEM1{7!muze zbIxIukOXFq0>J)+~!-V%lH1?(E(=I6fiaFG7K^q=EC%> z9Z+i5|0XiI8;v!{;d$Jd6WsNe&+ztUGDhJlD`LyLc2t!VDH3+@A8#5=h%8BI_91U* zc@RhJzH4jG)UK6S4UZD(_{!z?Z_pgP@BWOoa!55K+0|^QokA;`g)~Rp`wL``z!62OJq#WV z|7FIY_Y6CmbsuOd+Xy$86tYpyRT29Y#LR>i@!G%2T#5+KYYpqau#7~8RirxV)yOuD z+H!a>jK}~AncQ5t28yMtwW7>T4OUMvi&QzAs?#PG_(EOg4eMe=mzPZZko#C=m1mYS z5^!JEm0;4Rw-lE~TTgcNYQEUi2Z$i}xayH5;|(+TP}!X0uvl3!;j-APyr)mri-h*%7lxCg>*rm>l1e6Ns1$499F<|Q@ zA^yRVE7I*2n0>T5AZ5eI+r+gfEG?6BMxx@J{YjmFy|H^D)`-CpEHU6t zzcnR|Z^(28hb3}6wwTyayX3ms{3oHrE6h<)`UgR=ZJVf3CAdda2h&kG@?bXn37DcT zvhr^^FOgw*C`2%~mLf$n=owoIj&wocwt+hyxiR<{0S5H}`cHpbu#W)?`n=A6yP2^Hwq6)2(%~ecE(Qr*q|Z5whuL6d z<7KH*BAe+I+f8UCJK_T)5;UV;=#xQaoc_W5s@%DrUb;BM1f#qra$2J^4|Ee~y*~sj zUhwg(HXvt3mIE2z2W6v;atcH`g>=V3pj2w3YM4GeCPM1+UW8A%WbNh*`41%3$zC~m z-V(90A{RP~pt`cMAErS}bC$PWXnv=Vm;Z*RA_dvtRkrUj1+X4}-2`ctSG*&~M}ANO zuBKxZp;gLN@4Su^`&%&{gB3Vv3IZ|!Yp_feNO#BkWvUND$tmnsQRn9vh-a8aj3155 z7Gsn8@41EUW)@DfN?76?U7h=rZ16%n37T!x?QNvffK`QFtcUD?iNK}LhoROZx7lR_ad|LgRLZ=_B@{aYxhpK&6Sy0u%`bL@b zPU)8;!Z&O0?dq9;bQy!xTUC=AoZ5I6O@COyW^DqG{A|GJxsl3LHR4GjqhC8l#PKK*A?U~5ex?ABh+phoZLj+x3jICHUKhzf) zyG4osH(52JC%i}&Pa2}c**ZHMq2vaUN9R=&yaXCRF=t=2dM1)pKSYck9)2ll+hbMh z^&)!h9acZK6c7&P$M3#TRZ7TY7a4n=m&r~CUJ9l&f*68cP8~9jK~&@NU?(C;Dq0?W zk@{zA_kQe6_Q5bvbINn-xWwl*d;9#j?tA$#;@<3qNqY$`lRw=@aofml3F7mDNU(Jm z5Le$N^zzD%c`j@gJ)LNszn(APhfQZmq9LOv-1!TM*HI_OFo)oPj!7>25V{B8Cv20m z;7F})D~8TZBZD~T1x+-}%(TjI)#WaQbBw2bx^=-f!^0wJVxzgXXK3*Qk)+thsS{Bj zk4rcXK`p{U#%WN$;T(V_3)(L+6YNHF`cmZS5-W1IQ}1bf?%*O7e!3kgSz{xBVw45x zjPk1=jLNZ=Sa2Ja{4ghg*ATSf3%3SBJ1Yw#%j0YAm+Uf}e7JrTm!l`_N!KLzhBR`s zg$fO7tvg&Ob)G1D!B2=m;C4W#Xt;ad+-7M{`lq6I=2>A_GPy!yZOy;w#@8SGl$^2~ zPn*Pm!6Y4;K%NR<0V2VSS$s-|jAHg>F=YJ8G4=Qg5@R5ZHj#W+&5%DE0$L0dMdpxL z1tmoO;z*BD7|fQVw|d<)$ppZ}%erfyEr>ACr&uR2$Swz9s60uPUM31>m=OS`y64IK z2fRbu4o77y@wcoO)}2bCE~VQS{=4zq{l$Z;UQn2<2E^kgSY_GC!8-RvKopCLFdvrF zXa2>0bU<`X;oXxpHz^$Zif9s{Ym}3rm7C$O-;v& z*r=nFLp7(iQwXGBcobp8_a|gYcDrs1kQyx-4$!n)#H}4tG$h%i?jkK`_Vh142UB(b zK4fAhV_0O9c6R8Sg*hEbhgUXBOqG=y2WoSwvozwl<6_HZ_NsHatBAA&wB$gh`S|0g zTz)UzNc_rxQ0uTX&eWLYqmG68lj}0CKz?;xcNHiHPc&OW<%z zx0)DSy*vyTSc03$>Jmm=g*yTH@g*I`Uo#iqorP1^)}njJ9@22XJI&ImhMk+ga5fYpRb@p0g4|!jTBc9vlgNE7M;K{i+}r* z$+*<0N6k*9y`%{XqHfdSf(R~NzN1}TYAV)d+mJu>La5@-oK*}xG}mJI7#mI*4SzJ>XY3Z!=Ct^V1%~s9H|v; z(v>%6U!+*efrx2kFRzfWSbGU0Ah~H){J`KluTiCyUmGZF6=9Dw-$(D0?G!o+ot<$+ zkwMkf<%Qi==YZAnRj26o>bGXx>Qgo-1VU*9cCRdDwgbeJ7q|RDlmklIcFVK?ix40L#%S+_d5oDOP` zw;i8srfCq5^RQs2+jJJ~V9BEQ#QrllvUYL93=L>=^D8=q`V%|tDR?~l(e6|^Wf6fQ z*29{z`A115OpoqENknqiB@61g^b816Ra~X6{aIYGdadYPWy#!#Ss^f1Asm2=;J3q) zc?3pr%A&Zc?hu(>_$lqrY$ec;e0Ph<0xNrumpi(S=xc7aDuz5q27<8%QE3s<5*8YB zStMJ?q~E7n_E#n32#7yBi=>&$q!n#sdsSD;kuGrxB?%bxUu58=IA`es4N5rE9BzG3`I{sc-OQ1kdz7?e)*$kT5S``!H8Vp z^iJ{DYRl|Jt5dsMEL(|!!2|LG_XJ&G!TsPX9=GA8y_ft2p?}N^OU$euo~yAVy(`q` zKt-!~oXbmRYTEFt4De zdiGH8`ShXOZm#yawbq^8M9-UQJO&?4FF85c_vG*&-vTDh&4wtIfl7 zI&>Q<*WTEgLF+1K10<(Myea?R=Sr zAPW78jIy^NcFqp+b$KMmC`Bp^g3y7tpHAwRn*blzGJ8gNCwmk+u*4>p0`WiDRWzE6 zB&th;Qcrxwm7HE;Wuwz$rpFHDM2FzM$2((2!W{8$hTU3pe-`8u{i-I_xkQ$g2P9gs{+PUM^b8kJUn@?UkcLSq7ko zgK_|BQ-&Eq(53o!18u*kS96dk>tuH`J=7+96Pqn(`Tw&-S7z!$M-^&7hU3kuxDjvs zr-s&HDJr-uwO0_=3K$k3exRiMjI}XdmbBdi1NSvzeW_~CQS2#iP zvx!78LkJZ`OC&C%j@QxPQU8s!Iq1qf&sJu!+5b*2LH}M8xQ+w?bK*fq9|6gSbuOm- zD_BAZHh0ml6}r2;g(_(d*ui<|Cs9jZrNJGy>+sF8`cmBL{W6zFjs@Re2=CkCWqGlH^=5F1@(lqA=A zIvwgB!n~+PRrs<-N;m(9|rA4N)PdEB$V*edTMWUru+pB-$Y$Edy%v%51kPg~M>k7wCJg<`hWNCf%< zje({X6?ZWtze%0ICrgL0*4$LFI!pI|SeY%<8u#zNr^?BvracFJLPBku`x|^?9w3e4 z9MV^Q$=9oW!||T?gh!HQvQi-RUSn9Lb!r7H|6Q1SP(M$jN=`h%^OH~4*(3t$ zI^Eh}02G=7wKO&tLp2LJ`!VMc1u?MCpcrHbPsa_IvPZ)t00)o+1Mz<+G(}GLF5Tr# zXSP(IF0ndZ(6a?8%J9Q?!?RONlnuvcrnS0>ABy(g1rp~#%_9owd?qD%Ob+ayt@Ni1 z=jqU5rW?GxB{T^XVlQO)DL;e{s|wVY*^O^sqQ_Wpy1UsOlsK4o+15l@nqmRl5MmFg z$Ofp=b9`*i$&LeiOz%j)Um~s)TDfK$=9?uDiy3ypsdFs_r!2LdGp$>gJ|dvU;q#9d zAHB9F{f)h~xR-;c|J1SRXMm!QA(_JNP?|}u?LNhCz6`*@0x101L*6<)w*u8CPN&1~R~qbTI+3d`3nS1|w@eC#?f1XC(hS=Y0qBe#YEr@1wGO zd*RA*{pDW&_mzbCQ!tcVS*p`JR14W7%|bkeGJ=`kSaQtJSmA@=2wc(vWfzjW)V`aF z+t|pcP3glm2w97K#hVW6{35_W}Lxqe7rBoXM(oH+?1d)n)nH6^K4QsvM@WB<&f90PW^B-bN ztr1i92wwbIo)lF2Q*TQvs5c=om3jy$LR0gq0G#@BS0gOu1MmFgwO`geO%hr7PI|_T z*Q0rCYTODX=P(mei8DU4xb*~9Sr=Lg0(iobMd!r-^5GjT@rt}jPV)UU%f!V%?FR%0 z!}m|hAZFD?iNsVehEmMT4TMz1h}UAr-<>k=Rd<6!_bheR-})H2su(cpJ)rU(KSIox zQ&LJq+rmYQS~|B}Y?j!E7hv}YJFZj z6XD)r<5)FTNPmIvaWoaS#|=-@(tsbOqOkM-7*UCF@fe2^TKf9taEF8Aa!B}mNt<-Z@Q!L>fZR?4wA}&KXjR0-WUJi; zMGnvAU!!tprj+cmQU%P^G@AT;27o3PixEx<_SZ>axLcdMtcWhPC}G$`W1sh{rAFw;P+i0Kt{qaBs*ji{wgXT_K{i^SUgQ zyM&3r?99bIV4zmls3}|FN z@24lq4Cu(+$;7Pb4gNmLK;G!;GYzKUBxT1B)6JiT?xm1SPYnv4A0KT+; zeSgl#p(Vv7WP3EVjfWGLtx`^qZBk;slA>g8Ea9&E^|=>&^J92Z7YNYghli z1u0j{)Kkb~QbSXn^2P)k0FsGD;LF#1``WTeIH5O&_iBoL;TNk=C2;b5FejY-73|;- zaU9Mgdi1Sta<1cvZ8qA*SjNo@TiaWnGTU38%URS4wP!3(tcV9@>a7AdrXSvl<$_?1 zw}(i1aY2tpbRIlVnvS<_vO)c4>F@r=6n-F!D(`z85255rS?-j0joVK6WVov+3z0=uS6=h7{+=d(?TvQZ;Qczho^@C zRlA_>vNK+7Pv_*+TzqP2+@C;Q+NTl^`Md$+T~S^1!5@AoK6G&dq-S{fc#HLQOy1Cw z6_!-vldl})h!QjY1zasVumryZ;vP>H=+h|;4A8|A(MjU60IkUiFVZ5Zfff-sZP=Qd((nir#1>WL$jF=Eav4E$GO<%*oC51@kt<7 z8oiQy4J^ZMyQ1VdOH0VBoEx(y5%&&U{zEW3XP3%%MhNR(j)_6pJ4G}j7lleAWnpEU z&y7Vgg3M@F?grdLLwTD;E{h`vp!e574!^XtA__9GeCEE|l#;RN6)CZbyp=G_Q$r~& z*P`*x4Z&l7hl>q|#gjx<3`N(0fn(7itf+wtQP?D(2Qrdny>@yA$Wgbfo}Y23^$kPS zJj+JY8I$uhLh}d0Y;e%`jh-S%=^;FRCtc+@QfQ4_F?tm5w@RmMprKRzNI%<7q$`mI zT;o1K)1|1+B!45D!cz+zK4{lm%~m*0^LsN`rj=CU*`N%GBCerh1QVVcl$jAY3zpUB zuc+Se`fEqKXni6R5u{0k>$Zy8x=TEww)XpJXi_a|oil)MpyR4NJF166yU0qnaf8Gy zOGQaO_S$Lk!jv+mqeeUb0mlkSyTgk`DID=&&xXi4M|=!iuy@?)NzoRpm%WZjdO*B) zoPHEMiS<$^ktn&iG!A?|8&=6&q`to#%S>B1Rs?qdue2sxH1aIEA^cyrp zwxMLUiv$qU>C2W8xpd%Ww@}jOja@c4=MeOJ53Y|&eI3hEO|3PmC!PKVqjM8R$eQd= zx%Agp$gRq2@A#)lMCCCL&{+^=*k(*NI~|})BxoZc&h2cK0dN2}jc1Peptp3D*xd~q{zlMoUrepN6+XMHqmW|nt zbdRUoSM5hr9`_~M|Nq0X);BjYS=g0YX=rkX@a9DNau3r1F1Iajj60X(a`1mIQK`BZ zfwuZClRbj_TZVwehEUZySsYL9>R%w+J)ZJ|NoJoamw+BM@_4fo0{{N39Ok(rxi$ZV z`T5cA(@`%MW~mc|FGy!0*A0 z+ZrF?$-A63yB?at;vpGHpU7yt*j8V1^ctR$uk?eh4qj=20|ji_sa>LN-FA24j^^Zv zzPK#>`8R!b8!>Lp?!hc(Wo(8tGJ451&ZoZB3DXm6^`X=*Md5PUbioFAGnhVw{^msX zjHiqkCo!mN^c>&hr7#I{$`_vQ$XO7LKQnT8+JIHo^gZIR<0LcSniMsO5!vntQy}hc zT58E)ulrwD%9VTiubamWBmsVh*{f0m-O-mI<(qWZB{a(ni7pk{4F<|OYmp?oYB){h z)9R{r=jn%3Kvnu__V$;qh)qP|VoPE|s=KN4_l`xLZMPpeiS>x0(7~BakxCVE3cO8c1clLE3cF+Fby1rZY zR#n&Oy4BtHe&yTO`j0qdFeGsNl7Td3_0z2JZs;V2vGR%f6e_7v(f4pWYyIsjbNF4` zR$BaY+{FZq483l>MTI*zH6-I&l*FXji{Xl;51MUUUgF}G{dRiiwW@HvXQS5pHg(17 z8ri2qm402yP9Vsqe+JozT`8;dw-FzrgELq8Xd1FP(S+qNWwl#Zl+Ca@zh?{R&-7+H!p)qZWUP&`dJKaP= zF*rWxQ}wE)^kR;fme7Zau=Qp4}_?tlPsuh7bU5k4w?XZ&LR1Gmue zYJOpp?$vQ$tZzgxcehK29n%o6=fsBlZ*ZDKLO+*Lm$8erS)xW1$%`VJ(3c_wBvf%6 zE6}Y%tQ+J0mEFU1R1~ORR!721XC0JhMG*+`=1ju}B(KH8N2qZ>{tA5`8O~s?k*Ux` zeD%yDFHiQNip)0i^4YEi^B2`;4L-|L4aHgI5gloWAQTnV zFGmVF51-TPCnpD%-KX6(_T+5V*e=CJEs`U+00KZmQMujL+0>V<6BgAJ*MZuDIiq3A z>rUW3;ei(_=DR?BGR7mIFLT-oV8?!NKdp$TWb+|}guwmZ9m;5W1}psmb^eckx6kxu z>{7~0Xh=A0`)N@(k^#0IB!Gek-G`vj+`>b{%2{8!VmM7M`Lxfo$_lO4taAgSh+int z=WYT7&or1_K~X)GSKwaL(}_2C?rq!X{!th8QEXzP%IAI@J+B}-`ro7Jc&N10oa+Kv z_gRbHI6U#a#n*U?y<7_+Wf51d?NOpodd^i)|G%?85@B6HUvf7s&F|H%>o(lix!UcV_rH##stK;0iYWV`yl` zu##Zm!7m?!ALflg`!`usQwSANDXq@F&uf~IV&n0<%@5-H5OJZvF=B4f#3eP%{DN+& zg%c#@+Vsi0DU3Z2S^~W9F*Hhrf!1jdTI>f8p~2psnkfd+iK?4|1UX@}yYf;#_9~uW z-Xjln+Dsm_L%w(D$U~D@3I<>Kf4)|jrZ*AumM_r2Y%@M=OYuawip`BF=!uviQJ8FK zRyu@Hea|=4V>d>~TW&$(_a7nAK6(8Vj0Nh0TDLrKK4x1Bsl04f zJ~JJo{rM1n99kR1OaYRAv9e1$TN9Si%52#ooD3*m^bUI3N1vhk!KzyypOQtaxYct< ze0Ah^l~V9NBGF_ELDs7y<`6MtiEiIB3)e<%5Yky4X0c5}CG5~q|IV3ezZZXIHFsv8 zK_d7ROkf}G=_Pz}#v%V6GrW7jC}YUH-rmLB#Wn;pa&43?&^-h*HtO59B6bnx5{3({ z7p5$x?~lEK1rxS0^Xg?fnO}`LpUvHXe?lFeFvMk$<3Lh`aK@BkVySS+!HdO8Syt|( z3^Ls~UrwyhH_Xdi;p#(7J#zF@(3DBW4|{g%j2QKP8+^-PR!*{<_)G6s-i-`bbCXiA z*cunigB1&LlD`U{ggDJSW!-1geDQWUc>#O%8w31Yl>f(=O#U}13d~sjGwitE=MQ!u zjuujv0OYXF17_f2)^M@5;Fq``^gqYW@WBge~SwZRNbW`;&y+82d)9Ar=Ax_+yi0{CGN+FXh2+-ks16^P!Y?Dp>clYnM z@V1aDww?FtXz-dYJ}#Hc$8x7L5frz=bM=_mGuS|v9;Y1ZZ!=1ANV)iF>eJQc#YBV& zRiMkm$wguBeXJ;#|NTW(?>tU=1aGe{HWi5~&OOdLe`A0kH#rsTaWeZeCypqed#@ft zw9@|&MNKyoYT%=iz_gbk7uJItq-$z(!t7NsR6G*ON(dvr0TDd7{q0KHoCthX3=w~B zXYGv;_j_Sgy10(*O+^d3d(hQOcX>rWu3hfxX+60D)o@tA2&L|!acw*td%jlCT2#;Zz|?*a^lsuir?NeWNzP7qrHmFj4aN&jRcAPSgFT<#8u<5Lzb@Y3 z?g#BSJA(O}ACWN2XmGk1!Tnc2H z<>k?HFESWOdJ?#3P6&UZwy3^V>0Rj1`Bcd0G1=oaq8au4qQ+EZ=>Pe7_fVl(4rw9(xoq+^%_8yvRNzLRhr zak`-<$43iHU{HmaB!R~6ys+`@P}%Kw6X0gDG!0r%>Yv{7`JOJvhsg0-tZrZ3I`qKG zh#_q*FA1QRVN%(nU&@mB3`WRuF@_&6~t$EDcg z$H1l`O2=FMCGl>*IIQE%fzFhmvHH%`>dlG*qvk!YM133yBfdYySbSk*D?43Z0qvP3 zFLxEbSkLwdr|c)JCm%rhHR(?;urCVhdg@o8qtx#eGeIGHHhmzERi>oSd4F0U#WVeE z>z4~^Y#Fb!W|xyBFz3m^3O9?Qh`_nOi=r_g?7F3Onj}zu6H{uv@!px%BZlIn}&xh-#d}I8P)~r37p7*B{ngsMMT{ zel3e_mx=?jeuyiIYvvWxpzj8nMh9PyJ4g0J3JOmPk9+Tq6oW1sRA(xajYDgN(uF5e zqfxnSAakfyoH4guGvhb0W}iQJ2R1q`)YAU!#%M5T`9V0nAZVnml%>VurlVbRNSlaf zO&iSBYeT=p%*NinHJV@t<98R5O6D>d9uq3=3ocyj zGnDlXi&d_eHz%nM7P5gg4WF-zA%N?sIB76IeFaKjqgqUuLUQ zNz<+#s@>yTyfCom#nO4a8;+hg{`%?w)p`_A;H^Q^;E7v5QyYHhefAB~2v!0;y(kb? zw@*DMlNp-fKPL-AT>*3XA2$Tbw1TF>^6MR56fihe81I=J*dk|$#eU@9wPJCA;TChhFNthbRvlc{!pF|BinWqo;J9#Bu8nKgerkTFqS6Vz=iL&=G zZ8a=IS9^uzaz6q2UY?U@*LD6}`XR8JwGbmoYVSy=CN%HQ>RRZSdl_xTBX-Z=yRg;j zr{P`d{WhoX4~I~7-Rw(kWpfLG4c^=H;RbQ*bSb`PNh=tM>UfEIhJ_|P6n2HYuH5ej zp%v?wz!jqxeHXr3_sI6rvR6@1CRg^~PW=hV4f;TT7Pvc)e-9~VZ)Qhz#uzbd8VWd) z-vV%=U!2D&;d?>K0b$YDw>iz6F8fMSTL$o3@@4FV02Y*W-g@cT7_g%GeItbGb@ny0 zdHWF}P;N2nL5O7%Q>CPzt`!D)&8AvX=QaxESBm<4LF9j}9!r2oB_I2W^zprLfe_;w zJL-kGe}Bhf2!+QNB;z@)58zji58yU#M%cfY*QOeD%J*HOLWf^Cky3=6{(=`7!SkY?GW9u}n;`-YX=^$CT~&0{q_AW#C(I{yiBreE9VJ!5 z&;5@OKVFS`jHrPH{0C_+{5IVIh#%}~E#qN~MCnDHX9J0 z^rf2v%v8Riz)Wb6m}|W^^bFh2W!UiKv6j2o=a?(wL@uMtSk?1IqeNgIBV;VLF{PB@ z+Z8EKqwXO_C^11K>px_Y)(8p&*s!ST-LC{^i2BROo^>879!@VSw#4HFdJTJMA_X?W zN6fdH%FE4rlusI(tt6@#VIV;bG)zNs@$?_BW-G^2ep{gKzt#B0(aC4&eF07bed2)# zEOOrrGye3vdgxGuz0P3-;Z=C&GEowQQ4OeI0nC#^kU|?DrBi{Z>F>|VT$NHGFOMuF zF5MAKRG^+S-0j!$qfVA4JacUXwl;~^B*EZQCCF<)D#sbD;#zwvfXlkkp#3`oL-~5y z)lsMHLtVLwC8Dz(oe;z{E|=)+Jxuu_z9$I($zr}gFp|oVVvuZbx3^qx6fi)P)H_nn zScG+IqRSpgC(Ghu)$mZ&1vN{37zkQ>?}yZ>kAblesZDwKf~BSU-klJwA0A#|Yr0F9 zeJzguM`5$Eo{>jSk6O%gU>BJtl0!vAi`tT)R&m8vGj(@m|9mh|+gy*_-;*4r5nIe7 z@xtH|OFmmYjP2|d(_No7kay;c(*iR3bPrxa#_n75Jecw-FNNbaiW>`WJoW3)Z{Hs~ z%9u2;IF4@Kzb|)ixv!eNNF<>tB7-6!5K&d*p8FtleZ}yAdEQzVfcqoP=tCRpkPjFm zDoZu+@r%g#JQd=!ELyz^n!HeTQ8jy9HgUqQdz7TKE8~w(pMj>X@N|oYbZexcd0imn zujucUD>rF*4u7RUI)i?!UE|90Lgw1#8=lK2M2Awv;#DYw~S18Fv2E z$W_V8TDU(?2!iWV+{$}J=7YKEj@Gf~NJIr-HEdD57BmnU&)~Jq>a*!6oc!FRu4)9k zG2}dQKxMFWV%1MNO+ZM|;Y=p**o!eBgzJ#==ulLO6g}YZC2Pn;_dQ-J2mz@oJL1EiiG;Cg#ee--L-N;`QFXA zCt*`(qEmGPxx2Re{f4bvzaxmaZ}KNQI!)D+xT^D2KaWkXv(|ImMyo~P%RF5%Z4VvNbSt0>Ah8}6B^67;2f$tta_m54%-@XgC2Zue!A1NP%hhK-_EO2~7Y=5t0uj%>PAH}))LcPp_QId=u_GD6n zZ(NPvb-UMhi-)P?pb5WkVE(%}5a))v5r=pr#Mwsq^Ut56sL~g4ESClv0m=`Y9yLYL ze*CjBUR@+Hq+yC-{l0pP;Euny1Fu4CEL{l4W=x-^P{{kG^JZLe{prCr;<3A}klaHS z>K_%ym$S%Hyz<$8>pv`Tx|?HPlkG0C;hss$KvtoihpBREYgY|iw=~2G$~oUl2#$Zz zu9vMZnM%gF=0n|(v8Ty#mh zB8M*g@7)Yyk)LL6MA7lphZSCbFdQ2G`pQJ~k2fk4d(i09LfT80F)TC)I#9Z9C9<=p zPcn)EU?YFeJm!i9&1Jd#DNxYBO-%(IZ8}=P10b@@|M1v17u>9utsHQ!l1c0Cs8z7W zp^?f7zX*TfHtY;sG@a%|9qdgc-2I-qQ5Y|vfgEn((k9FD#ddZU&Xqr&8w@2fAr z0=2KS7EAvj&P3n&bn{p`o-g|PqNm9b<9dyyhF%Og z1|iND*GzrztZe-hQIa{aCjr@UVilLUj%f%RB-ER9PL9iQ5u+zs610%&!zu)dx;MpUjDIB7! znj7>&;J$V)=EHKQPm}z^N!kB#)NOwp`cSq&Ucb3QNSwCy%316GM&1N8SmbyuBV`u8a@FB_M;Pyo^dE=?mrW~o*BZBwyP`}_+ z6veL_38pRd8$fJ`o-Q%5lOS<6jfEFXd;kK-^M4vW21iV3(N@EwZT!g-CB&Z_xYjg> z=Dck(2v~dm=8Je~8$!<>-5R(r`)$BH67p(!9ccQ#WXI5zOs0u`ed2-%^C8CutuSP5 zo2@M>vHO+F<_9Yh6zEo2siCl_7c%1Y@FT$F10OC}6~kDu-!_vT~N`cy>Zn>%2n6HM?~o%-D3BFp76a;!Yk zKzT9h#|MD+*0~j4m)~Ca^(%bzkJ*o@d)I>20Ex02P*`c$emDi#9dm$PAln1-v<|`x zx_9TlR2$zQ5b{7Ue7UfFuoR2oHT(P_=;N2$^KuAhuFu|RQvq+F^O+Vg=2%edECy3e z&+K|kOjC~d{f>q9))<%1@y}E=;TP>< zDb>m+kjv#9Ro*Ku^^pYW92^VCWs}V8QgDVnlCN&FcwPaN<#O3rCq7y6HfJxe4?N%Z zRmB7PD_uXxfrz?pMp!PZr`2+!j8H6|$+T)@n8&S#m>lJUce1i^x7U?cyxSGqaWm&L z@86V+%ZP%_O&`J`UTV1_uTavWem#uZ@B4W%R*4_-s#T?`Sau(Uz^!5&fw#&hf|!@n zI8s5)?OQVR-RJpFPLu0uc|k{pl{6wq8uz}E2W@8y;!vB{fhY?Mu1dGTNiyB}cd0qc zdUR`A+TlI!aW&rX@~y2cWYL0dXEW=NJyV^JlErIfL{0%LItIx7?p9b@g7J|1k8BBi zyO?LKW@7yl0cb@BG9))zTgVyl7uLwWn=@|U*6oX>fZ45dClGGeO(<2a?^(zFcc#zU zYLLF{Ad-sJPY=L(qQQgDFUJvfknH?t*#_@ZVW|A+<|9~=)E9`fN!dA7i_!*j!O?X91nV` zS#PH58JU0qp%aVe=dlQI@LqymM-g_0T=BYledSDc!N9GPC0hS4BvmTt0|lFlAHbf3 z>Q9eM6S$%_({&?5@-B%Ba$2Ca0`kP z^g#D*yKMh$LDFn^H7U^e`;TQ8|FWwxJX4Pe)*U~Q>w9&mX8~;U9Fszd`t;@53D{?F zc{&n&jLUz0!xk$v`d2t7xR2%X?wT_vs6z)zTNz7AUA-xM$@+@=qX?BwLik~3`V~F{ zGGCGKna+701;iPA(--k7;}xyto2l@Ee@DtPR%M}GRH z2n{VAYV|rg)QkBDmLU0qL`XlvR*g#i2gHzbn9AwY$6N4rTtMgGa{ee+@V`N)V zAGw~mor!t#IVfuIWdVmX`R@`Z)S|Y*a~?5B)I&C=X~|1Dr%2q5Ar<|++-B{q8Ti}1 zw_8|(+JRT}7c?2Ka>ssQCZV(VUlrmsx+p#>VfPxsG|&@*mA| zA0p)uZE)Z~{&z#dIp2GDXRp7yKo74Yf5qd~*x3^kvF;WMflE3h5x;Do;B36q>CRe8 zlf-H6i!FaoHYHm$qC1fEMAgN3t)oSvlIE|3`!-E@C0_n#&>dVfL9O5@JL989- z`yr4cc7LKNk0TBzGvM{M8NfSKc(9wzpLN3fxmQvOIcfqx+9-Q`UXt}1oZ;tUSgO9X zpUPpqJ+Ms2ec7tP)H)_vDv+QiA`bH&9Kd(7t$asmk@-D8MjC75I3VONx4D&3$jB ztBkm7Cz{#Dn#|1Z9cxDmF42|Zwzyqo%J+UUeJ@IJWF@UzoK2q%6B8)lL97~KLhMn! zHx~=zqQyX|`g)XB3d{fl`Sq4^7jw$$JVi{XVF-y2`u;)u9^%*yLry ziMgRG?6vTM1fosCm^c2WgD0Ii-T7Wtlo|^%(Zeoc zjd3(hrp z5x1y6fjh8?x*v+URQFYLpMiRWd7WqsLvPRR?Ma!NRORFEAZLFu0g>cCkG~Y!Uv8>Y zxExky@gPhzI$_+MR15o>rGXE!_y>kyPPM))9V&!09c@k4z+F(qW}GG{U-)@McBaU| zR@gYl6>{`4wxw(GU0da#vMR+a13w`DG48ma*TyBfeT6>o*PuBu^e-UyI7}KVhC6`9N>sQHPyoD2TmYZ@WTE z)IQF255nt)Eo$;2eZKpK&!#f~@%F3pMz}$~ftd^(5q}Xh=;%P~c*p0xG4s_lWXy`> zHv9XAA+NT;^4K`W)4X}^`1|Ts)iOt}io!Gqu zAYnZ@2=J)RRbB6Jahv~`}-j$hfA_F3DwL~5A{euo|-nLtLmIhd-Fg9T|h zT=A;!(__$TtY#5rvB@0m+Re>j=^Db(J8*F^d~^9HwrO$XJS@Pk=is98Pxn%lov0%_ zkyTp`rQqX63;=LoliU-0;2E-_=pE1x6mRSJ=I0w{ zD60JG&PHzQu!5E^+$|by>O>4Qk%csW;GnoIW08X zUZ+(pH=e)ug7G|#yfR4G1Iu0bNQ3J7UY?)Zw!QZFlkhE1ydTZ$Gvp~62DdN+wTD&E`rq7p(O zh7S>Jwz>>R=a1o!qjI94qlog%3x8gpPE|qV^{`~0?pXyh@8ssmor;33o};4)(96>T zp9NsRZh!-@%J)>4s4JI8>SMnHOFX5kqNBp@;)M=hX!gcquM7CwKTOu}tq8Sy3W`DN z3Unj$NPX=OSU)pGT1QrsXOU<`KnlF3L8bX>;(0LDr-&Q>r>vXAncLEtCz z6<`jRbsTFf+Gn0d0LWtoFyZA7+7^9Lc|m${c@d$&Ir8#YS#hb{p89D-aetr@xsZ79 zQ1eoH3aVeP6kjv`M`fQs!#R8IC~yO+&$m}Qs<>?yTZU!cA)Ja&*Wc+jFx5xa-`)Au zH=NF%HrbswcFf)RRQ)ol5?%L{ud7#esM4ueo%gg^_PNErJ!))cX|T0x(+72eemF!m zHl?H!7M!+UP4BP{owKMhbmg_P`vlY`v@`nTyCRPNT28*)p%`uxF(-Z z+rGBOtvam%1GggRBJ~vMibE7_MG$O-C60jRm;F%jRapN<#d7{4Y~hX=MGZ%+<^{M+ zMcHoaXZ1XfQreV`3|i@QdfwTv3f&GJ{slH7CunNI?#%P0NnL`JcDq}Ju z35yN$4O$Cl2Z z&etT^b0g;c{e9QE*M3@dWBa0){YXqe=`)W3<>9mS`e{I2K>lHy`AD!>g_F6GO@5QzT)@WtMOj_yf|A< zpp?JVugS@AHRs^Pd3rfJY%Tw;z5Z}EuP(c@?hu$gez($oc$SBs?RY4($lG?V?y*fH!i?{Kh*YPL0*0a;rj^l`G&PvY8F6$SDW4ZYWF9)L_72WG6 zlS)6ERo3Hi&~$5OTXo0rnny*OqshJ5zM4q>x&QC$2#@RIY9p^9VXM{8xhCiBW!HN9 z?)!wf%kvG@&(01``y#-gdd~nGhkZ>T)mOiXB`*<2qvywlQ()G7dz=3MDPrWoua||4 zvss6W9?3nGByh1$3qf{@+u?gfOKk70>VpX4YeWEB4r8IuqHuSd{D}?6qyxa@^iFZ! zVK(WMvdbMZ7;PPoE(tMTtrbB!(_atnI(h*%LONP($Q8dy#6Imy9IZAiT9{K?Rh$nMSq?Zv={om#I?4mvwvR`X#5x+Bl?*rn8O=u0&vt)8wC*E?&V z)~7lf%`a_M4!oBQ3%CLDg;ZxnyJl2??HXMJiplhh!h>YbH>wGhLa9cm(PO|vA8$!ldk>~NS1;+lsIi!JuxD)%|K{Yq?yBj5VE=|o0)M+Wt zY5I?m#`rw|CJH_?zZRpbY(MM)vf}tFNdA(CrjJ}W<&=S~ai>zBg6cYegs zFVk^3|0bAZ)ZGv(Z8rZrk0*=KpC?#9TM^!n#$Sw8p#%&?45e5)>9D08LR~oGZZzSW z{u>SKZYZhUF3PNpUX=P|H>-TpF5!huJ}$Y`6uBUPX;5ZiV4D$LdA`F%=I=>tfbqFE zUKvSEKFcx|4K98vR%b~#t?rw!*+=k({(#=~+u&l^jo%J1;oVV@l0eB~G@@@dRK4uU z2g)~v;cmnSoXl$hP(-{9_%4daKK+a(!87{ctwx5o;7Z^}7v!LQIWL?3_b&Z+&tV_x zeOS4(+F-ePrh+GdrMC4Wl_67$(_qneMQ8Rz7pJ$63&vUUCAdFj+DuoTe^?x&a@o8( zC@fE;awvoW9zL>|AsBqqrK{Qs`Tm*V*{P2i*Oy)0q1SGs+M^}CEiFz(nd(m(jX~{@ zn3?UKS&2@Kh8eqD$aB){fS41Du*h1u+1ak0IM3E_7ndGfF*5aJ|T?|hpr%Rj=!F$p>l#8 zjp(V>F-X@EE!}8$xjyLRukiVc!>EhSc~?P}9gaard3M}NgL&CXL(y#Ac5%c& z1sQWO)`0>NI;l0ip-gaQaDY`ENzW5jm%E-Cr@5T`Ka`fW@Cl}7piI?XwH4`Nv-odx zSK*wo1=$@jX2YMTa5(DUld0oSM~#R2YLn_>XlC&ImH3N^JU?0*)y7Jmp~)QYS2F;^ zPDOk^R9T>tVp<26S@eg1lq9_55buCV#o3-zk=!a#rF=uw@LtYUnJ-X2+;7sa`i(t& zQeFb?zH`&d}Z7O09XJg zsEp02R8yAF16Ru7NIr~d&}Y6!pjB*FT1~EY9zFH39$n?fz#qz1-KogfwS3d2tBVX5BnrJ!jkP4S zeTXlGq-mo+LuHQ|(<|ajKDz9d6MkiE^FL>w1bfqhUWx3Py}%O({AC!K=d=!)j*+%p z*jnvGmK{~pGIo=)AYBioM*7;z`;n#{u`|CkX|df?dv^>pyyEkZr|ho^$qBS>4s+M@ z=p19+XG@Tzgu=}l_i5ODKK}`SbuKb)HV!hwZ%3pE!`bGh8R2TCSZB4FUX~dJPwEc( z*qcfZ(&<#tL1M%vHL?6&64SiS%SJ}hGtgd^5q zheUb>Bl}Z^6Xz7T1WcD72`FyQz~!9-<2o!=YxvxLu|{IapaPQNz&o^Q#?J@m@hWJ< z9#OrdmW5JBXUA{sR^R?U#XFmPys3gVknLA{04aj1uwD6P2){ObamKUd{N|f6UcS-N zz_rX3HchD>^yD!`Fj4O&dXF_*x!YL~Vmv_pwqy-`e_l7Ft@vdb6I|^1_bGy=pq+b~ zAep(uKQ}q#YqYF32jT+Q!>#o?6ekw?`Jq|#r&VIzB7aF^% zu;WaH<4%~j+wi6ME3~s=kmhiJUSXxpuCz7W{|KDo@i&nnH!I330T zmCn9Km~@S#N6QLQVP@W3m?(|w>m9#{deR0KHB^k40`?~iQn_r}?TA-wLp3eM4Rm{J z*)oDK8C_}3|KKHg`tKMnl*g7vL}?Pq9(*~4+WqW9u$LspNtX)f z+)Z3*maxA%VjOKBiDOzEUR6vlOdju#-FATbTSpeQ}^(Nivo0Iv5I zlih3gk~p+=pC)Ewzhy6Sez%t9ZU**lffxW?bW=_r?xt2Dj8cq7JEIJfEFYTv7J36l zT=`168-8EkGlVZYWH%PL&6l*qbm{1Ql?@IE`A#q)O2_N>aVB9ft0g{2YD#30$$6jJ zO-&*l(|JB?b=;^z_HU_a={M%T-A*5t0g}Pn&zGpq|+spD#$3N=6uNAED?eVOpe!Rrl9G zD|f54YYR}T2|F^|(ki(dr6+Ys;tZp>ATDS6d}Iq^U(``%2hHdwIbAzwvlXLX&_@-E z^PZzE%L`W;9A~cP%ZH3ud4`*!HuQV)RvDU?gFl~;$e$Tktq18?TdQMyu)wmaT$QdU zOipPW{3K^;*ob_d2=3ee(%h{O>mdP)!1MLGAg6oe`(j%VPT$m5-}3UeL=KI0ZJ*Mk z^_)xi3~@DI8&1Y_us%*2xPsO$6Z+VZU32jQ`xK8>iA``9``3)%EJqIcCp<;J*@m)| z!#-E711~E>8eJw&FRPi?=)p_W+ici*bd_3-7!Cx?hq%eG_KQlo#{?t))n{+c@`7w& z+IS@L*y*A)<4M9t>FVGFC=*&|g)AyZF!zV|3Ny<>f0A(boon!*=fV}dwd$SQ?e z=;6jdCjG&a`EN@rIue*=o=Oo|Nx9-C&V9``2G1oRIAy<@A61fW_~X4#By${lEAX1E<}*Pso1-8E*P>2j|z# z+bT*}b>uj{EpfpdsYqzC?+{}2LVqeIbe?=m-)iaRuhdPa6~}BTD6-%%f^*|Q4Y3%G ziQzO#f`k9TfE-@?$A|>xXPeDZRBG7B8dNNt zm)}V4@)GVGV~QzK<}JTD5lz+CF{AN!Zo#C7*1i5`yR$rDP_XdUA%!9wg*t$uQy6{S zxy#h2nCqslKb^8d0WnrPEZ?!1m9{Hl=Ww4N0kujk@IHFz9+YcXpf z4hXQlIXR#!Tqo}cxQY1p_^(11C$2=!Pfj3yF1}OXYHRQ?F;eb~q9tJGDG@q2OBX3S zMb|->Ef`={u zF^jw3P8Jm0sJ~9m74#S2Oo58>6~xm5sb3myunskRc>?HjRlTO=;X%t2%kj%NOJ2(+ zV6~x##=kDxG8EJ%hcGJ=<%^ zYszb#YmB{mpJE|17V9ZPAo|`!?0hV7>_8k!Tw9z$Y;wOxzji;|B&j<95a+v%@Z#{o z@R9&jwFvz+7uXL)Nbs zS8HDEL|+(4C0L9)a&f)nB>%SCXA)K#x2^g@EgrT!b8$SzbUVxwqk=~PI;e0mc#d> z9mcjDTI?_W7q{K}DETDN@r~8cChR~nXWxtA?tsu?Q&;EPjmR=D4S`aYvI%8iUfPsytC)%a)7}R$2te`X^IiRYQDrbM}LnJZAmj@ zXwMCasV;+E<$Ti3)G z!r%q#MkX5~SD#YIV@7lHTeJk0X^vu=v_KX^24nZo&Ys)uidHojzPl9FG}^-c802Q< z*rln4npJ@| z)TXh{PGa>n3dNR?A_Iha`RP-EUERKpi;g)KdrRiOy|jB9s?Yb%Mhs8B0(vor9PA&3 zk5}!0MDG!=xZ>B|g>L_#~RCA`xo;)8Us7iG|PK@vq zhYqJt;7ep1!CKi%r~7IMburl5wqH!@ z_hDFmBNj^Q6D{z(bZR#2IS(3&u|Ji8Fl}~TQp7j}TP-gf8?v%5%U@0$Jh*Yj(}j73 z4dC(|?vH3R%i|EayGFNhV!2Hic;YX+j_`a`SnUxtyI`;RxRXitnpRG>vVdUy1`iNba@178NvrP;bA#fr?fI=93;zHT==y8I88)P{-^E|bp<3-7`{rev!`5Z=WZIjnB{;(m21Yz2R@Mc7Ni3(tYesDYwO{MO>?sTT! zGq~USV!0T68N#M5paVA1_V@|ZFAuA85!VeFmd&aoXo~}uncJpy#`R{Da5TA|CoosT z^vyod50llRaU`V||Ec}&6~?ogyiAxG#ZK1zYLiIDW|?~V;KxW`Z`}$r+$b+{b#9@d zk|hOVg8mu11J-L;xq-fL*q1mD!y)>a4b;~>_!)jkYGwU}+F*&J<4&GwVUv`>E~1}q zZ3}8hn&nq0AKDP^%$uVX!?!?COu>x4@|3sC+P3W3&DwS-ap1q6XW6I2l1jA%;%S=v zt+@}L)XLtCV2lesob9i_zy>7ue{^t2VekED6&}`iJ;F66R!vk=@~33n%yN;()A_vF zaN)H&O1U{XV1P^6z+}RLes-nbOe>+JTqxGO91^rzsG2N&C)4R0_R=}tkI)0en-KJl zDbkem^vGa0s>718-lh8Md_6hy>n9n&V06YEZREndH$$3vzzU;!~>`Fh4O?C{tI$3;@%D{~xmnHf|(0L6aSnAI)X*=>@ssD+8e*J9y4X2&7f z=#{*syz;ZD2q^U9fCjKI!g(uMdG^un{7M>vDO3GfdX zJ`|+J@jb@ixE2I8QYdViYvBa{g5opsnzgH%Jbg`wz?tJmc6)&&Ofeyj3)YH)?C459 z7n09z_`46qN3sI*fkyL3EzeIZuuee4+{CK%oweZGxmM@jYAQ5a0_QL}6hjNllpXnD zALXOSOd^VyJ?V5K0Q-wn+O+&;6I-;+F%$^15=0|I91-PkHKd;~wHf{;La(%n-jfI9 z$k-F$n1)1Hmy^rt5IAZJn3jFR%iv~GqG7clsa+hOmDl0>T;~%@Z!2p5nuI!H9INtb zkmOLOL1O6UO3pS?)HY{PKn&gZrg+#9=2Px=GNLn0sh7*nG$`Tvs)G&i0y&rTVKxATE_E=3g z0ytgbH-9G!Xe|B&d^=6xy0L!ByeT|%DsfWKok~7Lu>H^>x|$KV9@rb$4|_X<_nE)V zTy072WVvd>AbF0V2E)6@f3kg~eQh*5pZlE8kyn+Zp?~5w{kHCj;VEWmd&#R_q#@R` zjKACic-&xj>gmzI^2dpl^Me{Q&Ihj#0?bIe0goz=NRKX$JdfIM;r4SIN};7FRFu%C z4TsQW6adP(!fFyrk)7r_cDvZdQ+Fv!D+(UM`ThE2SEI>6P{ny)=fD&8jTzD7`Q6&G z$lA001fgh~@7h^l%d`7^=xST*`l)Zjaana_+u2cT%dy_P`@EIWVf9(H^n7Tpm&e+Q z(7KPuet~MS8}RLK(S(Tcq2T(Z{6GgqlY6Y^jVHaZ*tejHm%cl><#S+H)yws1=3T=n zRb$8Wo$K;q<9U5R)oDqig@bMrg`MV7l0%h4mwacgKlkJ6j_8m<5>xW@5S;>DrZQua z?kU0lC56c=jJm~#t2}&B(F&HFjMiJno5`N`zgT;#pt!w0!#jNf*zp7Wpcm{pHm;r-sXqp&T zuGwi4;g{4&<_VK*q}{t2`mukerjEgEyWqXl?G2#yQNCZ@eqEPPB2#i5RcI}q9VCEE zbwq)f2^TTSZukd;@5CqcCirw13UJmA%1WNJc4gBkueo=xpKzc_)frvrrXyy?VF&ms zb^U@|-7sy>nM^6p!cemeV?e$VMPVxqW#`t<8BeaO2>vNKszsBFddg(A#iwIG1F+-y zj1!5dk2%{+hdCrSv!&N4WlX87Y%KGVy6V(L#xq0ltAFw|lU=qBug$ljOzZJRN4Tw| z>^MMxII_K!S#+VchP>``K6p#o!`e)DfQCa1j~MlvN810)AzhxS1qkKZ7_7iOBA~6} z@w@)coFo}FiFI0@{h~9(vanl$#-2P1na-3nDw5sN-QoG6$Gj@ukAXA@{X&Iju0qA? z-W+Ujkkw=C`pLAVtfdgUKwJC%Hx&Zcc@9?COq(fvzCs#>-EF{iYK`%&?}c5#lwzVM zP?Q<_Cf%A$;KPy*99c1Zdz)3ybJ(=-`H4eu`r9N+QkvU{N#1Y)%iVo#NBE-;;IENz z(t(rKgpLDj7`+Ca8Di8RoVTD3C@7W3k-K8>l--o0yuTf5{^vx*a%Lq~~J zT+h4A@B4~5V`enCvh0AAN{hU%0)o|qqn#@O8yUpa<0JCRTF2lkpHfMHDTPkP0mf0Y zAD=$s{)kSxFLhc}AWlJ)h9-Nr8KUa4cP$I%Wwj^ixG4n-y#CO7_N^eGB2|8Yk$E)z!2I~xj^v8YkuTGq z>QBOW<$>p#mPEDgcyy}MAI)zI&0oFc!DDL9k+~BUIdRb^n4k@}NR(F~r`UIjAhr8x z1lG-o3V*W120HDVY|e$9W?fSkXZ_J=n=V(%1!Do{A~oNtzE{yv&we z+9yaI?dtGBPQ{NY`uLt<^jR4%F{3=Gtdji8%;ESV0E;7(hT`-Mm_=mr52}NG@b^B3u(o$SQ+ix2IAtg3xME~wrmhNbOpX)X!aBCSR=4^dU4rVpoN`4yMd}L7`HBt?*xnkFAFJ zj!PF*rQEpie=;MXyGQQ68#z!tD**Z+>~3FVin){wNvR9Q2Lpd!;2LmNbJRBoX)rRe zGFhH%R5K8?qoxF7KrOZ_K1Te8si^RhZl$?RxQ$BemhHrvc zhGC0@&_{9>=ZIqUKzeYh(uCO@<_k6~6UfXCHKB8gE0B2D=+4voJ<24~X_lfMt|!iZz;uQ#zh$Q%ZkTaz zgrVw3&gq;4ZSsX=e6@t`m{6i_Z2LZ0+LEuJT&t1Rjz;>Gu1pjLzfBLhqpysmd~^$Es?&rip7OSH*}rG&OJc56Qg!x)PSX< z$iwSPW~h0ovNk><^ZJv(w%-VSrBd!gC09U|zeE7c_oeU5q2*|8oFY(LaT!kpm?0I7;TsD!S%^GCm;ELr>{z}3H`p3_-&0-*8E}hQWBEn#Hzpd#lgsNzfa=JO9 zDU>zuODjxCoi|pNC3R&~v({|@1kFkI#DSqb#QW*R~lzU z3?o66FEmMtqbr*F)NtRXgGQynHr^G?PdDvaL8w19XR2O_k?G}o#W?gWh=$ILdq>`` z=<{@m&U%X22L|a(-?7aM!7RH>&N*YaET-yF$Py5LC)ywPNA)lJhaFpY3=|;s2MBPL zybp8*EwK%S9C$Y!NcM35Q9DgY+S+zm?iok;>$_C2z|dE&sii0*<=3o;evzOg&rsG?9_%d`-bv`h!jO zI9P;vg)SqdtH;J>f89#h@$482q{(Z2W}FX}NPea<1w4n|WTAXWrqv*nrayP|cvPHuXZmQc}!MU}AUhG*GJ(LFOI&ROOhoJ!TV8@pfI~f4CgF z74yBP`Y^e5e5jR$S@NpMf4yny?>Cdn!-P^ev-M|&I)`@1_zg0e_xa)VQ;cj^zMI~O zNMh>^klW&Rf76&vQ88Vr&&|l48~6qwJJ?b!U_|C?MVKRK@TB!U1T62N)W??NY}JnF z6~ovg2d(t9_hU)8B>wuHsa2j9g`S?DzMZw9$BR~}%SL3-gaDiUd$-hL^@o%-246GU z!s@&#?-x1jGL*r{EN+pV=2jh9rQf$Nb{HD&Q(~}0CGhqwi##aLjS1cNAC!mKh>F|f z)v+zu_x}8>3xZiEKxX+ZaHDrxHOi!X6n2`lOw+bKC6{t zFB+9Lj4%&qMz}bfTWVZH)78OZC8vQEzBPS0>0X=TxP;7my8{gGZFs(0NtFNlBO6Qs z6#HAC2&bfee>Lgxm3ZDaJI|Zi@at5t6A=!O%@E*?sFc3fc_d=R_&)v_;o^N1Gv9A# zx{7oFN%i$%GdK~khQj zWT{uM1CDH7o&yhX(TclcFe{}Iu4V=@d;EYNo9FM2Aw!EB+K+&6v@pcb9cDXEuhYJz zy`>_ffu;M}&`)m7SwU5SThy9J`>P+{r*89TTfEl|Wt~Tjpw09!;d4IhNXtrz)RF8< z=fm~(^Ev0X>Im~l{`sf29L;R>(L*~#hT5~l0lj*aRjKTBP^d2P2e zeHO%jod3{O*?+JD!FKJI0#@FOEsHJlj{y$ZSJm(3?@{kH?=EjRZ$Uy;`uVXZ9nY96 zP%A;j=9!Hx_bJZ_&q2?`ci-=*8}syv&Q)IK8!a23+4r&cE3e}mn;j$B{sO9qR2Zqs zD3kPu>_$AEWIfc*lSz}#Q*2XWWblvJEC8}HIz^U8>)ng6cK8CUJc5sk$5Lg>b0%fe zYIeu({_p-$&lhK@&&~IX4D;(3@5NfgSZee=wl`{MB)l&=2Cv%vwf?p3uL|K`fm&O|D|(uKhM)8&Mh>&B|b z=)}l`mD|RON8QbkqSe8Dc7668f2^}*_A0;M_jhRlW-d4I=9TUh$raAkc+3Gm$(cDW z<_@>p+Uiwu%&HsFNsI5`Jn>h)XZ~yc>Xlk>L7UNLOm5@m!bazA_?f=BpZ$yTi^Kp%YMld^vk*L*sY*O(W-k&z`Aj zyw=bc2;NakaWN5f!kj!7tN|ff|A4B;IE@zFnKDqBU8<=F5f*dLIUX~9s5=Y7+!3qo z%CUlkMVVHbHZEwMwf(8-OWwC$7IG8+)lm{-Z=K# zF799Y&#x)0yK|f&jiDA;DZ=BrrD_ISF>wBk|HS&!`vb_S}sieI=&yj5R);y)EOP_Hyh`YCdbPT z?{!%On^biQCQO*3oHHh-Zj>&^M0FQs?2@MOzE0R_)E59m&DJ8U+AkKpns^S90;3o& zsrdpGO|A?@&;sx^MEU(4dF5W`!+uEiGf?nzPZG$EMm@UYnQ!dI+5)jlB61Rs=cP%3#*Ub4JC>#VyR%Q>`DZLb|Df{&cyzG*2dqf!lr# z0-l@@`{_8~@>}uqW1Xx?o0(aqZSyBYPH5lkcD*}3fgOU-$#@SP;sPtC*<$Uj2rb?d zS5?LVLgbIyIEmvTi?9td^hqVAK;F`RjF9Bn3;h)l5TCV~0&GuEt$CfCNC@I|NA@L~9Zx`}{js8F^K#K`^`||h_uk+7owh(V2RRwG9Y0HsdA$8*H-Uj@)viaI8TS$IvzcxJ-4;5=`j_09O zAm1eEW#j~Zg}i+gQKkhE?#si@Pw{nc>yWIm30zSzmoB$Y_4b8I*sYaja zyPB!;iqY$t6@am=isc0M*4%dBMu@vL0jNScDATbt8kJz@Nqj5bw51k7_1(#WH#2ki zA61j2YJCy;Gk_+LV4jRiCyj6AdrLKk=m#1JA2;VO#_|2)7&bKsPWyimilnPXMKA_X z>FKB6R2jH30yFy&TXd#}zcE?o>;9g7gPYw^K|v6GNE-Pv`#^$(i4kq%c$j=9EbRz9%v< z?m}YSP++>pFyW%(R!4&$I;1^han~nW@mCcF(0$8Df9vz7EvYUOm{N+?!1@zOT|@Bg z+<3%Im_3CA2eWULj30iY!XIVPKSVI%DKzp{Q+ zw?m3XekX+ zbHUF;aXJ6zyYw(Vs964BmEndnMUZ-B6N2t0QZqimlT2TYOJq=}+JF3Ik^Ecj7*ACH z+aD>Z^K6mx3XGl{>K_IKmG^m~v|=uhHWAI6lGc%1|4?q8tK2y*wj>>CUe>Iu!`{Ml zh$mH%+6X~zyh^o}AxhPzS(4l@4iJ20j=Dsws2_ns7{d|Yb{-px9haDh(L}C*7%271mGxL*H^MkRv6jIU;uBk^JBTUDIv5Z{8GiBrdHF6 zoNFB)CcbgdjVP+lrMPb0AkaTG_;&$BaYGow-yCs0RFl}R$*UvKDrI-#1}*GX^`sUB z*&&k-*#i!*fE=Hoqu^6kn%x?$Qbet_5*Sb`b)Ruo*Kl|l#TGtNl$&eaUQ#Td{{ieR zy}6;zax>ACfDsX0@5>HRxJCbSa`3~R3uHtG${jRP4n2=*KlLZuG4Y_ujPAS252oom z$`(xq9iLR|6sXII4uk_I;T_QZ1%>WWqYuiGxkG-^YTqr<&gI9=4X-5T)ED=;25bhD zX_kK$lV4Jh@!4v|GA@EeK3g-TvXOoFlv^>C@1s z)+ht{+4vp2Q+Nk>7NpMtUXZtlnkHN;>qH1+G=xjs^-nTp+;U8EZgEv%PB^VL!5=~! zGS?TO6zigsb&ASJu17fqikipWFg#}H;#_2Q2^}s;11gDc%0Eg7%t@^$jo)*L>!v>(!KrD3LxA zgUT_uTQs*2vE#1NIw8L%%o8D*;PjKKF_vn)ETsq30}eA<{?=L)#uF?!VBDZ!lu})S zjEi>2^a*%}#$0F;>2g}Cd30#mHyj$VHiz!eO8J`VcH{NOTHc!z9@KpypiVehDbn@R zAQfc;4PoubQ(}aDs^Kz)h*)3ug0^n7+;Qe2wET;@1;WCXY+A0s6Mp4M*@zcTMr3rw z7Q@4vrpFf-OhNu=MfNZ7+_Dh6R~IsgPlgUcinQM~`FmJTulsp<(uNsf*};lm1W(jh zm2eRK7_XO&BBMyA??<6t`N+1oo~+S3sL~6tVvmoACC&o;UIH=J@^3oN87F#HqZFI* z?FWG{$mF4qMzWW5O+Jcb{3E)g_~U=$otH$m-vg!WiK)_GieJ9uUuWsyu9G{*)#W-J^`eQ!^Cnz|?PRa(HPDF?Wy6 z1-6Qe98C#isY{TexJAJ=+BJvpDGNlJ;OzO2Dxp@(!2HDA6+JcJj?o4p#_9ZFp|qeH zRB^P3=1gdB&`Qf%ze8I(;n>|nen7>X2^BImU zPXFwKnzYG6pW`9V?RAlr0R|>(AM?M&bDR+!S-b#3Xb8{*eaL|QT=hBx?v0Q?v8R^p z2r&6*G_QC`QqZfr`>kxXmNE#p?W-3W0aDkkBNav#`3+H}7zJi!v*j-;O z<)?WJEWYQ=3LPT#6Fv}0uXYLXL}L?`ci+?5&;PwYfp%B1>!djtMU=aT2NFYd3W4tK zAR|074n<812$0NJ(l+SNCPM&MExnS1L)dJz7Ga`i1Q6WGqkC*?tuI{&93$#>6lNMb z<$<1Q22%dzS&ej8v`o9>Q9ZJuPSx$hlOZne=^rS?I-cn~uKv#y3cE{W3SK4Qyhx&e zvE-{lk5sfF>{fv&d|ZxYQib;VL7z-RwmeO7uu?;|o^dzeUqPTMOmF*&urr)#eTBq@ z$((ehm~oIQnMd8gDWs%s`UEq^ayB$lUPrUCyXk)ec8R@A|E~jf6-6-oYe?Q9;a6zp z5EMPQa$N!^;N}C!jwCUCXtq0vaY@2A88!r6_cd>pm?byy=OvqCg*dO*%vwAju<{dym>vfxT>tB^tRdQ9JvCMI! ze?qG4iKZ1#{#V6Ue;h_yR$GQk@fPuz{j{1)nu;6Q9uqtC~+L zTtq+0?(1CDy(B2$WXr%EgWWB!vPIR& z5^!ppu#41jh#HxTi>1D-tc~J)@dly0(N?g(6znBtrZ>F2?9K$2`PaR)?An?^Zax>x zp9@%z+Jo?%i`%tjCvm0xgX0`1LG9S3*|i(rhs^Z@5b625=YvL@lCxPG^BKwc`MKUI zlk-QziIlDxh~&Jj;^beo%A#4@07ykw2XqYDIP-42NuLWa`N*Q)D})#|4Arf8c+8)hm}*^TV>{|p|o1>)us4C^?b z-X|8_ggS3(vmegdeGYJ4C(hfg4-O|bvQOLw+}_n*BW# zk>?oOBXg@A*Xf=7Wi-=M&B7IPCxqdvW59MNcA;r&H|Rm6Uh9FXBQBSix9|&)$#jTF zY}p!a6}ZJVK)%UEsZp;I+vBu2L+aMmx%Ke=1@0n6eIuTIt1v*V@i2y zz=fU}U1KglqUc52?Xif|M!Vu+3il*aiXTPmJmL%yeABsim;^!*@U7FBuWsk7Gj~vg z+H}uZ8~!Ygw4wHB%EXuF1V=-8etWg{mcygCbm3UjOZmJ{OSE+dG9rl;(U#|ASGDC* z?=Di`vVdaf4xyz-R)|w&_F&oL`TLiz&K9&G$VjEhxzVuquDznRYeB-> zTN1NxwpMVVNucx-23=K!!=MGqkHRRsW=EUArsjo~Mr4-4R5%@JmL*FGeTkP->6RpS z@+fq7y#g{C+rypXGJIp4hJ`C7au=>Sd!uqu3PQq`663-m7r;7@t*H;1yB5f@Kwv^5 z9U0m?siEY)WC?r1#>@N=_ooejlUZ83i|B3~U ztQfvcZNgFL3txA?2vUGN}Cj-61qmbnN&tid@7u(~hqY*bNP zv}rx6C$_h7UCtYlPp#!NFI zWBhsdbXiw?bPhIb$?Y`6Pgv3cxH6#PDf!hGz)b#y#hMqquoVM!Pj5p}+F6Y%9_161 z+bkeNYKbI7G+Yj{_hjo2c4iSY#lB&M)ewX2fVkv_!^HqIf)R8`uSVjIX1HFHF5fu@ zWJIzkxs)+t;srr%xs1Cbe{8Z1;r_!qWB(T%+TzHUm>_a7#LNmz3u%KrUw4FiqKyU< z`=Hg^5c{S5@l)=2PHtG<-zQ&;^1gH^Z`sp+aO(9ncF!>GkxJm0=Jat^*T!0<2J3o< z*#vgFS(vhnTOB>xx3*rPY)gvrAA${kzrg~xn1J%_sa32sSWu)olk+IJe-_S&VABU- z&lSc8?%IEBtyu2ZHaL*#6CkF>Q1I1yRP^AvL3*8(RD$@e81+r8zq21u6t_9MEtDs41W0p^AobXhfqy_lWLZCEW73OQE<`Mx4l=yTTQvdM_wo(XrsSjirr|7v3=1BK#lWwIPE{ zhdLT_nq7t^sjM9M+z2~Vqaf;P(y~)<&S1RHZ3-I7OG>7egjP)?6KRFgaop!)0Qwja z9oN7b4^(}-DiYNe=t)IWLd<8=Jqyi*YS$KlcuP!#Y3D0#>>9DbNM>|FDM8n3Pw9<+ z`++DYn1IX{D??K`rRXKQ^FV~0l)X5MvIMQrJ2xTuEpnUCA&KUv`EvhEuyHE~3r^4W zC?*+EwGz+9x?b4=8z{&V5AIX`GBcqNuee6|lMjq7MX)hG-|KwjL>Nm!5JA=?C7(%) z4e(|3G|3n}FI?!2bL+J)mZ3k(1N7S5ST+H~xoiBZqE16Km-u)1hAs=WyLN&I&7}G? zO(iZR+0iypz7?bYu@UZ?6Lou0#c(H5`7}X8@yL?>L3OMCq6wc0!igTV5N88ZQE)2^oe)so6N$v#k zMl~Y=F6N{3qZMS6zn>tnE9Tg=$Ng%ed}u z`I1P{7JUyAcU&LeMk0?ZXm6i<2^ed1V<+YZe7A(Bsw5)_vw6pug0B)=h~>*5>)CKq zbtSnKI;&YifJHJ%M7AVhk07=iNt-j3C;KGyi1()oYt@`rak1Q?M zf(C6#__?@vU5}P7H7}N)Yaf?Z3z(yGNPG-#|4ok^HWq|RJO3V-evs57_3+)>{!e@d z51X*#{E$DojRcVeiHG0g-S+wRUGIFa9f~0Pr=P}0yczYs{511%oDDvu9*2RqghvOn zn{u1EV0IrLzFaVUM{FOwxqPQxCs5c<*ml!y)4=Rh?iBcm&q8n?_3ZZS@tkVw<+CuU zl&fl(x2S4@J|Vs_xv6}L^N97R*JAz48JlKNzy8|gKJLEYKGw3?(%Z7pGSM=#WO7TG z8Ev_%`^NccnIV3&dXDy#>68>Mv}!BmKIxp=SkPM_Sv+4npQ~O(STSqx$tbj$KW==O zaTtLeb#;7eZ_srqQcL$YCvF*t4Sj{t=;J4Q0(x-sM`ql z2-TCNM{eb?3@@4Aw+^Zf%Ip-9|GjIq$SstSp(FEQdHjC~NUH*KNwY}W4eh;a{OoTG z-nKwx8Y)UfBnm$4E?&nsWk++^|JlP@8kKY1)%$CF&B1rK-g@_6{VF7EzutunwF@=V z_0Rv<-|a>J$glik`rYfr?&L%4!}=rn4f9QGDG z+**7f5<5M+QQvm(cCG=Mbn|O|$y{%C)8EuRe3yR!pTB?NRPNR;Kdg^x4=%_jxYWFLO~cKSG{V1)MW59scanoCIku?hX4?6DXMIOmT44Xi@U` zH)1Z@WcqS+;c?`NcEpMjIyA^2zKRllsn9;$o$qbHvN$6M1fIRy+P8G`29Cz|>G_r8S-!y0Gj+B>AU$ zRPW*hr!i}NU>+HnD6|KUY&=;}pyT@v%i)dkByg(PJ)lb1P?{%5eEVQ$-a1l#yyx8B z2RGngD%&-{OYzrqYmo`+2hshEB1+(Zt<@4vdQPsDU?7>TCt=W()72di>0}(Ps31^B zobaqCq^ADnlGeog{MTZmYeA-V8lo$WBI$EG5vz&V$s~zG#q2imlgu6xo&Z+(olY{P-is|T0~x;8+2d=XLtrnH_)EZ zxT!-#w~sP45`S+7K!NM=#l8>wgGD1pCeRof?{*o0Ilw_n1Y6cDZa^CfZQRB6N2_^K z{MWH**lmPrStu#fL%6#RUT+L*QgqWYGC?S-IbP70g4r%~2w48S%ri`wOr`R1f#6c{ z)5kmuju@5+@(Fg*3%FOtlnyi;h=IcV3`nH=k_jLd0VH*oyT2i1l!WesH=nqixAV_q zOfk(Ju$4XMT~=wpRlLBW@4~J;4Uk#FKObG@NA7wG$~j$nBowqD9s0-0_w!5)PDIpW zYURP?>CXmO-p1+C#y=eEr-F1|NLjjRW*=GLnNBox>hdTMi2$!JlQg4hiDzC%_4FYO zdJ1iT!|6&04Ra8mgTz7%G(L_&k6R<9a?QQsU~#9Q;p z27rtN_a!;~aW(QrYfde;LqxC^Y?ogup^(}hIPb5EWH@M}`WBEDC6lcZWuO9x5R7(P z^6cNY!mwL3Ni%>HUZuM$vU>rXpS5)$v1qRvoYKbde7ZEEv=GJGs-6jClwvOl%)Uio zK^XhD8=`wP=FiuWHf>8mM%*n|Jti?fla2X{z*uQf|IpRgaQX#NtK777^ zE(FFWg$!)pt(ig}XG{pB$Nu^sln(;e9#P?8#s3hK-AK5$3cq@{e>QO~=6V^Dkmfc! z$6`Co{Y$mt@=%_=)dx9=<}6=PF!cw0m2dtK&#|YO0;F1`MZ>V4TeS^nIIXu>sF4SD z79%Uimk2YLNz`31qVi3CGb@#B`$9(H_JsEKaXaB=h6L^S#zTc7b^hqm)I@x6Jm_SpQ|2+O{6mlr_N8gvPcA1>K^QjZ65(X`gYQ=J)! zG{fKm_9zSWp0vkpmXN#Y5W9TiDT6vr8sHZKmv_48kYCqU-zz8wR1tAUWNj6L<4XD34!@f`wdo5)o5hiazc7~|=gE0|X>iOD(*%+kg1W zsv*r87QbV#N?`#YsN56!HdY7jh-0kV!)@qfC8ha;axjrnT7)Rjw@%4!tSOwD33#zb zeDnb9r=1RDYL080{lC~X^P{ceO}zqGBkN0NzL)YZtIs>&MYaXL_`qhuV7%Ra(I=QV z8wf|6$de^37RW|$?pDnx=`jRD)Ipzj96z~nj(`K*!XqShIwO?75V#X8{@k&{+v|iF z^UF~gB*35}TqsRg%YOxolL(|Py$N zP2cGP%T)qPhD=}4K9XbuKUsn3E|lB*;2`qpg+U8;`!^AKQC{PpXle5h`v&b><=feN%zrZm8=V9RKBsC$lPbqGi1WK4;&?&L|a;Dr}~L zDSUjM+itOsrk(vfjoP^0oL;D2;9nqKtnL+D`@sW3hJ>y&wvB$k4x0{RL7>lI+i2S~ zy{3j1W{ROrBSBSe=HJZ8%-GC_%;wBo?vio2apn5WHEZ=uVNN~P2?A6?eQsM$Av=H> z02q}B2XQy>@@)02`dskr^Gy7V{M_yv?F(HP=-QFo2j{Nr1$aS;Mz?Qtm7Z-VOque{ z2N!`0h4O_;eHA)Hvo-!I$Cg?&>gZMs%)2F}CaaE!j|h%vBqfeiI;Pjj9cgee+>Ll$ zpIqNu?_R%M6JO)*wR(*r=@jYdUV? zPMD-7mDy;%xzDC?HFe!cY^USRZ8tvnH9R*l&TsX(i}2Q3OFmWbRq_TN81Y|VD$$V2 z`!dl<{Et$}|Gs+pp&>%p*K)L+B9CL1r4sx=AP1M|QNpr6rl7E&4G={;h9Z2E1hRQC z>Wna}h|}Y)O4Kg@u-q*5;QE%htYQ&=Lw4!V=r6oP(1)F+Vrpx-Lf1XWeo15m3f5=B zs>3cdKl`GA(cWt^X^Bmpy{p>aYSij2G`fsD-<|Dk;W4|8!JP**3+!*^-ejRJQ+-(VdKQRIMqN-4cL+rP6V#cPTDZXH>csjn?7ezU{4 zh)KplGJJItVlc8+2(l-|Mf0K_qBw+$Q?~5*^IRGDLEL4k(i6CW!(;r4PHA~pYw>vp zS-~n?xu2e!U|BjH$KT&iG-_`ZFT@vN@Q*P9RC9uPxpFJi{tdy^K6$TsT|$!E;DTiB zWrGcccBdr;Q+by=4wU5u&m>En=jf7ZCHrJtg*7Tv&W&rge!6a_vg?l>ME<7aQrGbE z(HJ_95U!1C!V*#C=MeM92Y1&5R0m#k)HWA&4KopB=?lYb%JwHybG1-UDW6 z6IrpRGVK@JUoMR`Pt`wpSIfg;(kyWbaXqvKuL)?3D+=#NTVY3V@{#FJPMjE>%q$_uM0UG^h%EGuf3Z;9Ej5!>siKI z^HOkdv)_-19%A>FQUwi`@Ipwu#61;ytKJ+a7ujvr1Uvj8g)c+qOL$8Y>=`3$!kXFM_)9qr^ z6|n@dR<0H~Rb3C)=IVK-mDRR|rj63a#!2Iwn|j_y{IQ2$(3uRzB{z{U#xTD`2?;v9 zrt~EAf7EdP-lyL@pOjkc!W+CJ_XBXF^F1JiuSyRHraN*IjRP}OEnZ z&1UHqm^V*npH+TNue)C3XPZQ_b|)lRGSHCOdN9@36wzBcr%UcWdn0%8FGsZ$Q}(wg z`&HK>Sh6dksmbw9?8b42>ohgSK}p(mc7Dw%$0*E9IYJ?dg2s?BH-D1lW<^!L1+5fD zC&98GA=Fl%mYsmJIt_|eH~XQ{>dic%$S&{=IQSBEDXn$J%%T;#uVjPI!ee-xeSZ<8Y3l-RfW_KS`N9 zQ3u$k%}U7%NC&ekqkgKzW0}1ir9`j_SDa1f^toQs{e7+NtCUy9y<91LSZvPuC2Ib$ z0>769YI&KWgh{}1klILB7b6_@ar%hCu;g&Ei8j60GZ|m)41rhLD1mP)lO1K4kq;>x zAH}k|)=*Jm#H6pRv)bWOpA6Z=nAnp&C1&7D=ioXZfI6%MTf!ll=BdeYocH3*7P zZ2-NENP=vr_Kj4^RIt2J{rv+032Q9Ipe-@@95xuCnUo?cw&7@Ssvn9)YYGGD2{^p> zTXIv~orK>AJXuh<)5d6u>0zN@mQr$2ejtTIy4B@NoI7?CVl_pLKDlFI_8N%ZlbvfP zAmR2z@(|aixZnQPO^b*)dOoR?^KfoIk z7p36yTBnQtCeX6U-;n0Y92G2r$1vJ0kE~rXjop7aC?oE=AuW5`SvgIsvkSB)>dy)U zY#Pep|COPgjxJstxXUlkIE)@u`6eVIVCk`<^29767l%8ze0oN04qa2A<R`y(FxB1nc$8{Ay8f|&T&G_dqnt~_{m&NHn7 zv)FYoBj!eP89BK7=>$r>xZ5INBknOfERswIFTB4dKC@h=297JoUsMgRf(J=)>x^h1 zFJ@7X33Dd2FnooJ6nQ(a`X*ew4!V$P6%Uu+Op;KWDJFwiXGQ5IWrul7=wS{ey{Bq{zF*rFrv=XTnSDx#Pt***@< zN{x&Fn&57t6(PB>FbELK@|KyWG4G*qDC||*Mfa9mo4mEC{J-eLA@<={ zN^p<}0%Awh*n6rF!b}RDMb0Q!G0Os(je>XiTS}Z`83R_N5te%!=)FB8qA0VHt-r$e z1$!CAp1PZz_SZ@co)d#yK22?$MlS;!gdIk=hz0cp8c{JZ5d~NO;cyUnIPc}eXG&fF zFV5aEIF~1C`~7Ffww>(QcCusJHt+b3o$T1QogLe@ZQHg_p67f%^}cnw`oq-BRM$+^ zRQL3?*7{wfDba=E@jn&?*NENBULG&KyVdYHcvEnM*j>&iD>LT7F7S_MY9}`N?lW3YpDFPUmjXdL9;PO6hKL>XJTn*sqe72pVH5uaQ&uxY zul!qf{Zi*+KwVk0R{dwf+%y%o5jpFarxMTm$NrqAXm0TGS4#`bHPusY<=*N1pd+3A zf6;i!WBDq--@x|?Xks~$CB`&M19JF-TQ?ms#l~N)9PVi!H;#M~J~QjG%9kPz`^Y{{ zjg5YSrSysoXKQ@{5PJ>{Qfjr@t@o`)aoz#TRO)>CWd}*k%1siyF>1Yz*Z&#t1f7{x zNm1iyCXGYjd<{|`TB&t!JjvO>N^>011C`!>oqh>A_*fspi1bxRJg#q0FM;4Ai5)!9 zUyW@%nbqoaLk)`mhIN%1Y-Dn?c`{^ep1k-~#IYl%4qJG(G1ZV@aF`{9e1PMv{9F^; zP78j6g;4w}Y2woDK#Y-x;;%h|9MZ@xl%discp+%Zz($c#-WeqBDnm3v@#3jg9epH^ znCpid9%Ny1Y%WD#xn8HA4F+tgjVpky?-u#0c8Ng2;U2S)phBft!VX_>&y}g*Wib-M zjia|2XexjwA%d#?$1)TGn(H72+^`k212rU&l9EUe!N4R^GDMRE^AGhOJV@#U!y&@s z4)->tG0d?qotTMH8<2Er5V;?l%C#x~?eDq~QmjEPK}smQ#N9jTRxNnoizf3dsW{me z)od)I_+a*)^q+DpyB$`u38PMkMazA?&=&R}sYh0kGoq&)8l%(JGcv4f$=a({t1cP0 zs$ZTIP0B(Kw%0V;l3Db4Bln04L-)I-~Xr$zByC%&%bhjw$T0SDAI0+Y<&vmX>!M*9;F=WC`UH^L&oaR1|C7q=(4fztvq3oeX`H%1%%APDL5fBK_30ijj9~;PQt4lqnX1su9AYz==zu*i3+0W zeE3&|O22BCAbb#5cQ_Jx!P%Yx5^a!pPYo2X5_yPw?B5VTp&}Wkhic~!Fg!|k_m^J# zjgS(9+Ss@8K(e=pFLEO$P9@}HY^mHH&3yBl$0EdeWna)_pIm2+wha=I zMNIC$w|@!Mr9u{ol6~MvBtz;kA2Gng7xE9ye_wy(H)N%@Bf6=*ZPOjVY)k{a{-vCl zkqQPzV;MFD8Qji|-W{H6hy0bm7qk-|p|2~l#6>r=-qoqn;mP5VEmSGg-2oHotp}-j z|Ma+Ghmr%58iK~M&)|EN{Ff!@JZL|FC%>~s+Q*rB;L(UuZ65b18G-a`IEMW97+Y1I zYNS zwM>G?Ik1zz6^BbcvBN!sj+yl|$CISFe(-L6upO5mQKP=So-P8#HtL~b;vab*ExO=r zLn=byOk`JK>>j00gG}wLc!v}Bs9*Od(pB=J4iE-vbEygk3oizRmT)qPyC280DxS z+0>XLJ{i!kLsA>r7LlXE9X4$;ECj6?D?S)C$f7^6dIXwO7G{V0>x{4^!jdpVylde! zQsa32m}vgZBCzK#<|CIR^+mI+s9ho@NjaLvl($<bPZ~15Ax%1T6`XIqYC8tFTn6?Z(m%+JG|d7~+=ff}&3vdZ>kmz>`*CrtZLW2;ZOKJ|++c-E%=7ET= ziLvdC&0g)0wmN?)=(f+UnVM%7lFM~xx9}l)L{6k;+j=yFRqcJxGD-12naO+N9 zH$;|vmUyK}k!ay^Y(TQVGCS1C)1iPR<&ueby{x#1CrfMw3Rs#`*rGOoL`Pf3E*O8T zzbss%@n4NBYqUV@t{>jpV-daax)2f9kcZ{A4&G|*3bePreIFuUe(k)pGC%>b_%Bpq zvOYoOZbOr!?Jz%|%sLg+pHS}m!dqi#)pP~X1mn6-dhK;MElSSG^lO%WJ*LUY<;9%j z38zoT(udnFMP#%)H+5^LyeAEzlJS?TP&XMm;WE4=1xqc%`;QHR?kdqA!upj)0uKkO zyDM1VxJkc`#46RMXWvA?5D%b=cBOtv$aiRi^cQ~HT$lBGQC-^F#TFDAx)ba^GC*e0 zzvVyW%6hSCY|w&c`KD;!*dwqES%AK-TaMN0J3qi40MM}-*1)d%`k_Egq6UPvB~zz6L1W#w-=a zJ1$Pb@LU{k4~n~Tc~ye7y3-2q+JaB84C3+l#K=f`iCfN<@gB9~>6$%7km9M-sNPHs zIU7Es?L~ndRH?1xQRq-WDdlF&!KivINeG8*|4vaO#_5y&z;xsDHw#;)5YjyKvPG?S zI<{oo)gEk8#G$`XE5l#^`j>jh7HpRo?v|rX&Vs{iPR2ile%`|Of|s;OBkkH08Qwkc zocKNip~ z5cPx6Y(>~QljC0@>C?ZG0_5WQKdp)`)JMYr^V~)F`r$V|H*>oe2*z&wlN~` zRNiy_`^954eyeGlcG3K^xgbR;R-JQXr#^-i;laij_65z6>0zq%HJcP~1D)sWqD_*c zx08^;cMO!7BRKeRyS=-3JA70=?>33L&}d?NUteUWOtT(>!aNXwWDYma+WO1@sGy0%d_EfyUMPbx0AE9MILpQ^7(SRDh(RKLVa0* zgsx2fgf70zBIDDl2mLLMt>LZnt@T!DL-YFQy!%EO zH}Wy!JK$aT-T7VeJ-*X0r-sgRLhNaJ$$W`t33h2;ZL@8g+p4HS5D@cN@S65o^&0=0 zs>$(#(##;lwZodmLd5w#5jf2`zdmI+mp?^3bN@uxoGtLk_$qiRbdekf2+;Xw*fn8Z&G|6D4I6yh%=aQ4 z1m7NSi>L2<;`>@10?~UZ#lESZCQp;A$7V4yi1Grw@AoQ}Pg!N{W;v-^X;D)yvY%Mq zG*4a^lP9FLkOe%o?MBVwAqA@|LKnlMAB8Ndgzwh_`YbV%9n7~613_Vsq1f)tg^3o4*u)!H|vKx3Co`?xUwRJhs8n8Xi!{loOA*OML~k6*uVS# z<9_Y*`{ZCddJrPmH_ok^>&+3(=0LYUg08y-qe z7&#I>j4^M=}pcR{62lgGKW#cJ$J@xi{ z8|-$KHFb8Gy1o}3e*E)&0yy|X@3SG)Se3Z!b(LBFcMP?`b0~)y`+_pz<5Z;l6o1_zjz)8Z)q1Q4BctGzAnQ}&otpMspK;N# zT^B1^e(}`Gz__GRnK`5(Fby3YmmikyR*B&4!+TP&>5;{|A!&!?-Cn!tVH@F$- zyW=@qrPC?di`K5{XuD&!QZUg19;sD&%Z8=9DFX4OW~JKNFJ^{`xc5|SEl@uZN#_MR z#@`#5<@yGY{MM z1weZ>c9z4DY8Vdus{(a`on&in77frCa2*E5(eB)R8FYz1IHIEd5vL6*ZDfT$R}%T4$BkL_Z7Xpd$F&M}G6~CLF=m*S4fdvka2`_UmPdKAUdAYrk?N zlt>jS2;1q^SPyi&54P)bJBwed8Uv;-Ob<VXGo#nR7GT=p(D(Bml70p?M(bC{u3sVawBFP0YH zEY)~S&*+%wZ4(iCYPn0=&!VpY`S~2(OxRteM&{h_T77zsjLuPLr{P`{%O7XO4#|%H z>r9s6;|R@701V?h4lG`+bI!$-rsn+7;ahz|(C>LqtyXUGp(VzK;@_o;!?M!&3Oli$ zAcfwymA3z6qmIh50!dgq>(V-W+v;##cJFkj{<)l@B1J?Pcxav{L1C#&pfL4-Nf$>LRH@bHaA&*btG3v`cgZ;GV= zC7&{@Gpg@8-kAAom7(48exA}qvp9dy`}~(g9_d_buYAh|)P~~y7mKbHIKvMW&4)9k zpovDfF9q~^Zwb8i>j%l1;!@`8AX8y!V7f@HrDZ0 zJ>wieCXe`K3gWuRuIDk9BC!!9Y=qkj?e786j4w3z45K2-{ky4JujLQ*&;G9Gn0FD! zNA6Z5H$01B_5%vF>%RC}(-JJWsoYqZcUCDGTa#ue`LsISj^Jz#&J@n#w(8N zjOK*y`_!3`CLFFds_cL09r(wGYnY|dP3s2E1$?XAMiurLfZIO64iG$m zxXKjSh#mUt60aFCs}?726|{FM1TAUD(gX>FIVR`D9_gN}v;yZfW-`Qa@c!lHH=aSH zWE9Gc0v4Y+QGYo(DdpS_#YMuh=XTIU(-q>v=)+ZGQ1L3a8RzaWKo!;M0d%^jo%LQr zh^RNhg#G-m+=i>{w-atuD%*Xm2g7qwnb2S5^~{EtJi+h`ouDr*yctuZ5ObK#nBdaZ z;JfjnN5NGTbaV!7N-8&6l_>HVK<*y5ux5gKD{fOA6+V0g2-4^A=^#8FW*F>#3|g$XP1PyvzHnuxMJOSd%qt=QR#5rjXYXPL4j`a5||$G7re z*0?33M`{Q2s$Gu-z6HH&UE%XEc-E0K`>SW%UFPgq1aulo|M8Je(blqtWi`gdC9irh z$)QAxOP#o41@}RTCwI^*Z(}8DKe)$k%T@e@xQe^qpTjNDvK(mLU-k15sN0VrXq()7 z7#)UNGZy~Tp?DJwl~$S@=o{lTDEE2BRZ3BPcT zVJk!nS_G(jW$+HIi85P`dVhJoXZstTF7?ouU03GKRa&NDYLA`-Xx?Gy5Rm|n6vy_^ zHh)J3tqFLp4mWhUzt3?Z&Z@RR{X;tni6TPbKrITiGM5UFU%V}R!^$7)xshU>dyL0xMajWxT| zw-vFJoM%n@bLZA8NX`i%cD9kvL~TZ@*-40#0YNab$`)DnH}SR5qIWV238E+a7`uq_ z0hqNO%JmoA^KVFHeYVOvCQ4&st<6ae-mm=-t~iLe)3<6|1!6L1KbqGC0)$K*JJ^@t z1%C+cbj;T>xOZx=_)&~`1A?SgA6RzuT=y&^FFjiEnTQ=r3wvW zd5QK~T%$O-I3={Yqht5R44c`%vP%E$jgZ4Zd|UMTX^|nCWtH#|^RektUbAj1T*FtQ zh_idg?n0U_}&@zvybi1DKU z&9S(BSc9X7UkZ>Q+e4^D_4TaR>TM)x1CemdYPJKGOAM9Lkw@RMcWqNL6Yu2&TVvAC z=#Iljvc0bKF8x}tvzKb;qAXBY^F!SFeSo?fXcWR&c7;xJh;U9lA!$b01S5L65^7+3s- zb4sO2qNbDG(%L_Or$qLuR+hwX7_7pHN&i7r`g^%Z7sI1k_rP!Niahz@0E(3)S;`)x zZ(bmQLG@-RJ&IXnH#D_d$Sz2vipJ11-I)?^w?-LyMEi%O&l%MsIJ>td{0vtx0lTID z&`M${;mLY7Je}gH-%hVe%_Gx}@qq$@No z1jPKC5jf(f!CqW59%MkD1miYae)Jd6DpCCaE@cYYM6A-%zb>u-*Siw~$pyr04mYX0 zN390M;)`hn7jP8+9VD_v#43<2uHX_KiEjbW#UoXuvrTrRbftDh9&h~B&bk71R;yTc zWvfr|V{>CLg%>*|R61=PnZ-^A*z}8W z3GHUk*&ucm7b+Y8^@FLpMPAXCe=Wb%bhMi8Jap*1ZZ&QaW%vPgms~ytTHTt zQOYnuON}lzOPN^OYXA6T_kw^Z7n3GyNJXc&X@mlV&2I^Ub>jT)QsO=0JTl+&gh(uF zF@M=PMZ4s)v>yf#4pXCu_HBTMK%OsTjKwaI3v2!UVctl*1+H9(S5f+(+a2K7o4 zl}iCKX5LtmJ_d={fxk(Y@_x<51Ol?}hCku>8|Yq|VaC5KvU@+(c~wj2Mn;hhmuCoO zkfKWQ9>rhbpt(r5E;C1WA6@5{bN({aLD?HMG=G*tQJ65sn~;i5!#tC$Fn!pC&*KlY zS1(X7kU#q*WuQxW9eN+40)o7nRxSG>A_%!g`uf_dv0 zYf+``KfgIwUQr9tlZ#Yaq6ehF*MxrTTSJnD#L3iR8PO=2loYfWk0^~w;fY>Xem8!N zzm=SCS9CW=|KG>x{{P&0`prCBddMsGA2i_a6S4Tbbtv1+@K>iIu>&{~bb90yNrK4Z ziCGaG#Nnl2C!H(8N88uqoA|k1Rd>zTiih{7|6{`WmWOqx=V#8v^eRI=p^5%9e~S0- z_JmFbPg;O!!D{pcAz-`OyXom_Z5$w9$=A}<=(4<)*LbjYqcy%}muAP3W3n}+Gs+v+ z8arx!HDbaXCgTfX>ZAMV@mR3Pp{}ptZQa=Q)^j$hYRA^J*7-I)Hcg{nx7xFJMLXHA5ubUUS)TEq z$(|*j<1<|CT=_VzT-tqk1i00+T(eR#DKe#}l7S_tVWyca^Oqks-xl9o_n#9bEb5o8 zmnj#`=Giqr?>NP4Ai%Di#bv1d)UavzZTV@T+ql(umEk_>(Z|hQFqv6TP$Q>Rz!H}x zP=Zfm<}I*k`e~|{_0{X;`c`y0J>fb*bA&n{li_^tbHCx_^OkCvYWxFk32``na2=u$ zGvnspy`_IeW{2`uT%SCjQ#_)3Gvds9 zJ|>@BCrEsI+Ge}b!`M!2vrpv08s$+a(JzPE-7lijqhH~Wi61kJwjJ-^GnfN(<#Hll+if7z>&z5T$ zhQD~8?pC^`<)c3voSaXyihw^JaGfnqPPA<7yiMj8wMF`fKF&7{534}HS0xUj3U zFDI9 zae%OD6hiVX3ErF*--F$Ow1LeqRMMf50g3gI>;BaN*Zv#?hTjmVl(*v>%fM>{U*6}r zn+)LeG#`_f?MrPdWbiofc>+J1hpA%Hy}tlh37qf8@ApOc#P}xvlKdFXYu?j``*b}? z$}0(`hwnu4Wz8mR16ok{*nL>&M3@I-1dHu|3e(*J>Mc%|d#la6aX#$-M+Y+(!16M_ zY3&VxB+Q_LS2MpR=&*a5*_qX}aWS;FcX6w6wx7OeTxs;LhUCY4;eIl_cwD)!<$xrJ zP&IxY)WhRMzBh`2&%{j)J%qf%bMrXplkfp!Q)c=9==}{@_r_OS23jNRZg}S5sZ+?% zOmVitZYQCB77&@61W9+;SbS8(+{}Cg?_L684Q)px%9!~%EDODw#8uW)F(lE7m^O$1 zZj@}4s8xl7qtHY+iVJb9B<6`L-t%AZN#VDdXhZQjsPee$=9$ns-WMP!=jfD|9|Kha z)R*BiF~gfgRHR9zLnu%56lnT|lewsuU`W$Js--8SOxv8jaH?8w1_T#l8Y2}U-LDw! zD0*p43gyQhogCKZ)2}MR`2rLtPm;R!{;ab}r+7^DY*vkWn=CuSYgRpE^NPh;dRmTe z&@@7-ujg_NbR_$;10SW_yCX@|m*ES!B>3bC#0ARi2)nx|b#OEZ@vbrKCY2I zK2m>oMZ=}Uh=Gx&`*SUD5)3(goiX(~O@u%U@aret|Eih}WU6F%@T7zPC=R-_%Eldv z47~H4sK*fTUrOpoy?(F-2QotJuM-=l!$1GI;T963!kw3Cxz|KYDmdMTfl&Uj{D@(~ z{S`x~YCjiSNR^3Iu*v7 zhstGuI|o~i<-BU#6|k@l*%CaL&N!V5Mgr?(CpmRm90#B(YU93hnrFw?Og=NoK7R_# zTxU|$lIO|B76s{U0zv4A0v{nRgTpP%ObZUl(*a+UH4uwQvBn*7WR(UQAkl#g_vts! zRXLG2tiiaEnoG(aW5!P*ju^&__@xdY6Nx z;4$XMW3M_X3%?@wGi1i=4Xx5U)HJNxl8=ih4(pJSXNhvykjbpUc%A+T85#`7w)`_v z>EBxjmdreA?D-YOkGw?x%rKPIaAa|+20r{Iw${5}NyB+1Gd_c-?pEeXI2hGXa4^WS zIJ6fBrIVL)t7MF&RL(RHaHdX;=m_!Sn$OPaQXGZ71p4!g<1HF|F6y*A9MPZ#%ZxwH z9Wj-GRX`j1=LySOOO{5BgKy1>CNhH9|EG^VT7n>LBXsKIAtcBvt&YUOS4uJ!8; zH8Os~+Ql^}El5a)hM3!nycLBQMC^D-hzfGYsdJF_RkPu9YOb+a(9f1NI=rrHfS{Wp zk3wF~!o?Z{r^?@>j4`n7*E~p<$WbrQEF;^w{2}xA4x`D_U@qB3#+4QkWCjc$pxn%b zU_#@f(JBiG&Q6xP*WS!^{H6Z4i!%4gY~}xDgH=zAh8ExHD6q>HP(1!R0+m2-P*;KK zQiL(4dJ5!imLl6JT3AR%7m;?2Fiq#4&4|>F4c{A-)FD0TT5{Qg$;ZAVq4+OPR5Ha3ffAvwWw9q-TsN=HQcV2tT!qF^nSV+9;jB z$UtMq@{(9a+BI*6U@L;;ts@O{>N=OvtTBkmSgY4y5KAbJk8k!#ysriFZBVBaZnBjt zQZuAKH4V;K45Q2(Obe$ARw}6AV3_q91C5)*UQnMOW$IjGw*$j^*$)1A@ztX)#^B{a ze#YQLGq3<#BXfiYA)jG&3NKy(aS1ys6d!t!c}vFC9?gl-t0nnL9obv*gIx?xzW*4~ zTZ(^jC*7WhudIfy;@C3_MgF8J!qxmLFmqWjV^QkRT5u)F<(F-~$NvTamV-F9R?@_A z&ilo<=^i5y{sr!GMpx+Xv-MuYts$0+tTo7!kP)hKx$%c&_MJc|DPmm&23VEWCD1sJ zvGq~%OV?zgGKW7$kS7b@*@Bxb*Qk(l{bkECc09LH2%wu^EJ4T1(Y7c))@mO8QDZC$ z{9Nyf`b5=VgCIg`tC9)g0XC%K9CX7{(m%pA4m0uM^sml7X2j8=Js>)%O;!EyWs5)F`rx=;`^LOkmUVT>!G^}{9FsJbnHYg8=RKRMNJ~!cF%^yl z$5WjOi5j`}dX26!S5SY;VbiES#~&S|D%baeY}T7%6<0n(L_1n6Tq~EUSRIegwa#Ls{a{Y;}A|}fIh-=!yl(Ze*ju8gj8mA3cX+H=1VO&G& zq$z^WC?a;d?GiJ=1)){uybRU|H%!4D0yCrOw%Rb$ep@>l$>(_S+L6=jbH;bq$gPR1 ze^t;_LKCQ^a5hVRJR)gO0;d1)7LwSiSdofZV>K$WA&I$V52M2X2N`2YCMxwI!%@l_Z?=KB7qi4_G9<*5bNk z1v{L0l&082^}G_|r)LqVLv5P=g)S8liFt^L>ZreFU->akQolNt%ZoN7zzEr?{@VH_ zim#zLBfAO>Ub;a_EydpRGd~>{tzYa&`<>0nsotOPYKi~M2Q*7@Na59Pjh5h|CheXP z;qm277NjcjK?*HGK-I)jQXnzfloyXM<;092w)5!Ln2xG~NEI77PF@r7Gp1N0IvL?9 zUY_|=uRG*ca3SJ2S)UDMRZn4f{N$|#6j=rVI+>K3I28DnhO(mLPF|*qU$&RT7-WcM zOVyZrvg}h+x>tL^U~#4Cp;w`-oYpHF7gv+IN}`S-T`47(-WhzJ2K>y^I3-@K;pis1 zP%Xc4$ej^+&4o64Vsx)xcs#5%Utr&b)z8>bO=v=bHe7}Ig*bQa$hmD!@=-Cm*5*Ge zf#gx_>|eA*EbNUXD6(NeW|(1Oj$j4OCM-3@{JHJid=ab)(xR%Gchx(?!5)dR?qFjs z9cy2xK~(lH~F%sCOTQZ zjTW6U%OCg-A`%?mv=tNnAiA;k>xW(Q*VSX>rzhTy5?<-8Z2edQx2C}1tlhyq;jw}# zilDW#1O7$jH+_n_sZ~s&+uu!g@!G-Y`t|ZtA)#eM(1rO#Nqf{%iN{pvWFeO>P(k}= z4v+5-8gDhBP<%YJ9rB%s-_MNh^y|JM+*4a_L-kPG$8*hDrv=NsRK*ZlSd_YqKFv~W zeG%c&=*(&JM&vpwqC%;F=+e|uSvYn7N|DKniLt;a_#Cbrt82_f1rGKE8k=-I&JF55 z^9$)H{EeWWb{WbrxC^JRR52xRcU+DpkgLtUV2E)%eAl;dv7?1M!cb;BkWqL+0=w8# zmskd9tH3n;7%Y@}`CBNJY%5c7f_+|GW#X=4>t{F*_9W*aEehe(G=73UR$XPLO;mN#V9Dp-a4ZALXyJy7SN2dWC)5%UeuaD%$3tPY zfQj$#*GKW?^7H4)9F6Ctl4YKy?=hd8Ps(?-C#k2=i=Yd!Ckys8{Hd$auePVfHF1Dl zrLLx@*X73ACxBvEZD|ki3GiIX7Mf3EoIxEmaUq_L~$sJT0l@;>z0dlhJ#t8=9LRDJtj$kJR)arx={ zocO$!2j^z`*W~-u`+(=T=a%OHjpwsl$Zh9h>GGZ~u-ir8r-C>KNa!+t<$VRc>`1W@ z`rdw@bsu)$1RVUy5#f2}8OH0!25Myio;aSwpKzaGp9-F$y{o<92<1gDLqFReD<5kg zo4|d(k1OEMfAh1X)sQSzA8{N}42*v(0*5=6d5!2(`+o$FjN7PN=a2pu@uBlu^L@nO zZDyCf=i!(^eMSPd$d_m)47;dr_V>+;&LPG91kFh|PnQWd%crna_Wg|u(0U59%2k1f z>xa|T=+$mxfkFORftb#s?p%T5lt)+N=QD=E=o9$?rMK3Vnl5o)qNjQnlBcAjLH-%N z+3Y!Tqm%{-+Uie!VEfbML^#HLTZ2G~K=!N8>-=GJvc857@aQ#7+?+s@p5c08qtj~} zZh#3ZgR5bKIYFn3{d4ibx8yzLTk_AH^Sxme*WrHCPAOW(urY_Vv#I9t=hqKw;ldSPiUu8b$Sp)9Fr zwroXdS?PQsXOu8rJ)v8?JIV*eo8eXJ?({%-lx4KJz?HmOL_Pk0lKDCEo5lA>w+^IO zF)pLOSYD_1cO!fK32R2~j&8TvqsfB+ z*lk*`7JHMssGws^0^L%2iv;6@xKRP}D^1S2T)LmU2!UGj-T5&67@?cB?A znCFmybVmj!^OxgZnTXX#??9Qz^-ETm_1-qq&F#0TwftUrt^N@|-n-*fXRG?r7M7>S zRbK1y@YKX6-}8xv^To}^r|~>cd^oZ;thT9jd+-041@lwWaQa=l5b%N957kDTBHp^` z(W_W{DU@Y=iAIC|U9Y|2uRSukGsKi7N4oAGaV*_?-HWh-!Y2TIM50Z)D*-kUkbA^2 zhP9%zzA=W`wafJagJru^&YNStTW%zua_8S22-Pxq-gYgQX zK66XE-biN0HVuYob*^?wL+&;T#m;vFsBx~dnVyvDW;d`cF*kwpLS>^1Op|Ml++{dd zi+Df>ZLKm*tztFj#cvjqt{%slaI?`G%B8!Kw1jgiY)`lCEwRZ)-IKCht9Ss^AUc^= zyCd>~S`YtH4#S%cduTX1Lvd|Sm2YS;cwY)*`nXjp(wamc2d^pR!frr~;9quj7PjC^ z+_j>V{jhP1gr%)vU1zLx(z&465Fj|hp5!gftukIqpOPMQTy31AY*wa9qGpr<&t4a* zGIMKl*PsSv$q!kbPc*V>(SWloKA#vP^RtPuU2mgE$|aq@sZrr0YJ`t$lM}) zNptAZ*rrwiC>ue`;rKttT24w*62MK~TW}`)ap7*6mTFaQjR^ASwAZobP+%`l@+j&+ z3}X_&v!5G6*&v8J3otieYgVSzrIjep+27IJKg&{*#jBBKj&#ZhI14T~-9oP2L^A7B zv$m`K&(A`m${IN^Lc~(O#wCE6t_Fn_9w;J6e1wDr9#IOdXO?aziGp06-O`%uQlL`> zk?b06Cavh#!?4BXj7wYiPY`ik!^8G6`Ig!3pOfO{5`03H%ArN64*oTgRLFYmYpa|WSZK| zTwfalBkd=}I)BXILIy*Te$-hR%2!rH^k6$qq|1>y!;T^F1+mAzqaW-rvoJjGswt0Y z2gKj|lG+%@^l!2oWxv_v{LKHTJ?qwH7V|4f6GO|xf>>w&-hDJ98DEie5w66XMPIlZ z8ruvGM4W{O{DMs^M>03`w=M470cMjv+qpypnft0 zS@Cg5N{DjZ{H!-G;=3asF((L$4OpN*4A(3Q9J$piVVgR#M3gr-HmQz)k2;?%53so^ zP0ci~V?O-UkfpX63|DA?!AYZ6)(NuSHBKRCopIC;duWJrgSTKLGwuRrGb6sC`q=#a zwKinbcPPgB2x&UW5@(gFg)<~osZfajv`DfYgNs`&uR0e&xPM5}|B`?&6iOQ%tZFQ@ z<5-DI6*28ciIKai%QA?gSQ9kO`v1y7nO02Xhgq`+)(1CS6t(K-*|D5px|U^9hJY^; zQ1ut+r!*KXMA&x{2J}sQ-im{?q{?`XlHEXn{H)|H-{Is4%vk&P^Eb)jFRa@ zz{zmxz=tX~_A@O;+;mvQBbDYv*$%9VlxQ+(63^nvuZs*-8I_nJylF-54YAZ*T3u}@ z7Hc_rP&=J!y--G5*>nQZ;ut%mG%QQ-CAHXS|K0H;v)Xjz4-6vT-e|=$B#vu9Yb5!` z9K)?D!ElpvzV12?M@n$%hH$k&7a3u9tCl6h7RBeM?c->8Feo)RR}gIf_8t|iI{&lgu;K~ zKjp^;bw#RFe&fF&bYA`Y9}Uw8(?Kc}L70`5JM+=6?#raJ=*LeMHyBI~VEk7&I?_11 z6ARrn7;DD{4y!>~i$G7^(EPxQLaxJpG)>}dbQk?ggr8YtNu6bzQ`#Qok$6YQUwYI> zQ_Q71(bwmfn?5%9ev1@^y2YXJjH29NTR=AHC5Uc?gf*JuVn>6kGx3~1@%u@OoB8qC zJNm?*Y(9!*=sqeV@9nH)$2~fKOP5hDVg-^VxTp+tm3p{w)O<3~@DxNHPY$|KPskjc z(XlwsZ7!VkiXmeRZRmca3(1R1(#Cjz_av#bGIVeWDeLm0aVWX z6_Z=ct|KH&3+^dUL~XLoSU$%0!o@@O>p)SR=mGrcQBc z+;g5oDJu0U)5AKZY@Vo?is(_wEzG?HFi(Xy5;IN=8#eMoh-wu0^7T%ba2lJmr-}t! zH^NKoI(>#!hm|T)@v9yf5dMiCTB<=+KR#amWhN7SRZgbf&r^au3yNHfExKf>^`uHG z9co<4_0~zLv}?H4t3Z*WOC+R8Kc-*0qTxs(@&`3v7AKC<1|h%Z=?Zd@e6*F1$klA^ z3HSNx8uHiaTdJQ&E_?7&7=9>uCPiLgqpi2Rg~SH0%R2U~O_!*|3-&t4X5c@Ltqm0K zq%7`O6O2T>G{GHz9>z=RFho~8*s%!)nR?@6Z`rUZeLzG}Ewj>MGl3aAkug?3z5+>=96 zS?*v1A9REuWQn4p%#2=r3kWc}4Z+mi1>0DQR+_4&i#>%j@??j+z?an%fueb*U|u;cdbe5CTrQ=US3^H?kK4NL?bSwWLA zlzBjw-`b0g!Tp7M2YB-5939lszbI-`ztp8iGVsNn!i7X*YfbwTHmXJp+tiv|CR?Su z(*0sekXL-x~`vx1%-O)5v+!2m*N!Sta9f==7vWHNecBku3NX$ zY^*r?w({<&-B>eoAP63QbjHA1??2i>i~$}1ul5NIHVF)5kyRj41ei`na(A@)77Bj_ zm3;%g!h~h(nb>ZA*?|~AHdSz|zTho^2&OS)6UqY0R5zv3ip>}UdVrFGRuxn8Ep=_={EeV(4h9W-vnc zecf{M?1^TSg`0S45^+>kVzUE3PPx?1xoA@^IT1l;uef8*ewcr&82kx|SN^h)`O+|L zup)nlaC_@p-_*(!C}Bxm{FLO>nL3oIlF$3jejV=PGwVh1QvA#?Jv&itXo6dT%W~0F zp=OuL$oruW&K*7~5wG(>vPf`42m9#YV1_P+ z#t7IZ41@nzgrbX(A`g+02<`Q5{`SU3%&r`b1C>z-t4P$rrInKF-~9^BM2+OLRq=rb zfvFp-CQgnp)MgIpBPOaS7U7Ik(@iqPo`jI)NE4CYb-1us@8oxZB zGAhbs0&jIgE?+L!ctt;-U-1p}Dp!?+e24HXNu%>-mR`=!grTj(n zh52R3h4Ry-+c$qy$cg8<^113cG+?+57tn>;lH$WIz_QJn!$inR$XIW-Va?{t#LuzC zmd!%IO28P616?rqqWG=htLm%7aPDew%2)AS=AGAB#8;_1b2ZDXr2<&=A@6#arvFh6 z3|2L>HEcD%xSZJ@&Cb#m4j0tUo)S~iHyK8WFOK^85c!1#Y z;u_o=4Q|2R0|a+>cXw@|k>J`u(=dGZGi%LSbx}W|>N)%D<6iG^a~xiYcE#S+bbT@a zMUyA|L)1a|mh(*H-$|NjC2xOo@fGhQ;j zVGb>>R}-m!nGI<6{jAfbUhYJA*6v%cNg-P#|FzVK(DPsTHjAEj84j+o?4NrhF5G=F zSQukU^xI2ty*{~$c2t}gFVqH&sTo!eJNTJIPgc>ndx5OfT$JF=x2D+a^3|!ng;TrC zKfD)t2V@bhU)W(#6PiljN_THS`Q(Duatc$eSwryFm|dCOrRyuR@lpKSt(Gc9$UdE+ zLb`#WJP4QvYDJeWT=K0Ihg6djZ*>$t_EVynH5qQD`PVa};v~Cn2`$wY?>SzEjq;{b zT1(3Ucu~Cx5?}=+mn*~VYJTyxvV2sym7as9wCIRX8-D)7{>D7gi-}5{ z?w5Wj$(H=#uxqkw*PL+1pLmwyeG+WNy|{Iw%Cgj8Y4zpPtpWNTR}Q-&^t0dw$`+*; zkkq&qiiHmS2ofg9txWOs;v&U2RnYNq_W1hZRFSf|Q}tpVQ$=`>(BS*QZ*7 zNe?kf2(doylQ`bwm%t;td5Fg9(_Jzf2A-;V8up#>Ulq5fIpO;DY792z zvf$XI`HAd;YMt391uP$tU#re>)KX0RrG(-SkH}C6uU8nbQ;MuZm9p5aCp_%}jH_{H@^Rt{x-w2kS?C3#E2K2hVJ#CBmxlesq(>*uY##5|xM^iIU*)&8ve_rRF*eR?#ssMngCTf|VL+=9Pu6Da<6P zE*w*%eDss;SYtM4!n5l-Tp#XT4pFkP-(FOjABF*C6eVZ_0m@S-VzPW%ykU*%SSx5PO5NLRB8L2%iF z86DK|ccpj&cAv-ctplTEP+|NM8e!pkTPeg7s@|E3r|1?i%*5c9E!bnt`GjMaeBc4@ z^)>cMFQG~~dgbzNK9>-gQn}`~sO3a`r_SGa{?)-)1HMSQHbw2{`cQ z3p}Ray%Sl5l>fv;Db2ci`c4PZ)h_Zlj>ZbmpL|aE5xpri3bqy!XJJbZ|D%l$zL@8d zDtf(U_Qu{AQ%BM+9L)$)r11cmx??kr|9VIY(G9P7TPbUv*ImE=WjD?D@5^7;rZ0T) zd=5m&u5Ip5(VDJdP^lxi|GD?h9I(dYky?HOH0xlSA zkjNs=oW!R-T%-U$gA;o}Y&ib#XC}rN7)h-?`h$*@ptxiVrQxO)IbKVWA$H)~6oUDn z&9yKCPnZpQ7exW5%XBAeU+WD=rGoZmF@|sQ6$cFq!VrpjlU%XHkYardA`b_Ck-`$*rgyx*ePn{`f{b|dkQPk-%pXCQTCS0`u0T>hGx(gU`@h7-&O<15jR2lTXJ*Ad0 zIg{f94~emFGMMl`aQ%aPy35DDXn1pkIKgBs8b2;#2W0WC+w#NuEwRLL?6xfW+_Hs()x!?LdvVtkr(N_Dh%fI?TDTkOiASl*u#Qks(A=RVn9 zkA2sAVJL*^4rrvy5hxdQ{oZAKrNg~C4MgiU5GZ??5FA_iCl?a)RclIeR?~pw0S$SS zxMQbW9mfY*vC{(tkx{QPuJx4B8z>R-?%u@|Ue>UWTX8ZuqHVC73wrYQiocW}S;3P2Q9zHvHJS>KD3+%9 z!iz+)sQN}wDxdl3Q4c3c&XMC#BHXPOf`C09?SNlFFlTzqRK{mxU2r8i`q4p@-*5M= z%_5Dtpgj@#&4|7IdhgvPiP<44;?n)Iga)$bv56J}S$n9^b3i*zQUY*S$M1SFJ>b2O zvT%8H8&88Iuf%GunWO4P;(w4)iBZx%ndV<&7$agZpelr$wCXLFJDjuMj4ks!-*pd39s|K#$6;s|M^VX(h$B zTFG6UnSGnMj3boaQUdfR1w0I4#Y*;3C)>4YL+%(q1!y;PqPSS|F~Xv1mC>ke?R z##rd01n^LjV39GxhNhLq!?zVJQzT2iRS+9wl$abXC;>vLLgEnb1yRx^7btkTk3Z6q zRdEA=J+%E{^6dG+;;iDK-=pbG>+#?`d;Pc}y@7NcZ{53v*3a=GxtV_@Pp`6OyE#q& z%mgBQ7jRTOl{!_N>3UQXoBDbBwew~AAEa`&R;K|}<$nNalEGuPASIh0wMA<3- zLPD9>LTg#IJK)9niu5YrivH@0^Q7?pH@`8+3Un@DI$&V?=annB)6(|A+qwV0t|Jjp zN^U3F^r_cdOK;0=W9?ePn!%%@K_lb>P<2%euGXy@t!6(fNv-RsGMQGJ)}3aYBEQ3Q z>yORo{cJ*VMWP!!IRf{~T7?*JwtGT(qI)lY&khZ9Iw61Iy+FG#x~au4=cN!(K%BCL*ogyZaOSCp7x6w!njMA|ej9t!`6xbk0}q_w84tJKZD7 z%-LDWyo6c#BY=v3jyLno$Bz0>fh2`jVer9LnVa*pD7TmGWnTN#){x=k*6a=EzVmQ? zj5slXDv+lJC*f_@$|OI)>T&BF+9+?W)d+0_jP%3@V0YsBiRi>{6L6ase{|-1@}FIf zCZ-0tv8Ko}6{nhJUU{H5_c4pTp%Q^q57?o!te%c6-fO(7efy=n@dzV zn*davOPKx(u&wqwAC~oRzYp%mqsG#`^hu*iOkG4d=WBkD_CrTV~tz?WPrAB`A zXDce=WDZJeIs3`j?u4;dG{JJW2iU7P&wN-p$3El`3^M}>))l|QBslG_765K&C)5l{`iXn*vdpGah?5vxgTA38N30>1CUahgvyyZh6-c}0 zHK?P50dkhG=vpn3P_vmqDVJEVwK_xgU5{Q$7iKt_VE;i0f5G*0V9XHtvQoF^h|2f% zb;f2`V*^euuV2(rQaj@2DNMJk?0Xq~nV~Z)P=~1iX^VPvamKtt#>BP|H2i;ABnQ1I zC+}wc;%xzGi5=BDto0odN#u)(loZzOQ z+0UG%VkIP~#B*pjdL5-?RMZl5x642Nga)6PAMD~XHxSJ$E){@OI)R>>qSSVh85WxB zGAl#1(r`Y5-8qPK`c(G%OkOWt({6{7R3Q!S+_Tinu}XA7U|JH&pyoh@DS9P_M8Xh$ zjozGuXM}6PMdrou^w$3-YGO$4comrD=HPRs|M@VKN=Rg^G7DSZtSDQClEKY?LQABw zEu|>xNHr~GziF!d>_MaWh#Bnz-x+vvxJj~#;4duck##3uDXHvg?ho5b(e&RR#V6J0 zka%{GSumMtYI+yZuwm!YaVt>QdoFErU~B4@i(M+OIQB@nkN|(*Hf=Cnmj6cReo`m_)(VzUR{gl-tFQ z1!=`52egV_;%%(%@rR!!@FkHK&{CM6liTua$l}^cM&@$UEDpEV+JH1Ug1NtwB&kUq z5{EOI{u(46Hx|*vYyo%8w7P|eXud?w?9i@~2S`!=7qFB_3H{8`CH*%=lIItWd+z7K zTsGaBO|c9JQjJo_pnA=8u7#Mw}^ zjv1=hYPSZBdDFQ60Z>CVB}=^6tR#NuvK(zc zK5V~*ZXtnv0ecA9Hy)NuWQ_jM)^sh{!g?YNM;S*>0;JoiB7BX{iC`hIT*puF0qt>G zUYo>3?&bKdbOm&zGri0%`;f6XW( z!0>Jqv__EtuWtp!U;Yios@ut&Ht%8M5UXcMN@1Fu_eUTYwW!TV%I*mKRz1Z+_K8dk zsVJlx1Gy&IY+H3=Pxo#;t!Ki(|nWlgohgxn>&fo_5tR6P>JPijd9-0=?OGn zHt-bIs{hHdi7Jz(#ym^NO_cGlA%(Ezd{to@*6-GvS-!sd*Ix@VYR^YXQjzdu--COE zJ2@ceYD;1=Zc9Hko5irf>cAmtBKi)=L|)6wu=4J3Xv)4Z z7_*O>cJsLlZ_50aaMn-*#_Rz4TPo$q9U+V;75>soShV&4-=3KNtze*y;5viQ?K({G{WXQhi3iCC)HT7} z7b++azsU^ZLwgl<5$GKI&RQzf?AnvU>Sl$l`hxl7gFQ(%H+D7YV`#QKCYhT|H32#><(+O0bBR!Oo zY>{ktFPL8lZ&TCDUhivXJ-+MwP7QlizD4pIE{Uz!b=q|~&hFB%Mw`iGgFVGn66Q;V z6&=z3y18b8sa?ObK*IRJ=yyzxB-5tCLHLW&1Y4C>JAWf5ESIVtWG~6UCQE9uE1-+* zeR;&0L!gM|9Go-0;O~OpgiJYM&%LnZ*mOs@xK3#P(YV1x>aJ|XGDNiAOsZujCVxx& z_lcyb-_)wSs!b*X(*lGcDvy%;ko>V0OMr1_$K%mPvOTDIb5--hf!b@? zGt-stBY0gdBj_%*RVPoMxpu4hx8bSL zGd)!CW%{1`6loEwB%!4FUXP>{a4mLge3y5_;YWW(aaC^Zv%T|Ss=xdWb_5H88~jGP zCq1@oTc@Gh&m+&ZcMW%8i#U*gKbH;yqdA<`JK6I&X2%m#(%j$~vb^U})tOgQfE{?| zRSfzE+5|0tg6`oS&4kUoHm|F7<0{DSjPDLptQpD}rfMn(?}!e-C#&<|Vov$|NMqiK z`PEp)IJc4Qc}VP_+`tL{+d}Vr^@+_r9CUX){ekg5pW*Cfc^v7UpHb~3O{`Fy+s6m@ z6ibC7`}GIJP+7%O?d{7=&ExP;C!Q`t-YP{N)3=O|}=k z2eq-ScH(S+hS!l>i-TDX@iu?E=NU_99|uA4K=&<@e2U#hvd{0#4?h3E^{tvFaoyq{ zYmhM&@>-xJZVRlq?bHU%?Fz_9gGDcAKr=q6pI3#)`NRUP@+xLv;B{Q+b7>joC7DQz z`*Hu0=H|lexZ3YpdQQgfP}1?@@$kcw{N2lpjSkGF2LpEb*PJ1Txj>}`|GuH6er%tVf>p&s<-!( z)c(|UK$LI<_uviCf#xgOw^y;Hv?trkv%+`GZl3$$twHIl>0A;a!6vRHVRy^Rn|1)= z>=cyC+u?Gte`xD0bQZ&NN|fhG{RvGgig7dkO5%-n*|Rd%zIm%TCcp?cH!OPQtkD-SkuC`|ibv|9k8A z^nFccc7^5y(c3?|8y0uV3YLLfXy6M&!s@!B+tbQ_QeiikN@hq=trffd=|c->YC#Cp zPtSfhVhA(w3pAyrTIrb0=K~+4i-eV?{LVa_4Bt#qy(VB|) zutYOFm71H^@9hK=fs~a51@Hn-L{54EEKY6yS!hi4-XZowf^<0pxTqh$0W|YYaj6K5 zh0NV7-ztZg_`4^U4EZAc5*nwHZ<2n@kiiE5qlBZNQsC^qeWoxt;Tk# zmy@^MuVj~ch<;UKCj}m7Z&gL~{V0*OKbU8q(-~YbuSnE-I$%2z4RE6P4-eJLKE8Gkq*-DV`Y@-hP zk$mt#wDSdjXXwNf-&Y>Cm3lN$Kz>a4*OX1-?#_iZ{!~PxKV1j$Key~xrSx}FwzME} z7WM`HKD*)6mxL~11t+OLB{wKLv}BUhIlKDq3cEvFh*AVcy+m&WpB>kUDzUpdky=nc zXDW}Md3L&I-D$AqbL@4>`zA5F4Lo{c=8ik+@Y%I03W#vXS`K{dVI6#lg@+)AD5D{4 zjSuo_C#_=K_F)hJ9;#vri|pYI+qYRB;0~$yzh4hJwu=yx`iR1&D8>_rrESL!APBw7 z4V8hk3R_&QX?hr7Ih06VX-Z!mJ2XXatz9ksUstD1)FmQYm;Q0wS0}2I^y3 zEJMAry#dp72;{0DbB!oc0e`KPMU$g1l=RUDB!+ricMY+_G>3yb)vXf)=CVAD|0oPY zo3^gK=H`1ARSIckED59fn5c01+6~Pxe+|TZs)pYt*q`*m9%>y)_DJg+o;`Rd640uV zNW$G=DB7i)NErc0Y7`x+P>-YLto+ft)JgIe#^vSMt;b9g+$-Y2J+DJ;uprMk*p3_` zQWsXt>qE5HtAS(B??oc%B8t>_?>0MmB`A$=7dZ}?`tf{c+T8C1#)|AlL*q)e-U?wE zxZV~@2{2$1Pjxf2IJIb_JC-Za!4?{8gDUKKhrAREN6Uy7#9kkhcHm*iVhajN7GpV$(BkPxvdYFZgGXTz+73 zFQwrd=!>3qP<7<}25mL$yf~&(bD6+rwK+@7Ld0I!{&uQ3-r?~`NqLYIJ~3wouTQF! zoQgN|>8G+VW2!R@8~?60b|-)Y!SPp<^$qx!$fZ$q%WN_y+7iii*`n$)lcHLsmq1Z+&4hYe40E<%I_FQl0d;6cS2a)FG-`h8j%^7) zu9Za|%w*3b_x}sJ9xY<$Zk#hBuTEu_hsQD_<+ zS2sBzRw{3R4UtUp5&DMQT;7gGltR!gcc72l;#+m3qRv5Oo33-L?C2axBj!B%5S2yw z*&K^>AlknssEm|p<9pB1YDpwU(BN}$Sqr_qF}QL}iuSdeE7yGCugcHF(H}cRFqjL(8EqT~GKA7@ zw@}UQH8XAa8$8(b8#OtO;Hp~wTh5nbcvE)CVf75tmLc7H!=UTbS_XU{z;E*2 zs|>^{lVp3f1X75`3D2jIcF`^UOBRmS7MDW2DBmyrV}fdO|3E7i=8z*EA@JpEP>mf!seyWJuxFRN{mGzweVR)@Qf;6 z9-kCNRp>P|J_up25hS7e;B^;^L02n~FGqlu!InDrSu&0nk+_?%tK?FP9ti(eH|M*k zc0A>obx3@gxK+*v|2XAb^9}6M(JM1xWfhc8)s$c>OhpA{M_t8!Z6rT|Z`8n2C%iZk zrJ5^$x~1fpa-(>d)%Ao%Frcy9bxP>kERk~6LccfSlCV;Lm5tfJ6AOirP7_+{(Co$? zL5=4=v>}zG;81N7Q=9!7(&!|humxmw=C}qP3h@utzY<#o%<$6TZYUBn`VvYBe>U~; ztXm|{xEN3r6vZ;v|0PZv_Ju&2fn4hL8Fk9sBu?P@7Ww=R4K%viyH485Y}wREn35TWFwYx07ZFC$nGtSYN0O*vU!}xJ7DO|Ow)q1 zcy;KpkGU^rL2gc4Q;C#$@tXJYlW+v;8oba4=kDw~BJ;uWk&cNI;a>HYfNM?#F@T{5 z)HxGQfqVoxvwev0swyo0wKB6NPw{VZN!d)zaxDuJf`l$3hQ#0fqKgFCwTZso`e`6D z9iOBz!DNCDD+* zN=Tav^e80v11seP&dfT|*Rltx?lxt`M{7##Uzku<@q}p2vGO1JPFPQNH1NfgX#xGK zX5YpucE@$Sc|?uvGt-xLEPk7I;l$nG&7s=OYB~um6g0Q{Gc3_eaA~@WzWZ0O^^9$C zT3z8%wC389a= zqJ6w`W0});BS+2ZO|m3%U)i(QPR4fn7Giu$RMnyB)?$?q23by!knJf=Nwbh+q$e01 zpk(Qy&B>aF38zKVJ6!vr%364i`h|RxS&F`gYx8mbu8hWq`i*XMyy}>;x1^t`=3N)JtCh{VWy!jn_pM{kvN(toXh% zwxS?%J2ILm>Mweg7LM9AQ|GgkJh8oGI#Az)wV3I5WPK8iW$7|DAMm>JOKH_Wct-TO ztD47#eP1*)!Ql47A9MQon&>pSE0#7!E}(zU8O(Ag(-d)sT?>`(DAQ;+zI2NOfmvJ@ z0pthrFsxfO)e45rN|UCEX`omFmU`@&fK)7f0l}=$xC}Yj+apS=WvnBHbo?PT+23on7w;&_eBD9MO=uPg zcB4Q7&sw+@aFXoGMJ^xNxIsO?N^;QIRZEw{cGX2RwDU0#ht%STF*=e^z7Rx{tFCM^ zxxIExRCm4|6c&J6_7Pj>o{wg&3B~hYwOXefuqAYc5C{e1Px{oqU zRZ}n{CF!FWi`RS9xdkt0Uj7PFd!mG)oIyH}brv4ao@#|87RmX;5 zx*B(t+1#pLNj08GMHOz30ZF;w($8%&lP94om1KQ<$`Rz$VfH;uAvJ1D1q)hH;ucpZ z)i-&K8;>A6{ewp(sGE!UeFzjR4??gs!dih*MW|3q=6ci{zutecUxl_(~ zoR&SgPyg*tM-eenWUpm;v$=tXlA^++v@$QFgulMW`eoWM{QjuGZ!1r! zH>0Cx2f8{}*w&nB&;nk?j(_LBHGABHFCE++h;Bw1J56?XO0SGJaCkn31@$9xxfjRDt# zqrt#y*1-gH@uS?&Vmw`-;a~6MLjUNybUr-OorSI2;9q%e`J_U}d88c{Cr+zBlcmWo zYiEvCH|Ne~)It*k%5tPtm!3_pOZ(kI{cux7e(vXmM+-%$(!mM&_vQ}w*0@b$h{e!`(Up6R9(+M3Ay-v&3ygC`uo-L23k`YQ_Ima_n^L790T_JVH9fg zJS;Ahz&1i?Km;&+vH^m^BZw}Io*EYQyc_Q_XWqhdXEuf3bKV1u&&bYS%MOyCDTR&m z0j@n_fa5N1aiMqTeaMhCIVI!Hxi|Zb`+on>9XXbV$lT91+I$6fvTr50j^xlNz)@fZ z#N{%zU3^2_8PExReni?-YtFY>-Jr}Qb=+NiQ{K#M7e3BuuNUM2E59zbv~K1(`?Xy( zKIv?3-rzaTj>fg@w1_{7xJS6V89Z@M?$_Nt=tr5$@$G%nb+m~O8Zf!3r|SOgUC5&u6i0Xu@EZ^sS1mZG zuf{TAaF*_2*sN7Qr8%vPs{5;mBM$BA$>jfU1M`sK&Z}S&p*2GYTRo#LB3v_)ZwqHG zfLu^7+rLYpxOw3g5!MhF7+7+c^J337goD}~vi=Q*E`Rz-`|r(!Mk6eV#-;C4+G6hl zMvv^h#+<5Y1(SS95;Ie9?{kHeS|aTniJ8Je+`q8d?_l14i?C=#6l>Yf-j|<>()sE@ ziRMG~WQTjWjDe#go6 z)ndID6_~{-L){2NdW~dCrAkE@;@{U@K_WwZn=B6ae;+(5(yXbMtP6cPbZdz0{Jpq7 z;dkY(BkFYFpZq8wb+{$d6Cdm_sL%o4l<(1nY?>yKMOwj3R20-$dropufHa*Lqr*Cj zxe2=ZH<%CqzW(_k1b`DN`uyA6aaQ}5i29RZ;e@n-WulFrE5mU_4UH#mEe=^e<{m_K zvCWj^-OqfVAasZ3_*cW+9@+BcBGi#jq1?tr34cz4&nl_FSrc{B6fOvNGQ0Z ze4boPlnaP{h&X)8@Hr6VqKeWpUUDPRCUVQz`rj7i-cY=522zd`1zDjKfS9#|`DeU_LxPeq$Lt*%jIB`>s3 zzwb!I0>ku+tz*-8{>4gRxE4(5yBJH@;*LuEkya3RF81NFwpn<>Nr|U)$P73BcU$4` z+J+#Lpaw)6V7tJo0LeClhBlfBnsbyQDi7bdH|6-X3&2Szu+K*JLNCP&3^$Ovv1%kt?RTXkP%IA3aO!SV(gwgakvmeMqtW zN@$naFz9q@wupd5@H0|i>Lpq@8Q1x1m#st*LiV19&bWUut^`l4N!dM_8!^g^lq`ug zVg|9xkcQLJfFT6~40zNQfrSwDcQ?D->z8b~{T%kjigCvuV6CPxl_mH4=04?bOb?Mu1*MGo-BvYX)+56l)R~KJQZU2FE*?}k_(>9|! zg*KQzy}T{2dzK6HGGoRhn_g7!J@P?kTjp>-*FULQ$G32}y=is-iN{UC@T&&iyEhKu zBqAYXI;F`m%|7D%)@6~)G{$TQ`eAorA%H^q&u6(^kT~?YjXKy%E!U*_2t{-ni|$f0#d9&WHA!@4K{})&&Fh-XoQOP^K3L8z z)W41)2JBp_?LTt3T`}#tsAZO}bY0b@N!K@w9QlFdrmfo}-J|%xfJlx`|3BmHk!DK# z&YWk6Kx_;WJn6rsl(X&1U6CKf@vkSw$bmH8Zj{(=OKS+_d$hB*?pkx>V!OCA!JDhB z-~_ds)Ea4~wF^aefcRF~eXJTI1c;1#!6ArIYXKL_=&a$rIZJz9MFXT@h0n4t7K1?) z8u5|-Q@{Phj(YWUZ#w>K;9r2#g4UJTAfS&j9R z5>yBxb=uZx>syPPe1xJ~cCM<<(0&Xyamyo-!T|}?lm>Z&#u>Zh7bRT)j=*DDbn7Wt zL(j%r(tHi8lq?)X!KuJLeW_mKKlVbaHJ=9A2Awr*V@~Zx)P`-^WwL_%b})5RV!u8U zNGmxNfzSeFjPNm<{A)nXn$g#C2tWl6aKX=P^PK?>@N^mb;4Rim*9P@{0^l3E6bVlu zZ|SdoK}W?`I~Td^J&V;DCdvQOVVa48Z=M39j;zrcWwMUN;+!bVEv8t~4)2XEt|~=8S;d%bwA5$#5Bo z9b5563{uaAaoG6oU+XHy6hxbppJK1FY}q5@;W9_Ot2>dy@5UR37$q{blSIVbP8cHF zVkMm3bf}@SW<+*g3e08d;6wofui(mfDqn1>W`vJ&#y7(C`wX9CO>H#|ALi|EEBXwHg!(98` zk)-!^+0pm2vpcdQ75+Pv`DBY%3=*bA*?;RLUc7e$@A#Jv1hzzXf96Yv{q2Oo7|B~t zuksco=vyIeqZltbiD2+bT`HSGTc9s8gSn$@J~ARK;6GR&r+kP68BgG z$IasU*C-olwy`kMBT3h@&xXE;;%Pfp46PrrE8sGH!-r3ImxTb;QorU}wwxe<6J-+a z=e8PaZ<#a82R2ZhzZ1QnJij-pQLp1N8MVf1EoV3X+5bTMLn7=13cEbPg>1|QQz@tG z6?3;=82;oQy(d;95%g71_(LZyiRRSnd3I~BgkQ=c$I2f3lcIF#WOJqtbt`O0OG%D> z_2UL6`xDTB>MVvtZw?|}`+`@)t)cZ~aGJY7PKzQ%KQyvFdxd2Eq#Kgee-A%Gph^)U ztZc?2cSw?`h`i{=ZYbO^G&E3Jf6C1m&)%-}pKf&j8iyiO#wqvJ(m1dzXC9uD=tpmv zJH!1;Nfs{ZdWHbdq|V~(;hJ;ohS=mCwAsJyxHj4jG)a1Ni{o#L7&===(&xi$)e-*!J_@O&u4bSkotdnuf4`d=hhm#Dj#ZEXCz5lNEJ;&DZ}Eh6 zh#?YWutg-$ruJ!;$lSjYKlPSgF*%><5%A)pRy|RW?=D}g&@M26hIMm{!em4x(Q#q^M!M5zOd;h327Bsj zhe)f`iK*SWo(8Z`Sc=!pIOeQK+U#R~h9A->?TN+n_u@Ar7(?3ngqm>gX_|x(vc{Rv zKnWk>7VDy>tfmxvtXAGz6g{Z{LX!!?`YWpHMTY}Z7&s!I>TeEj9evFd#4be&738eV z^$|O&ROlEK=&Nu~3Vo0j$ak`KZx{;KSF)7m#I<(LJI^qq&(u*ebCrr*jI^F6tZ^vx z{p+6elKRP{_h{9j(s;M6usb|lp4ogG(- z+^unW+=DA7Xl|P(3DBf3$p(p|s;;Vx%VwVewPYgBcHt zyfaSwU@x9^c5pBv{Z~cCE{aK86~FnDwMmoW1r!-_CKJsr+O*<~W9wMxz;r8_L9z5{ zuEZ`({UOpD@J_sqnMju$h?x7_UeV@^QWGEUK7wN>o~th{Xa%qEInSF)>p&DVW| zwB(rDtJ9}UK=HHgybrL9_&in#9)t0cfP`&IK0Kz`$P7%E`NFBR_a`V#wM%t7{z6;A zTP-6T7N%;(l+3v~+v?@MAmbAr5Ho=y=E!ZXIwBbfj}p^fFDf>3^s|RMXt02o)374R znCDLt?;4LZ9%hW5kd}_D=yt5e1cPgdYSb9#kJ9EnuHJw2l^?7+%2SUXZ>pc@m_)*l zytvCu53bs(TSL{qtX3qvD~wF9l1u7UFxI^L5u)Gev|qsq!{h_gLAP*cpIC06#>T+! z8Iom*P10n9r@#M>RqJ3e>3rf%mqI)K}$yBiDlS z@$b&VP9+{2XG|D~>zP{}K#6kTln7}?1+QyDBn43UXBNQEmdeE@u=X~q#b$pZe{K}W z91t0e9+FH}7GERsb!+X=llaw%7kEAJ_IO!&?Y-d%_O(NlieB)U-E};%YTfA4 za+WnmuR}Xz`U^K7Cn;A7l#|Ed^%1OD>P|d3=sjJJsEk89`heV^jEiMg0@5YWVUin(v;7H@8 zBc0m9eB_*QLvip_nNO!FmmiNxC@gwl1MG}@)Q)K7MPT<`jVgvM@uTd~yR-4lZDf*e zzw!PNgCdYF3N~}Ne>c{iprXAfr%z68muLSG+e&fLhj1f<-twt_ZS$$hqSQfnCB2BVSGJ2!$97ak$Xu zv(Dj{A_e`I9iKHz!j&CQRT0k1)f5guzRMeKwX!e8mql}#5DL8=Te3NaaebGTqEBi< z`qtXD^f4-G-r@I>P?50Oo8{~Lue3tG%-9g%4qaLruQ=2mU0Qj$ny)QlT-i++Sv83l z!w;Y$`5}%UCl8o0o|Y@R`p;y{IJ-;e`QKsb`9U&h9H~LR(MyFS`NPOPF3#AKdomNq zbu%yoiqgVkWUQ$+GHPVl(Id_EwsW+;sJ-c2?IC|ReOfdX`u&&k6v+lW)0Qr_b~2=IKBCH#NDAMmOGGp6pG; z6u*~Hl0iA38on0yLno7`@{2d~TBpgj?Au?2a(0s691pXT6669Oj46ElOT=zV!6sJ- zS7ukSURPsA#-4Ns-~8Bjc=`EmGf6-DEJA-iH3C;7^2utjb_LHdurS|6@53O2>f2C< zleiBD_@x^sy3ZxKG^=#6w9QF+{}=GTJ>k%IA#q_70W~ZgB%tFin>4ExOukE?cd0rhA8XC-qKx1ck~&^7-(fNpEgPMn}^$CS3+xJ^wj|=J7#6dGdaNi(;!gixZ-{P4Z2UCfck&+v~}vQ4i?$08T*f z*2}Pt3bvnA<22C^Eb_B>K5?~sHUz_McTWP!RNdbbU+6ws+NVpUDJuac z9|HQ44z2%jZ9Hqx%LO#$cZ#Q@K2Rwb%TUnew6CJn53`D_EkMzL#IPgnmFL;Y^}KIU z(S7f@#%&y{)A-D{>n!h3>1)>3sh#3m*VF0#S&Wy5piu(nqt^#^D1FSB%jqRH|Kp;o zbMdT-bEo@+J>El=?XB~AaR7p*OQ?R#NuKW^(r|4x+DHxx7bc1gM-m<(oEuQP%bz|L z-pK#@8pjXJm`cvgx{P@GOqI93!%x6tDEnNB0GV9pbv46QPz5H%DLL4IZzx5ysuomY8B%PuzNYVSFXnMGXTG>gnTe=ZnnTOja7aCSA_Ty-?yk0qf!c{ z#+CP4jE1?aH0_+U4zLzUo&}K9!=6Yk>G21LCI6w4FTj4hu5FERe~R)yI9fy{!o~1Y zs81vUB*At6Y&Ns?x#^ZI)D2h(?heBy2y4Q1B;v13buHAH!idg^X3o}wZth_3Y6Oc} zrYoyJ&*TT9!X*YRV8oTbqybw~CG(7Mzi%cW`y^|#ieqs_A67GK?!mih9MR-YkZ|0~ z)hbxN!|a+!4WKwDhMvC7!M=~{1lzwF{uhXL`~^z$&3EcJW2rGPZyB+vr_)hhsW}qOUoa92-mEYYw#BerbIjEUx9@s63lq^=cxW5g=1@&kT@ES;9md~ zPaH-WRcoFB8P^hn{E#0Cx7@`7JF4+9U020R$XWTj8iqnPi4Gv71#0aC36k zw+zU-TxNot{y&U-65>^tw*wwRsvxjv@%akErcfbIM`dWo6!q!O0(C<^lT>ky# zD#m_h;ZELXKmP)Xi9$x`@hs9EJK8|8tz2$Z;B#wHnwg?R!OIumc(W)%VX6eCl3M<3 zM}XSTwbJ{b&|D3H+8+OK=6C@5p6$q-a0Nj_)ZIk6g%Ag zn_oTPv1Ip7Fo$JJa7v@euZc7|5hE5Yp?a4=!ivO6c?(VFLEYM| zyg@ZTfptdHrA=|Mf+)2@TMul>Z(zG##}hB2lng#T|9T;+B<+YDD8e z=n0(Fe1HT4x~-vgxy7--fkc=0K|@s7P%n%_k%qsY|3V>t{c2 z)iytz`Ilj9{J2GoV*@>#iSR>G{P!9nrdI{;~ zg$)o~l32rCF|wBg%4oo1K$u8dNeN4XY7fT%F@i5WS?tO7bzfKHz>>0;%xTRJuuHTw zfxm39<~S1j1XhYNRh~`gp%HdgsJl+EH$cXv#KD9^N2%cX%h9GS2{2k4MG?`G<&-iJ z8B8?t=X4?e0wiPWN*Swb4|{Q$-2ln2u{)va_zR&x35INcPK~O_k~EEo=!xWe!2QY>fOL!fZ`D&|pVghh-i`lY&|5lOH4Jw{ zPQ-?ryIm@{EuL2#BAYjKC_N4Ej}NZOr>5L0I|S)Fc}Wx!`Ym@ygP(2ROeXlS7nfkf z7z<5-J>f*#d)sPjc5I3-o|brFeK_>p$Wh{{j53nnbA+46KBur5a0maLzG@!C3UujW z4AF@b)E^)J99K}wf=v4C>9=d-TV*skt$kft1qr*yr?A%)ZeK|qO6JK>F8zgy=76bC+F3}N z@n*-3f!zmycO%Hlq(2s(J;!g4`iagg#qY|&JN8f}3&@MYUD^dN!IVhupdc(g8&4iV z9~t4TPW|(s<2rol7N4QXOtl2k%aEGdSfARf0wgS#Q<3plqgY;<@`4DtZ9H1|-G)GD($^PSme;@83%jD`)6Dm|<UvG6zmUPMh_Tjbp6xotBMBJX zaXVl}a5xr=FUd*4&6{?J!C|FTg(&z*o3m(@CPQ#^g4wAdN_*d~%xX2ejj;QnN=TXn z9JqloN;0;B85SV}*G*HS4rNZz_^!ZE&%nMClnQ6;Aqh0#n=s1CJc!T|GG5L@GgQ)YU#EV&>y^3*~jPY z%E7={pE;&_mwM`>i_1y)+_h^Y16%LrBPmNtT!Q6 zt_+SOy%8k%^bS%z!*q1+8kwGBw7(Hf*b-Yo3h3_ehc+tt`wX56CqV{p__+b9sk+Su z?o-N-Nme!A*_#KbwSmNeK-sXh=U*6A=0`LWe)XKu4l?IDf-`R@TfE1!*iWAkvcRKo zJu63CwtlQ3;(8^l@E!*bYslO%hj=1Q#B8QnL=q`*&ztE~r9D^LHj|yHP{#gk+P0$6 zjsfFxy^Qi;)@J2eXq|gip(B4d*65)%tj%b2H-Cg9m*SVjr(FQOG*9dueZ6Xx87O+K zR%Pc-`+_zNz&98qlSa|v97+|dkSJ&g?~=$(*M*5|{Z?q9cBDnatdX{|=3TqihYy|c zA_gj@&nW+K9A}pj6`_(3u(8BAMG`ukH21Khz8tV5x#xw)oB4`(WSU0zn16;m0;AI3 zs0S+Q>;{>%QbmNvl$SONbbF?4B$FW&eG>#OjsYRc_QEJz=Zf`OokeiwA*kCuboi8Af6L*-T_#CWd=c% zTB3SoWy4P8JE7)^MdIyh>e^Yeu9#?ftpDrGzhzj_>0Gsi`KIniQ z6NV~txSvU~;70b+;CICA_Bbk4@WNxL;vBS)>6W%WPtl{1EDIpo@LKmS^onkvYJdl% zQKanW)Y~4&-jiO2S%@uGkDn$7ZXl$7*hk3grRf^TTXgTD#A?G5f{>*2l##@pJEH{3 zA!K5k0rc@lvD}?~*hcKPT_HXMQ=T|``6;?at$6!!sq2b0R+n2V%6;VSsI3}k*V_JW zdiY;OZVvS2AFF3H-B$Bka)bNMHrZw*`#;!=kor8B9yxKOVx0&reFdomT*E;ON&3;h z*iHzV;Mlw0HF5Kvl~EzH*#y)<>+f;FGh&Ag5%~ccQ+j=CGZyyIpu7#kM=;&|FYTb3 z#KZEBH!Bf6q~cu1>=&>8nD5{m7;MU@CEyhTn7hf_{zTZx(7iz)ZX$HM1=BtOXri_& zV@Dk`Vgl)q#kv(r)B-rOmaMHB0z`iMz7@xjCO=@wJ3inqE&bcT$=xg$kiL0C*EbpM zW=NFW;!<^gZ=D%f*;2B+s|PqGIDwI_M=fMJ(B8a)!;~a$tMoZ;XMIIpTF{WLFuh*4 zI(h-hZJVesRy5flUv@oc;{et^azD$;m9`kM8r|(1B>7uD92g zy!t}=wuS|Cg&y_2%k)N3*U@SvJxP6we>1v67T_E)4d;ZqWr9DxdAc{?q-Di=Cvr!0 zXy;ErTzS+vDMkRVFVzItA!!uBolZ2kf;yoGhek;9j2*qH=iWn|!&xu8-$b!k>WB&! zp8h@nV~Blj87`18K-K}a!soc9_%$v%w)7>lHc)%>3SQ9nbXxfUFLd`U;q2=@Dw{z= z3n+Ijo}C1>cLQ7AboANj5bX6LzSa3)Zmm4|6$;pQp`8+}gy{jD4WvNrW^e;>VhL?g zs8ar@aBL<6wu)UYU=c9gd?8qy9cYrbn!dWIn=~`h@+OrT+{~J#+z%V)W>s}Hd_5hFo->xNza^M0zmeFg6gSme4G;UHSX7%5 zzoWXxwEyTdCDLab3z{A5K2sm_UK?J^+$URSoZG>zqaVXxn?9pH+ufJB4mb9=ZtGX} zKlVLNd@kFLxlZTPKGTe&j$r@M5-;N}SFXw~p`X8=5xGs^H-6OhG;TC!w9=}(TqftM zeJ|u6v>tdLKpz^~O>$XiomM|bU&metUhD2E@1hmiuh|+#0vXLeYy8UmihB$BD{oHS zE--yG<4*a_^UUik<*m|w*LBfo3qY@MYK1CQ3mS zJ83=Io@QqiB{?PBsY=obC3%VX4)TVih_au|FH@Ud zPHM-w=~XjZ>~8LKoa{U=^1qiuez&q8H(GVqxu4-qvyJ*=^qjQ@Z>pE&LaR}dYm6A+) zYxGLiJ;{5JM9U#Z!c^IOIK=bgVIi9`^;+$5P*Ry&9&mNMLo+1voOIPSSVOwlv0fsS zohF?Zlg$UksFyEaJCDMvUUX*B_ z!DRI}m5R!4j7st(0^%in2tu=4!77nAs4OU!;EfY|c*&Pl`cc4QT;wKn5JK~iJ%li2 zBX_fH8^*p5QwJWw3iwYpAaR;MVQIiR#$> zsS4q_6P_aBk!v+A3G09Xh)JlH#09v2tg_I3>{iXg0T+*4LC|!pEYF6}?$XzR>aAl! z)DNfE)s2V3wtuqVwe|fi{Kcbp=MMfCPY^~EL zl~2+eDd!gbSm`g`>@z{;WMfhIn`GlzmnAdlD)?drwrxT@#mg2N*$oWZ^O8XxEiVR!yHarv{gnP=*jC&KElKi4R(I?WyDDq{6zss}i z{Z>o=9x{S`4s}ZQn2b@V>q`VQ6b=uP0q{=r?-8QRpJ)0CMGF1Pq%dTmK3Cw`)|q-OGB}55@UA-a zDlUv9FHBC%?48lsk4>8jwGGG4sO{(1vXcgqf<@IOcJKrB4Nw=i0rb5-Lz#apiumLg z(tR}bS*1ghRwph#h(cCN`ok$?0Mx;+ok^emW>xBIQ_pZEP>>4~Xfz=H>sNpxWU>=0 z@MK6J?W#Oa!{(5vz14I1XcFkNmmiZ5$bvO`8=J*Y3L~iHePq&JTrx|@Z)c)H=E+Zm z<_?w+kPrR4HHT34`rekvoCb9?+303ZtaR0`JUON?gNSw7EF@DXArQoxF`9J0feL{e zu(1J*g~rm*%0K+Yfb+pwBVMcs`;Ac3%5gp^3opig9aqmo-sb@&bx2$GIo3>`zVMOU z(s)*z%`waOQ1%%Aj?Iyf&`QJ2CRn4i61F@T|k z`mrt!ku*KKWu2tZ8u?dxpDXv7s7h3hSBX&cZ7xWfh} zgd!c);H#H%(b7|hEZhC>4Mb)LN{EqJt3r($!glo^GSQv^rweV+ZDg`_A^NGQ@rM&! zWWo;()Ux<`AP5gB=PVRaq90IaGRxoxoI7Zh-9B?%v=5g2K}66vTW%5sl71qQEfhc4 zeNc(6qi&$3ANV51lr!kj6x|{fB6Tv4x(HfRt-sh94v8%pcRBG!)t|Frh|JDbSi9YKK1<5?T})xs$(MH7?b$8SRt>%DM(%}*?Z-=GOzr!A1` z!Mz%!&j8_+2)UyQ#M>o>P(2%wLun2}To8uPOfhVD0ovQC;sA}My>oz5cg_C5`dUqY zD328XPUl&3jirBpHHGnPG@+}ReG~Oa?N5}5I;|vwr zW%FLJgp`gg$Tg}fXAL4d;?cWDxo_zVyiYKx1~~Ji-|i;Gp!=ZTl+LSXX$_loSh??4 z7YVy(!x`ZoL-56OjN=7!1*|M3?RSkWRpn1+k4VYdU$%%_a!&*hOyEVheVQ)?XhPJu z9VWougUra>pBjOaa7FPENI2(>ki3WU40=;ZSo=CM|31lx?(NqVg1>L=3S}~g*Be7_ z{$D>Fv-PARO7JYUkV%X86)gg^6LoH2D82}2-*^cl#9b?*SZnwLkb_DG?+h;Y3I}@N zP(dJ_?_(}TK8D>UUz)!-^3k={h8fctlbe}ZHxDt?9^xn@<}*D?j62Q+|t^ z)t6NbZqwGamFlWxt!3tA(hI(ec+^JCn!Kr*N%pDHsYH*mij1ve{E^lXuMyM(#Di#u zvIJJK7{AB7$C=00C-leB$Bu7*-%i`JB2SUtQlFA9H7`*wDK7=?lC4_I_2A{wua?iH zd+-T*T0LF9W^Y5z=FQ$uv-7OQkBT&`KpR#WtaO$Ve=|KYsahG<4(v;;f=VT zx;y3j-hdT!oY=!$(p(vO(`kE1|EUkOpNuc_TmNBIQk-mV1`nGB%_AKNxnJIoVUOE= zYXm*ncBY@Bui5vthh2}`ck9pwavo`JmOJrpRB2#HZa){&YJWO7!Cdx}<7I5~bcou? ze1G(A^n`9QC$rP&I_mCn6Z~MToqZ$^+t1nCnU3kj>~qxJ=kxon>dx(8bVM(LAK8!N zI{IeF=-_2kex!5c2lKM+%~T=|BZw1@wqJt zf*;N0dY$c`@9%`i<7%_FCuj4%{azhgm7+o0aJp@s+g^1m%8MynUGql}r$||l#z$6A#zsfb*+x{b>5;Hv#X0#omVl9rU$_|=YJbgXoZbeU zMrR(%vZR0DuKwO?;zA99xrVxzg?Bedlr@jHhCuASIMCeUTMnwngAd*gU!}!6jOy3s zcKykNv7bY&Dt{1ll1-@SxMW!nJ$-WCES2Ha^;Ccjgba8#jK7wp_>b7V!(IzWj$G|h z-<+UZf8uP&*AWKN?InAJhE$%QUo}%18eLy?1mMP8fE0{Hkwl}-Rg%-vW$ZjgxzD{U zr&YWXnR!e{Y+w3-R|(D#V_~{?J+5LH$=1AXBY&su2|uzD#JYwr?u2CcbRyAUu9 zq{WzlVZe&WhX=eBB*@Ft=R98&$m7od?D~9m#_kJ%faukIgqII`Go9rErjZiKh zNl&%lNC^LJYw@l2Z5uv9o0)y5{DU6mx#0Xpjt*bWUt@5jt0rJ7qjBK-bVaFlY|8J8 z=z(o7v|`MD%A|KZ*H2M>siSJpDf;k^*qm5G(WHiDR1r5X6cOe&c!q^|eGl+_5=)5| zvKmp0xvK-H!gfeh#^NVj7Q7fOaNt4%8!)o!N~hu+I8cX}B8g>BYizE5)%&&OUO@gxqilD#uiln{gJxWEOZgTQmFWF(Qh zlEHm+qALvpeCyF!7i9%35%oTIm&gOe@R>q#yEy+b(i}s)(Fe&*)I^ui_dCKa#JRhP z4CoCy?-pn&3iC?F=bnww0$M<9q)7qeMoNk7HA_pHk(4HWV8ndnJ_S3A8y-PTHXy5` zKj3KB4v3(r`pa>q-5)o}OhGHU72hn7>fK`CCc3EgTtah6^k#=)_L5`8!BYVEzOWS^ z;Nb`-vKJJ$e}o>x-5d#OEWo1qo?6A1&2JOU67ALbqAZ`K(=t5jD5^tdph;>&NmIpU zKs`PY?MNzY?nL?rVu z-q~2Gbv^UCv%CoiBdiWcf*r}#yc1WHATFxqR@iJQ1nz$ZL<>(WjCsof-kEXNI>i_3 z)j0l}h+rnTWtTiu4&nU-X)f~C%NU&`*ozqAEXa2Ff%)RXl0RQ2V!#6z_7Nmi#3B?J zS5&w4RG$Z#AkM+s>pxJUxYm%DwtS@wlmVED{iy<-ra=%8%v0jWSmdB5ls#wr~_&vPsM%6(L?@xN{5hR98!0J zO^Q$%`8~nP{Ws<_p{+gvO%!Sca{~30KjZdRoVdeS%%V3<7umdNlJra51rjAj8MfI! z32*ZKypy_n3Baa(aoIFkV2mtg&UTe5P?l5HngQ-|s3}TOh!oj(35W1#b%}> z0m9Tnlcv1fE`6lR<(OUb8g>mbKJK2WPsCu4YYCJsqTGhcL?=TG4-qZV?Po#cS$$@P z$|VUT>%0G02y#V0(oNn;A%j2x9i+`RU&!n;K2^2^PwI0!NBPKY;Pei-IC+!?^4f_` zR{WPh;mYv%`3XD&UCLuGhA*=3BfvfLSGwcLngA0h`+G{7x6i`a=iI2~II~SY?eP%P z0!2OcrygDY&33EoRT<=czGT`&CtHdPsvZ@F>k{GhtxdtrwWE*q0WLum46%bMrdfWl z8)lMT0(5V~II-Ujp7ot9OKCo%T!c6PW1uAQfC<4eG0j#WgqXRO9g3KGSU9@8NOM(3 z|B7P^u`od{j%ewCKYt~hN17FjU42epz{&v1^ht2>mi@AQjo}8>F))qu;fp-~bpQ~J?DgtSOuJu2tOikS zmK6m=hjh`5fFvKYIb4%Si#))Hg+wG0>Qr0ZABS_boZivnmC8@Hqo>oOMPch-#N8>6 zMY08)NCSWgxw!@I3H=JSuYYs!YE*hl&I9|EHa`}a6d?YZ-GR_VS|m^P2H{_Li{Jg; zJ_`-4x;f5S`=d~wI3FqpyfYYKpr)3DUo9`cb*VCxL2LDk)2UhbG}eYo&2Yk zNb$SWq^zZExCCfT4Mue;A1TMY|BgAVd#SqZkmT^Ee$e}z%?JgOs)rb*HUO$$rJ!WU z76p>rx=)kXwEAv2*zxwatx2{31u+g2M>c3{4u8wj8t>Ne<0#`ckZTn;3Br*n@JDT^z$E&7PN^_Ttfj3h{;Mx|7o! zyI>#y#5~WDZNj`))3Sz`QbtFM-JBqoP3K1$i(b?yc z9b!_H{VCa~9g&Jl*io7U35GY#4pGX8@gf8+{~fvroD0LYe}r4|57$57dGpkjzZ;{# z8X%6#LWah=Qg*}#RE~cHCh%bmLbmQ0RBs?-Gy~dKCv1$7gu;(8k`oRzTlB!n?d-z_ z%wl$0Ra_q8bOgJ#;GdYs76OH`cPFs$M%PZx1?Kxk^d$@4EU{$1w+ISh$`mDm--Qv1 z;$$QepYzpTKCwDmzOUZPE^caj8@?ys2QNyVI#=Lp=&N7qZZv*qf3=auORqJx#u-!m zG1W$RV|nA6OXn8ukDK+n|EL=c&(rG@R`PW^y4;PfGwXA#Ha3R8+R4r2Oa?vE&s0~U z2ib$YDY=v1v9T#>(mxneO{U&D-_G~D^Q`~y{w#@d2ry@!S->g`Hx4$=)Pi&M)V8$ByhCavzqO67qT6v)*p} zZu&0xF7__;uF>v}*1fGglkQ^nbJ4Dyt(&c?Yrn0P&67>>t&YyU?LE`2V(xRXP5vDG z9Qthd?3L`3Z1Ei26W^2K#{Hpe-4px|^be~KqYs-8(htgy#GF>UR$gw~mkqDWooc)4 zxN7$5lWOsrj%I@`1CyF!R!h<9ohiGixGDB2^xxWm4v##K#l=a*#^KMg*ZJ33_bvD3 zHtTH0YeqP1Nh1WkdYy&dCH{H6mAobVx&KVngg$~dD?f9-^R<>Y=Jb|vmL^xWv{l2+ zbXPM+H>v!(zD93rM`5Y*v^zbQH1ag9bb8W0DZh;ZQs-lf?TcR(MHMQR3{N-n^3rN) z+Zvt@XLBX@^IOtuC9g^=^Lb-8zlzO{1Yedng$I>~*Sw@0iB{4ITCb+J-(AWuIr3tF>&<{Xv>N)hhP%VwT*&>v7W7)ktI*0o-owt6AJ5OoXX8WK$;{*qv!6R3 zCqGBrv7e-O#w){}=6=SIa74|}Rs?Z`iXq2e^4|!IniKw1H{;9Z=Actad*=z<6zwGJ zOhLQezTOCbbT6JCn=fY%B0r|v{+zqoXWbgUDSwua+4K0NaAV1ums#9oscD$m%j7qf z@c+Iv;Qycb`15|Kb#len+)+zBn60QdCECKc2Ef`KfN)MQh*ZtB0pL=RQFLf#XDPuP zA9<>eyg^Tt`2z=!p&NvG+wdw%EL$XpD-H#vSjIvh%q78Y2VH>HFgb?+RZwcE1RKLb z=PWaR(*GiB1Fxh&8$*-zdJfWa6B?!klNVs=67~D73(yCG;C0rf8VljldXAA0wfR+C z#R{gc9CHeYJxjddD3uoyBC#YMKxWEa1zb!+GZUm;?6uZQmJLM0?*5i2U3JtFPH+dH zptOl=!4;%?!H8e8Vkh_;hl^A6@tCNC9aVfO=|Zm%qP)N;(9-&hksKIlP()jvT_ccoXAV7v??K<$+%IqC@x#iOClnHK9|%@TK95upzA8n*s7H6AN^6x)m6iC>1;35q0W zRHQK!3X<1JmYkDJXCaFlEE2>rGg%UEWWyIAv=9=<;S2QnrK(Ve|8os`w1VXrW|1QT zCH=xl*r>$BB2xZw2gXsq7QtA#Q(3Yz_t$F67N(khR~mj|Au@C;d7BxlRG&s1L^;ap7^O5dp?|2{K^oGf02O+MQjpGSD7;Vz~t4s4eSU2N+#goyXcm zo93_D$0OR8u=5{8(SkW}$6>810)+c&Sp)$0 z%7HmBHo&y;4@y6?^&c$Ggg@wrc(bVlghMuEh!*DO_WHBg4-ITl$1B>vIO7`T{zl(TF<=~%FVkQg$Kbph0POZ&VOaZ_ z%t#{!y4YdZy9nQvC4g{%f=N`ihp=ogRu4rT&dUx zu?qlU9+s?q0d@zl_UUot*!!7?kOvI0)4w+`E{*lSyHJ4(anN@%odCu`H`>_@2qwGFwE# zYS4Wo-yW(IXecCIG52HlEUiXSq&kIP~+3s)zCs@lXB!kXs2XEMi}D4S{ad8YoK{3 z;ct1@iz0~_RA?pw?v5Nh5$G^C%?1~SkV9A`7G*Lt|4tFnb$np@NJAioTxa0vu_S20 zgqDa%cp*>eF)_AXE<7hzL=44@y|yBgiO7)j=%1n7?~$LPqm-}5+2i5w3_1SWl;487PZ!Z+%OmTjO6*QlusJ*Wy_D#KmX z1!d&b5L=~1afE+^Oa4bxC5OyB(xTaTsDw~m;eCb01Y$CxkE$hLSP?LMwTBCb6v+rY z8{ML!D14)4^w+_W9O6ee8h8;Zmeg;XkFZokm&h1~8B!XxVB%8<6|xiKbz0(gqIsi(x485hBi*pl~qVx)N$lwon$ zpP>w81m3Gkh=?>AB27;n(aj_(!>!6iE#kkI;)~u`ae?r6))8#x=Z-9Y6t11c1*@A% ziR98E)4@@O7=HIs>3lvPA!)C6hVbVVPN1JlpsKpA=D1)Vjp`@+e@J3xoxjrj?%ZvO~b=1I@!jwS>q zCXM`QU!u!}tAR2crv`sp3ywTZd<-^n1GeleJY0ATj)a)UItLx4b5~xWUV)}IU(Wnp zQWQs?vQvj=T>q9-eF2Fm_nNpDQ59-I6^@vVgu6;uM#w$`8lJ&~*v~Y`L2qCf2@29& zln18kYKmTdQ5oW@APDOc?u`rhssR#bb`}zOs){y6FCuYJ^smdJD=AjHN01w%kWPA7 zK@5ZIZd*ePkMdmWwujvwMf+4mriZDB*eN3@<0~L+X(=IU82KAC3K)eG@m~qnT}6oc zY5q8V4L+@`Kn) zkG&7P54)~;F0@^zIovok$OXGT{F>_4Z`SG8)S+(^4b?Dpf@tDNj6c{$bc zI?d>q*?C*du18mkmGf%Fy8xUxs0@6#$*k!RN%EM>YOV;~hv6eZG zk2p&RJk9QGES_qIYQiNyP2vcwN$__R`1CQGuyy#jJAWNbKAHYOXfny}f|etQ2^3@9 z$vbqLPczTt<&)K@T*V>1fIXCFDOaQr&G?6%ZbaRXO9*oY2lV(;N=Ag);tEQ4t~u?O?3McIJ2VPv9tbkhduJW=e4aW;4Vs1XKVYuN*Cq z?Odz3zX7s%wHU7CgrEp$Wl);aE3w{jSau9McX)BTS*~|1xemb}k6^oBrmi82A&tHaoQvEB!AC+M>0F_m|vX$QWRE>h=)8^OH{AY{+vC)My4o${de z9p4H&`==SSDs1$TIg-N zBBno?r=P5sww&m>i=R0ofEmYi4fVgGUp%Fafig~LjF$-oX@oK^t2K0WO+C?HHQ1Ez z;zEE9=pig~MiV~2VQ!qw`Ui}_+~V!jCNbNA3+*h1K#382#|>UU!nN^ zFeJrZp;U4P*^TAh#Q>eGZPeYUaSt8p2?IkzD#dz-*nAQ%;E8lTG3-@*+$mKhP{Kw^3`T?tS0eCsV!@-xJbkJ7aZPvOx( z#r&`%CKhma$Dmp$53*|F5|^0UMFU*gOGs+p(&Ni#yGf)N1Qb!U)$<31*bc=^#GAmz zZL&88viB1=#IY5o_@8XZif*XuD4gqvhP+*or!a?ggkJU9`vgUxcXL%~9WZ1o?m z9I;YCp9VLOMd46cp0ZgBzz_kPQf$$F-JpCnCYO~&-c>+W>#$(+brb~%L48E{}r3>bmF~wCm!VDkCv%@7n<$Sd#2(vQ(o=&Fz}iw z2=;rBU&&HZfIQj95Q4JCpRElsl8jp!lxbWQrzJ67E80Ce2oM4U6$G8u+%9W?S>gWc ztBl8vLouy^07dS4=CjEeiUXanR6D&eaX^#SO z#!$UphYE_KHp?KR$`Q9uc2o8uOt{?OtOJLgPIlkdgAStp04UOHU^YUzM1gr~h}8Xz zvmFjgq}FLTT%~OoFBPSDSbVq+(9$!NdNkl+ewJ-Tg3v+COGGJsn^sD9(yG)n0Dzk? zA1$I6pt8Ru7%Nt8vqBW*g+7-JdVIzI$GTNku)ko7fyoh<$G(_F5}~EbUq~NiDwa4Q zhZgK*`Kbl<1G#Z z&q7+;ef5(J1hzg`eGfUOr9e6Tmj98rvcO%}O1X0REidrqy^LDYg{c7i0)_%8;IJNZ zxm9(%nmlr@(kuKxK!)WavHZ({#i?Vj;4S&6tDv!)ba_EL84gq_D9Q+@Y<6=9-?FTF zKhElE3JA*U{ImaWx64w#-{xbrHV{Xzrh7@E7i<>*ZIJt?L&dQLTasn&Ci?DS`C1Fk z#?&x1&(s#-@9WsE5KDNx4mWs|u2P#~+V*=$^5e~bm3PWvn!En1=a0WW{1!k}k`ZgLaa}hC{ zNIb#7W6L17yZA_R+opp~i%+e}IGqpxh!NK^EZ9pIs`Id*ZB85?%V(zK=NRER0~vvlG~&1 zN_+;2l;07cILviK9ogXSLE^N14=qhW7s9j7kzHVYdWkda1ev5?Y~iUS*%vK<)34oHhuF~W z(7_hvmAZD@{OF1_9P5zqJQ zfHVHDvXWmT6zkb4p~nxz;jmFBAlC^*#Q;Kq>tIGkKrel2+`~w+Ge)H2-#=tSeF{h=yI15z5h!6vEg6PcfIOAbdq}g9A8o>pjNQS0NEY* zke>kIV9pp2XUC9MF4gQ3I5gXCeMt;}B#;OWurqW>LI`If)H?|Z5N%7uFC4e){*k8+6>k0)0tvB%o`)iA(>#(v|znLflH-p@+UD$nMwgd163PFEYvam%&U zyd5o#t|ys|Y%R}r$D@-=lN%W)Oz2@4hRuFY zbt!ggd>S5JMHXS2x9=Y&;OY45y`4GR zH~)ImKAD{7EJjs8SAZ*tE03wzo!^w)r1I%{nw;JJZgF*_x95j`=ZL(-JQ6)AzQj$b z?S!Vpudy98em!riyNcoSecY%K!<&74Xg#{F&fgirW$zl)sK9Nn^^N{mkC^YB{6zTy z_ko{;PpT{1o#IYlU%`;&Zf5i4zx^!N%kAdK=D4==_0B8&o&0{?5PnQv#13BTe$id8 zX>YcV-E;dIxY8s>R4)~ejSSbTG96QuUx|3Qe=qVMWNUlhBoKlq;iN`s#LF?(Jd3PpSlodkm>i?tc9J?cFqcxmK zCe}>s%uHRa_%UA3S6T>HA;XtM%Q7b!aB zFI$o1^owmBcI}{xw{%0u_!)m>n9_50t>u0k3HQ7YcrB7rcgz3 zCeeKoK5XzXPvm&uW#*aHNU(kdwslCbrjHxfF2B;G%apM+)QA3etX?b(uoZaKhS}&* zh=l*|E%Bc};}r8kzfKf?z79oUVd3!quHydu>Ev*5I{+TK-8}EY^F8T4e6#SFIMbd0 zu8yT~eC=sAK%-L@4j@3n7VaPrWnITQ{v2*V!CV`eDhC9z{e!cmy2u5KLR|+0W;r2U z26@!A1Y8vSC@E>}{rTwq8w3fr1Y})MmiJ0d@ofcKsk`zvLm;gC4X97?y!ap1q~%*{ z6(ia~b^)vF7JHn2jxf;+Rug9maAn)^(G*w05w18e@1Rt1ESjH^E%He~LsJ2#e%mHq zR0YT>!|p@p;k^Z4?~vR3-=vq5*jY7IsqqCan5j|A87ZU`EgAPNAiEVeB@GvXTWJ@L zD7%o2Us}~d{_`z9W={Jp4oh))H?HdeZ?xyL}(d!DGA&c+>L=}NR{;h~VF{f%{E)mZIG z^6OoYY{S;1W69L=Q`?;k0>Q%-4!Q=ur(K_%QNK#qxe#VChqG%FEG~`3$rTnueLqxd zGLfxjAoL-H{R$hYb%x&^K9m;(MfT>$(x@l#jL+gFoVdC)Ga0di9mh^&1FBH2%(!k{ z&_LHvicRq~?XhBaz-3au%DupvND*6O0Ac_xc&I#x+pU;UZlF=21ff*D#?Kd%l$>Fc zZAWvQo2K=|O;S!T>;4+A7g4u+$xppkG=7`vch z*N@4TZnnG#z*&AEYPV%UEuGX=+$2`VS;N-KkbisxY6(T#$S8xLvBXYd(*fEj`^my6 zG+)9g50e@S-Pj|$L$L-8HNVgLfv8L}q7RQ(&3V5`_ZO43_7kxQJidQEvbEBd^BUNUDzo&as@>GnEj?DRV4!#b*!qtz&X zrkd~Brb+dA*m*P9n82ID=FH005PfcKqv9f*Ih%I@Xq(twgWi5|5+Y_|e6(toq1(1T9l%UTvt*^(bL^kymrNv zh7{Yq`P>@(n~t^QCw&Ix^E%a}tBK zwfxI+S>g+VHSzq=r5K^*x||370eFgP{wDMnsb8X`?9UMFiM8+QqabiMunJkOLou^@ z--OdH7Jot{P6kRQ#U6$98OQg627)LCDaupU}lhziU3W3U_itmdi zYloce)wQ=piG~i77@A8Pob?(Uq}(|CGjDM8Kbn*LnnrTBTDcqvz@Tc(oGHG37NNXV zf#wx3NzY;uQG4W5J^Oe(!@n0`;rmxrxt%?$X8E@yTQiFqP7gTA+^B&bV3b~&-of=D z)H0%T=^4=~@y;<+o$v{EWB1k<W?W0`^$dh{rAQ6YDJOrPa>*$+(!G_pVUu@6aB5eh3un`X_7)8BbS ze=Buey3mV(BrCSWs0t=j;ah|DgdcHLA(VpziSfAFy=R}S!>s0t5u_5lA z^!Dpn^k#Wa_NHU6`!6*|{2SG^!%l7Aejj|;APO?c0%~=id=7&h&xgWW`IFT3_>(&~ z4q@mg>MO^C+u8Y2M0IU7wBc#BZ5h|Y;Y{OK&s|5|-SBwtt5Pn`Z^+lnS;rfaaDP%H|fLv`SQkeTX=YxIXMBcfZI)vA`&KI@M6TSD+ul3T$ z{Qu?R(x=r1u*li1vtE|mC~LJSt16dSW*reQb)cOcv%9~oz_vgSxDv90}`v ztiUA-9R`!L%>eXTubBWnQGmrslxuf0gw4!m zXs*J}&W^Z9zST4r?c^;6xZgG_a*kqfT#-y!;v}hXHdbpkEN6xw}9-iQrCzB!kt$XCkqQ-ygf2dMaTNKtHtU8&+aZ)wHUa zMT=H0Z+Q?TDD_{Mdo4Z3N>Nk}o|Ud$&>)Xub4htNnb>$(mK=gmJ2*$1qhrTxrXDoO zZ8oIyT3%fXw%#=wIf+Ggn+_8~A1!ClG8?u&@S=vc80Zk|8V)mH ze{l*qc8VbH_J9w0+p1bK;IuJ1Ro+W1?>v$I%T?Pw-z=IH>F!f0(%rZvenLv{S zUCqiN7Yl8mQO2Bl(6SRMTdP4qAbd5poGnQS5_j;3Y4rCh7EK(N;UTsY8JO&esx95! zz7f$gF^^<4gJXmsUVEK1qP86OpX17vI9Dr6eJnuan*3D}V=EW-ngeXY3@oMd;LHj* z(LUvB`YRmttAGj$lB#lv)q}mk#h#Q8rz)$e@D8>K zuhUps2N5uOplBu>gDZc&(y+(rdC|eIlU-?&2!uZ}Q72&F!Gu{~J87|7X?W4HdlhD- z$gAvhnKu+k&@h&95bJ7A(Wg-aeZ&7&P-r*^BeP|aOu|wx5yr-*GiDPw2JyqFkuhp(95q)F|1@dNVW&-x$>C#nqf~v)5?4dM_GR} zctdxyVZx_QdI5bY#exr7WkvCA1-9sK9JqD?4is80#ZtrAEhL)~cKf2FZv8h0EDJ-+ z0%U|Zb~l{tP&zxybq0CM0q`>EiHAWJaBQpsThQeF_fATqi^=kTjQ7^n!=SgpPc`zH ztG~nGMRDND!cWC!rxa8(XY-iPSn}*ypVZVs|FS1Jq?YtEx9&GKcn)F$FUIa#hwsssw!C}`CdiP+5>VFbB-H}Z_-a9u%Ba2OpGv$fX|+hD|? zFk4AoXgo*x^UgcXraO&EkNHtbDfR6E-eCWDeS)wp;0?qqIXLJH0t`Q zQR#Q(_9G_a?bK9Q(v?4vu0KV8u;F3^q!sNr8vwd2(q;17y_`Sd{;eKyZ_Oa`UgHjK z4`8}XMO!6~hr>TBOs4lB(>WJDs)O0)lp|-uE?|ta%v4p`a zIe0@XIu>2rv_)v~x)8!@pGdPCq^elobR3pjn zE5u(->*QVL>3_!=Nh}1n48}nY6FlPO+lKzzWv;DMPHsb=M#w&?JBQ#%NVLb0A`z|0V3vwvhBKvJmUDMeng0DvN+)3$*KV zNVlsboCBv;i(=P_J(95~AgTN(7*`@j+CM&?f+18& z-zh6W&su$hleTc|P2rJOdD0hZ{kloNuVmQdgv4PuY0T=J0 z(;odhPVPyM-bgu^mI2@+r_v!BG6?3xZddVX6w-$ z%n=?HYONpZRq#IwysN$k)@)o6BRYQ_p7p%^zN8=>Se^%SPE;czkaPxJ;O`DfMl zOjR-e$FUD={z7_1l*UhK0^thWR(gIxC?EO|*g0ph43^;S(^Hd*LZ^+Nz_v0%k8~kf z8wX3J)~t>4t24|vWfPrK$&xv=G7e-Ke*S58-+Mf%x~b{q@X~yoqB>b|A#Y1T-V_iTsfwbtskjO!rKc zFO)BS?DnbQqj!9n1o$|m%B3kP@~bn-r5@8u6y zhO32FVq}u@Dz+7>Q}c3hD8D&8ukM3}p-C6X_%yvOI2fO2Zcs@xTf>zdt zZgt{CP%HgI-%?n0E}+vW7qAY{GqH*GPW0k>wSO2sJ3b3oKF!UHCKxxaNfCWt#uOza zv>MwO@&pw}Zm?+L;p18nygTik#vzN*MtLT>5#7!VEytNK#YC>LDB`3MEDv)ha{M#H z4Kx10x)^<6-^L8qI9R-MIi8&1>U~7)1bI_9S>E}y~~}1%oP?l zDplrXh(8B^;=R}%&1NSQw<=yDwX-`~lPBSbgYXQCGZS18(L;KK^vJkEJ_x=FeB-YI zUoIdMxr6IzWb3&bv?nwyG%RfIa5bc_$H;kv`{uPZ*_Q&LN46o?mgwrS734PvMIg7v zCyxVzz@2DH@YOhy^jz?%{!IHg?o<8LyT06UxgUJt3w?dji{vBl`Sqc-x#*Su#N|bF z&1Xn==Y{V<_h4vQYQyDhb7rd7RQV%4^u7M>ToJIDlz-YjNe9wxu{aL+GFuZ#4Z3rs zct>Hlzqjj&rh8Nh``owV@u0rXJVsxVYzj@2c_P0o-4cxjg0A zrxXx8xZVtGp42YYHjvvf+b7+8fSY#&H^FVXmtwbkjh(Dt((3Hzdbdx{*|)><%ev0B z52>Tssr;!g%TZn7ir4vdCGZ}Cp*_FO}@#Be9z*w_WpIj=YZGRtM>k7 zjj4RKT>4d0kI=d0%^fCjm)tiFxsb;hzNzjpunYrg?` zY{L?1^2OK&ssq+@_08ZG+HVp35TcTeZ(bV~XvvGFZiB?;!a4{J;m2pvk>O@@u=rj% zslEDnHwOLs1w&=J0Y3fG3LnLc36?L@p8;P$6VVsVFQoh6Gi-9Kp5&&*o>YayyrON8 z7!R2n;b>k+Pyj?!On z!m1x;<*g?TAJ*$yl&~}M^^;-2NPPYKxx#s$`PEVTd#|oS8qdsg@cyEfv|1+CO~8lL zr0P#)qt@6^l3uU$M+w)>V>?bM=~EDHG2A#K%dJ39bG zc8Ub6QW_5?#Hwb{G)_Y=%>ave3iB^zqXbbyhiy!B{4d1GFRZvtuOQpZK1n&ker`>c zf|gUJTSEb=!Y(;5!WLn6`vBDB$-(4_gytb}Iek#hLgtn;x|s)fD;1u5Lb15uywvZm zKaD?d;Hb;6xQ>gyYroJ~6!5|m6*E+njJ4hY0Hs?{<##Q0M8zTI$MaZiyn?y)?ntF* zi3X*!{-Y`bSVFCFa7qhe(2;YJkw0WnU=?ji{*hm%%k55WILFjtQsUJgoUyq+-3dqf zI|P9<+4UKfWr0tPK)>jh%b+VUGqv3cNDYA;o@h+{e_l&opzkIkHMX}Ev-Slj7swk^Q6!s?FGMJYTe`mI^Q`>X%P6d! zl8{t(8GPuOQvWkB=REb#IdOgMB|x=lgg~epB=ps3E_!HNxb$gQ$=pTy2L(NoRPB#8l3an9r5PmPh zA2_VpTyS6!%T26gYvyrHy{uBTC}is^ARd=%^Fw-_c?-gQ*`oGMk$6S7h?oj@kiaU6 zF_ZiZ7x7`rJqNEyo&@tnR7iI3RNNKy*jLuwo@?_~0C$EVI5V03H0xd* zg0S~Cw&)iRaxEjHZlh&xs_xM;yRb2~Dnw$(uX&c-Dr7O6HXDZ?)03OLLvMem#1C%I zdd}2vH^m;_RE^<>p;sWPH7m9U;cvYPLO(YQaxOxIJblOCR`Qs$-5ZVyA>3FhNg~`> zU!soC&LcmhZNx;jNr|yh<}go5KNy!G2NTkpMRR*X!*i*acP=%)3NXB)zcOZ6qea42 z5=ui4@cm8fS_q$$KGohEDn)qE_GGHTy`(QoXTN@PW!$XFrL+Elc}f3k5w=mHQfQyA ze&UUsTwENN32t=9G60$hhZLA)E6L~|`Fm4lyKmuV=ttrzUDDVux!yO$Aq0nBb*%$Y z3ya9u0&XE3J%dP1LRF%=*?ji_Qy3x4_qwcV;R2TCcQ`bzP!j=z@^QI4+PGrOx;-m| zV>-mjuFl6e1-0hvtrsNaA2V=kIfnUGcYn@GHz~COZ01xa>@-`oCZo~${_HDIqrO4> z<#5L%X}mi%YD{N%T$UW>VB}`~+dDgW;OR;$!I6qRb=!EZW2i)0#me>J{7++rw!_6I zKeKAmTx4q=eB}qeb`*C8i;`%&P~eNH!Cr=4sDWGoYRZ-&v=U%}z8$Mn2HgvPWNUsd z%}I}K0PY*E<7d7-<>K0FIS?re)~&u*HOHK5T)e18L4XTkuaWe3Wt>gGenFh@Px=B? z5xPSKJhlvi16kgk7$GNx85!g;G(npnUrwJnnGu3WYqj z=j=2Zlda;Ar505tvxr=h=MUubd#Aa~%E4*x8fATCyt7kJtDb|(^j?bPeL{Z|ey8uU z6<=h7KMMYOsT~A8<i*zD=B!yG|lT|asQe8nZ`wG z4^*Tjv+@$w3XpB`Y=c-PITP#daatC2i6{g=a2bb#0O?gL2vSWE1Un=A#^m@8pPuR{ zHofe%9KNsNmlCi+9JkB-hy=$PS+^%|-&y?=pvo{4J-|H8#sx#RdYXuP*FqVDsm=Jafa^5!w2iWj$p@MYDMA=Egmgkv3d#S)DJAjfZorKn=q5 zK$y<6wz9ZhLKmz7@20O#(&#~*{~~+3*2Wc97E2ap_v=0n5MxPa2*24d6CEko3X|V4 zlGkN##{1m_yKte4o(lN>dkw>fD_vsf4IOKsD$7Eo=AY()ewWJsCU!gqCfX9N6li{< zlGGcm89QH-G~l7`POZs=Z>a9y-%7qPYWnY6I*eDBO{&A4@=1nj@bl&aBOE&_YwP1t z?!MT_pd3DG=`$C((N;#YBAIFZd;z^}(FH3brSAzg=T4Vt#W^VtbR^m*^)`VoyZ|8+ z)3L%;6it=I-UcX8tpML?^jQnwn)J=~wRrlcn3am%!K}~#!WcY)mZQ87-;D)TWkqWx z#|sHb!4X!iFCE9BfE&$kMh|wJRs|t4b{S9Qv$lYrI=|azm%??OSeNPN*STyn%ZIGW zS9YK_*Z3>>vNcDD|AYb>j>@cXdm>vS1(wXcXx7waqgYzLw|6ZBUHRMpS{+(sY>%5c zu){_l(!iR1)w`14@?k4w6pz^OJ@-4h@QG1tY#wb_Laz~)i};l4SR8yUX}jjGc#K;w zr*v^;9WDH}Pv-D8L}AS&r#8Bpqv}Xoqb0^mqlUKd^KS_Mc97k-Mrx}ldB9w`Vgxh^ zo^Y$U`|s@))TVyO!hS8FW35PZ&Z&08Ix*s{I0;$?YjXEGLETW_mgB)GKa z8A2{>qd{4{xXy{ZpOvya4|B*;63?(r;IN5U>$h8d5q#+h*O+VhQh{Rq9zu5C5M&_h z5c!{Mz_|P@^>KUG(KEQ0V(o5XPKNQs=5e#7i&+#M6n|o9gDO@MoWbB&M5JW^4jZo! zS(xi3ulb!sdYHf^0;5>qT2WNwNmul~SMgERh9xf9?Qc6n-smZ}L zdF9Mt*af?1xS7_TIWI-s{}yueyI#{W6nS%f}7}0dE^! z+GFo7>IL!}CCZ5S5%>LQ?eMbE7{7ZU*l`k~pT0mvi}*A8`5-gke66_+eYaD1R%L(= zxeH@^z4~dSUoOYCpG-a=6!YpxZ--}?UWQY+Yn1M_03s)r`s7`y@U@|W~ z3|#xn6VTXPT3QZ|!{kCY0W+v=ZkIOgE+WK{VTOJ-*7fza&Rcavb(8rVcFtT1ek98c zP!Q7C;Fl77rrh}%HeH0sS=pDv(9w^!97?Z~(mtjliQrbuGKiURi;Y!Eqm6&Sc{T=3 zi%yk=uPW@>MszS2?Rx$ywHVvST(b-s2Rdc5gRc1R@jn82M#v76Iu1>$>+k93qOE6_0aEs7!~w9~r}Yg^13?3;zGYLzs}I%!l6lvE&YfVQpj%YTzCMp~ z!!+ddjvGC?5m}s1cE5t9E_mX;GyNMOh#+ERvcC2&M_B**o7WvQ?ot*qswb1nO1S&S z!4O8Hz#)M-Yi&9d=(C`#;f`AQYV;zlRy!ht?Z4*a>%$}UovMymkz-J(qP&mcQ)!(4 z=;gMoVsWd15&6p!+$5Kkw3C=prcUhMv|8@vfHkp}C$xl!o&^Cwe0e4ZNl{h6oS7Sv zyknUlh$b9EgdGgm7Mn?N{YJRd@?}`Ws@%u$tEXyylwub(_6hjVO^+lUA%vnsn~LYc zz%OL*z%iL~491G`@Viw0gEy129ZJ^*UQlllH~j?C*7ZnjQ#M{U3!U==(Jr&Y#DUZ` ze-5_JFYr~>WksCj#zjEaH262+O$4COpF3lhX>VmSc~i0$>Qb`Ukz>~?_fa!49_Wy8 zLSr|qqYxciItoSj+S{cmNr$hjqf0sbFmQM&5$g*3(d}$h1N#nVJP!0rnP$jdWr|$O zOXP|7|E_4dV}HY@s@u{#=AEND2*YN+JGKy?)MG#6_pQ95R`KsR@G^%3Ii4e=$Z#5I zIcnwi5u~w%oO${_JBb{BF@q>kN-}=~qkWF7A}X_8A^yO#!Hg(!KfF|;Y;q;ItmVqs zcC^yZ9E`nZ3MNYIeiTVW5ixW$Rf4UVLc2bxG5k=dRM3>B*k;PV)B3KLJ zaGKcl0#!zd3_`M(1g|jqAj1U^+yDn6cjoFjOR}0(XGQFFDLW{GY>#Tx6*peT_Dvp5 zY{1-gtlYCq8#UeBSWHvIwG6-hTQH09IID<5-~|hZ$kAQ=pnngHfnc#}Nio-)!q@1X ze_IM|RlXy@DlR6w7ZhyQX-jR^)FnD*Wogvn?bzSE}610G%NAa-f1rkUc( z6)d=@oZ`ySDJknwfi$aGsmAwj3wS(9VJTp&15 z`7*(d9oc*@WDfiK_(m#UgZoaoX}n8WSga#K6G$-Cn4{QAk)(Nn#{@AQUVFg>r{L;D z>nTshD%GMC$PTV#y&<2dz!k0N=kR8C!6?YG4;bw$hGWZ6{i)KNzhi)QDhEoQIz~fI zsN*N9EQPP~hTm?SU@PeM{G5efKeZ=1iyRH9-W-51+VK@b5b+~FEYzi>&vn2awkTe6 z>sRgVbbi&sStvXw^A21G_R(!_xZsV&-|2zTS-x`bj+cv*h|;go=T1${FB+lZL*u2w zvEH%xPSC8?(S$+DaT%)X6q9?Y*% zI@HlVQ9~b^_olnynRKa1ny+r73q>Py!(81R;BNLvYZ^1xBxf)u-}KiC-n-lNXuk`F zp020i_1+FBBCfxqKRvFJ=u%8K92muorV$bN2NahMze(d+bM?|Q3!e`ENWF(aim}ua zvrX4?8B7;A859!~6NtKu)wWkeyuTbFrmmTx4}2a;13^xQg0x=pGC}hK%{HB;YfJ`)2e|x=d9| zH*qn>%kE*yN_yeyga7o)_O~ssEu$@SEc_FMjNhhl2k@)qHlJ2#(qMc6q8N)Dt4cea z&6L&4u#T?%|I!ElK?e#)u9)uF65D%H~}?=PLh#M)KIdbX|0(! zr7_KEc!BFM07lw$Zadv5%tFe0t*eFMESl4u@XX~`l)Xc)MhuI?9R7 zY+OKz(;YYZF-yyg;3ZL`DfbA*RKG!66~~g4!*d)Hv2E^;j}H$Vj&No)Up0o8{t-1gIR8FiR3V!3G!EfyR_|l7CApnFXXE z@Sk36yMO1GM7HD00!?K4X0GIL7?UJ^#u**e{qr4Rs<6F4nP|_);&dauA~%cirk@7R zV33T#GR^w%FySU?1mAXjbuls-1AA*4IS=aG52{cblx!p6WC@WtvWXX~2qHLlS@-s` z%sEbra&vUGtpD7_i{(p2H~+Bx2d&oCuOa*b9uSBd$2)|ltOkx)C@HWs?M1nc?4XhH zfYaxmi8>O*wv`ovvW8)D8GB@W6sg8WwGU`rUA=j>iRTioLE-!~1zFV7zsa%phB_Lj z1gW^tA~6Q%@o(ob2P_%Ulb~VbvF?LIr$z2yGh7)nQbI1qyH7zDlGx!WDQpMg<#>u3 zOP{P2cvntOqtX_y*G$$O(!6$odNYF1fyCm)VNva?ft3exvB)W5@4@IQpIN zkcmj@EM5ZTiB1{WDk(Sugw##)TH?GsY!(Low7|S|hybR!D!f36MmO}9@B1kXUJ%95 z-8i<#Y(w4yK`4?h3Jt}w;i$)GWvv}JLV5|@XRH_YMp2Pk-1hp|&JJXGD!a}uhK_Bx z2Ung6-E|fSFx)MpONHAWZD?liNNC_|NfU|$v+SvTTv*8ne$(dPDmHrBvZN~uA{n>g zBY(fr*`1WYxmxT05}eSp*!TBjs2qUZpE)X1DaI%;kK`2VOM>UOmwoYbKejcKB9BJc z4ncGz>2C9z7NFl5ZmW{=64Xh3zNp^1ORh+_NXV92F3Cb?EhW$_{xL zDrxi6I>&vvw<9WYk zvI8RaV`*YgZ6Pqs`|wPH%#cv&QQndWI^Ul<$j>1t$A&T+Xrx-M=*G-+V;;)_u`gNb z*RzKIsrKAizf0CAClKy%L_Om2J(up%TCG_;g-xUf?nbaeI-onLFZzvSWMzh;>a)a7 zPh^d=HL4DFrbUHUhu6Et9Zz@J8_f(`;W8I@9u>Dut!4v!Unnk7w}oS9_ZA0%T!7L3 z+h!8y=v9WVpgZ&rg}vQ+jixBjDzI~E*v`LU8XSqlym7w2UH~GE>RZxHX+mkLQ+%Jb zav4{*DK8ix?cC^)y|h(|)@hh~K7=G+e6obaE)4S@r+AaRA;+wA^OPRv4iPPlitI5p znm8?q=&(`lXef0vbETjMV)Q8SkK)aJ7fsrzy=NHNk^O%T6^Yu$AOO^`(DN9AwvItV zOq2Gs*^S*LWs;H!0f&gF1cZUv}pq_sYig*kwRgf4+NT|ejx<7-tWvzk-7Xq0{EkUr>3Tq<${Is751@6S)!k<>4( zCF^#L0)&&B|3~uhQCjoKE@1f?t z;T%}mRj`C#Q+Rcci-Sh~Z!Ov6w|r+)o+WAHPa1Q7pve;RR!oT=s6ANE_*_DgA8L*ALa>*yv`Za(59(Z0u4r*k#B2ROS%u z2HSt_g`{Z@+7lcx> zzOAWdIpWmAO^T#yYtga~iI_*K!NZxL*{(Aj^7nw1rRGamJ)y-zCaJTwYtX!~e}`OB z>^lra5o8=(J1GJFGqCpbpUp7RIn+|urH>*-PDD1SucM<+lt|LgzlUV}rnk1u+B6DgyZ6_6s+HxsI$jp{&GUo8P`*>cu9C6!*eXOq3%ep(WM3i@kDO@@jOlaH=L#k0b-#Z`|% z36Fvm!$;%8^Lc#VB)Ku=QcqV4iMEmEQ}Ze3%6dDgr&%!Hw&(|iBc}JHPg*C?hr)-| zM~^q<@lfL|pHc|K-cddTK3pGyo?PB8+^B>RwNEkC$cB{z z8KIz1{IGsQujT+CcK5yJ4~9MtK8qsa(8>W=oO zogF{~GBtrV0!;(W2SGE28{duQ-tmy}o+&ZjapYUe)qGDQUO1>QRL}=^W;Z$Hic?+#&4*D3go9EX8sf%!K1y#WIGd&UhRf;2ScNAAZ&mmbsR#NRj6598061j0#A*5`+@ zQ6?L?4rVv&gSa%NghyF|R0P=<($9ZQE(PqU!0QQ5zU%J3sAG%SOW96_>ytYsR4!Cx z89dOC3^3E%^kwR9O4!m}|0T2dRLOGZ<*88VQd%d~@c)X&{%>&_G|8#(OP1!8oH?*~ zT&Kf12%W*y@t{hp`&dfope#geyVPU>RJERk+5q^oX@vtdDL{{wn=J1J7T&>1M}Mfc~}!&BjRG2VmO?(%hm z?ISbTBmN0oZ2@KDV#1b@ivRVo3GlqsaynGjddYB}xi_^n0Dw)6s z6|vJDPw6FIu^VO_;83m!!}kdw+p)pTRD%=i6lbH6PO|?lcVYP^Zks;Vz*YGPt^elQ zlE7nP8%$%X16M(~-)Jq>iyL1$u!}H)RTH)-fX@7-2X*QZE$N#Z>GugYkyM=*NDHvf z_wyXZ-e&RW`lK3)faH}XY$;5d7=blZda^X*(CWz98wTkQIX3%_a5d#wUNFl zdx%E17aGEh!OK$6JtQ8;AGR%tfl}Z`AC){_3-8@GU{C>id|rXfQCD0kS9^F_pWdpG zu@y+g3no(~L9B=m7QSM!%vqmQe?Oru-Je^}vl}7%&|vD>h)Q~CrbTv?e^+w+tzh=? zj^{~jUZ?Zl5?s>BcohB%Z;Y=b3!(Gep9+QV7C>*VRs^V)NfREyJa4r#`iyNY1aaeX+I7{KrVvh#fQ0J9VrCPqz+M zfHh)zCT5=gJ|%#nI{HVEu5NcDCD&wN=B@7ySGr*8G8Of~uYQvW%8~kb}Lohg8FX zVR%bD)?NprwFhSx#s#uV-U_DepP}w;kHH&tkn>JMUxuTqR%sX*bo0fn3s|eyJnT65 zJpm!Mgx}^Onmod_ngu~{&;mC1t0f8=ui#B)7oz)484+2_bsM7orv@WC@A+0mVieR2 z`~IOX5~NCpRg**t*gKmv`}7w-8G{$wwPBR9X>A!MjY}|S)2ti#&~-)7D66QYZt0zl z=cTHNiXbw-i&d`9)=gow7q+XHyV#qxI0)L^(L}Ff$Q#iB2wxtfNM0JP9}1qdLoGJ{ za7V0rE||8!n3%DM5ay-v4BB|Y;%6@_G}h)Ano&5?u1(qMW=Q|7XtM1rgMrT7nPtX( zfo1KF+&YnA&vbM;3Z=!09d7HjWdoiYhlc`1g+VN`h(lGl<^dxsxK>YNi&lV619YT> z2Uyx0+I9p{Eh4AHSbf1KdqMRCYdgiW`{SR=Y3gHMS=>+52>6 z2v5w?6}Tq!g%)iXf2cOY7j*?IHPV$~t8P|xJ&*(WpXXA?&xfs*1;f$a;{S?NU1=8v z)aAZm6?s?lKJ27@$zQVgM~SREQKj+{wR{q9!5dcoR#ok?nRoeLy{@jAWBQxxep-p8 z(oE52F^nQCfE8u7G4I#N`TqySI+S%vl<}n$Se%*N&+nS1t&SCy zq;nkqQTwA)VFh=6>3kU@57xz7Dhs?fFO$+s0*CoP{-R~aKoEKNP)PKR;kK=Y`{3lM z;I(Y}SLg`|f5E!XH8#pTX^H}Crb4i$lbRk*tcUQUbz=u_d(_~on(vAdBD$Q@Qwm{& zCsn%IlWr8$T=&uHLon9j)?s57S15g}OYpP3BiSgwyPje^KF3RfKTN9Wh(UxK4ksbY zmkfx3?nmz7m|LRVG($rbh#Cz_sDSdne;sIH@!UF7tgTEX^iqEvW6fem7B6e_vyL;4+S?|$-F;$}Z3^r54@ z%`N!XVs2qdw@gVaZ9uqApsm@*BI9OQvn(YRkLDz3uU)!O= z+!`xMZZic|4R8cI(0^e^wWOrAL?k*dNFH|Nm|>Itvtxx3vj0-3coH#HSm`Jv_+48z z%dcD63#OyC6zxrs#~UH5Yx z9I>%mF5#v{;#q4oOeHD~yt6hsmzoGxyvO-z<+(6xf2XUp&#+=DxgG6F3_z0OG%p8& ze{dgc@mf`=A%Lb~sM`(L8z+NV6xXR<36Y z*Nt(HTN~+nS?Vkut0jt3ZG73Os)$trL!Gv zm44O(mY@k+qj-V_VXE7hsOs;Ao=DFbzsrvsgg|CWzQ{g>6dv(J9Ble;6+2wzTXI5a z!cUzlBaIKEL71Ru)&=z_Q3cxxUAE83x$+lS&H=zc-U<4@oN3+485H~q+qMgZdUXT2 zJ{7GYC9GJdtmfq+Un$7=I3|FK!>@h30*6^w0G_o3-JQ$)SKuwvH{;=i;4m*tN$4xg zc-bOJgkMTaVU=-SQ;r?at@E8G-oR(egORbp(#pyVopzu4s+XCusgsE@5Yu$pK^mVV zpPUc#`^(+)ai!E{66Neg+-9^F zv$NS*esyk;^!D14V|KHx;n_uX=8fJSJRs%vqdMhA24HfV^hTGf7p+Uc6Yt~3oRm4l zwm%uQG31rzLG>tjalIDMe7{yb4UwCz4WMN|!GHJKOF45PQWNVK(H+*sbF`#J}U9YYaZLSWwguc>nnw5XeC%+hM-TFO3eMo=Az7an=-z4od?+x#*4=+Mx z;FA-}A$?DHL_mXXN$B?LL;*qgJouzMHGKAdsJ&*MY@U1)+i}?0d`5)Tf7tq%d{%mG z-nDN&nr|w3#gGw5mXYQB4`t^R99bW=`(!e)CN?IvZF^$db~?7z6Wf_gG_h^lw(X8} z@}BSV+?=oaqHq4SyKC33TF+kVS-&T8qO-Gq!#BM%zEf~_Yv_wmjQd&qSiGs=&G^{4 z?p^fQvl-l>Fi7cz=k4$$-`{mj5NU8zxQpe3cH<&|DR|OG_{#R6e*V1N%DBH4 zG;$nyd~jJXgVmLWL>lc*Pk}k*c2EC- z-N6j$>Qw3e>7Re8yz3Z+ZUfd%$r^1Dlq3+ zRLAZ7=Pow%4Bp+ifBJb)q~tC%loGQMlTZ`;n6EZY$MW>8VJ|aAfoJmqh5oF{np)rC>_!D=S|C4+c$Z#D*7b&fxOtU_!Mn&Ky{WRCE%UDIcaI0Fsm3Glse={Qsj)idspplsI?^19srR~YbF{jy#;=IV zDZjZ}rLn8PG5RfNeXD1>Yt^yjZnv?rKxmzqdQf zWkbHs3K4SE}=F(TQg%R&d|*=YU#(HX%G^n30=vG}*pjdi+rwo=k}v49MsK zcsEUk>@$~cGIqu|JmK19hcP&~)i-6X4P0ZMM~70;$7Xh9l)$;L!(d7uC8n zjbC?%T>HPDNObG2U16uBW3~)8Q+^JBNEu(h3>c^jzo-Uq%Tcf9J~~#nNIQ+>a`*tRZ80E=E}=Y@&sfT}ba3ooZhv zY@E(6LX8v|DASsqBwGtGUVxG=d|V-p^aw^o9Yb9CS!47~x*AY!8cgOKZWrg_l>uUz zROZfLRuW0PsYgVOzDI?z^mf?|xeC>cC+G}^xRrQRM^L{Q$hbqw#vWU`?{+%O(&HSA z+EDpH$*I*XRC1$8}JXV`BCm7DJ`*uADh;lYkFi<7&zdaE(PyC&9d3S^hA)r z01V`pHboh>6t{(VnqJLj<)ZD(-+H5JS^8;AFmVY|!c z{_BOFYeoHz^`w^DI&>R1a)GLO)7XZbErAOl+nohc;#wA%gO-#D(ID5aMnfPq@%vrf zEyu(t1ol^HtsE6EnPdk8wxO~8TNvsg=Yrl4tu3w!+Kk*?WnfsU>+X`-*IuDKxgJ#! z=ien9p1C~75O%}UlM@m>8C&|QR>S>8$EevguQd4j@5|p@bn?{p+}$&5>GPr6-T;2% zE_&Ntc^i9fzZV$w(>e?xkG$Jt0jheq#RX!B3|s_V@@*iciR}8$sNB`+X zN!NMA`x?=1`q}kS+b%`xb(8S2Yh-QsYi?yS#qSw^x|Evr@K1=C9RKWxGfN($ws$b) zs3B^|@P)RB@BIx4OcWuX>kZD}po~$h7~Sm?nOnCI>q1Xh?!ESE$1or!C9M-NlN72+k# z*0?S72>2ap#%;4~t=GmZr<+uR3nc|2)qAi)At*a#ooRO+|K1v7uv&&y6w|<{tj(}< zkqOyK&JMc5Z{j!^SisMuipMiBZ6$obyg-qv4?k{a&o&#?95`OppP$F(a+W_!Jr*Kw z7=J$=pG_XQz6rcm>RBeb-=&x%EP2it5f-7Jd?V%3HmQ^?E!dBX;&EWSv| zvCKJg(&xlOlvzJm-u0gS9Dr?4sFG6Nc92BRt^ zW>M0*=Y341S9Cqp%hSSAC#QH0RXa7N21{|5f_KuoTjjkWO6lqHmRErd7}C&%bCj~i zdmV#t+qwb=cNY1Q(_m`mquDm}!34Fu zYdDvAXtcSVJRF_`lQXLpW4Em4?8@ZNyRFlQf}z;lRX@PqQr;%H?!iU`3!=h$AlQ+0db4tIg1TWLj*DP`_yL#aXodd6n14Dcv#A=Cc z8jW&abZz-Fu==~x68QDw;Bg+MtK&o_uxiI#vtu_J`ho}VZk|k6``C{8FfM@p280ej z8Yig}2z5~|^3w>tQA3!3Uo{4-Wc|6rn@aDS+E5&U?4k;3TKM@vGnSQn$#1S}iLo@8 zxq!MPM}%ANSR90ffc%n4#_DK&vV|3xB*s8DL;w_wmG+zFzwh{fc>*anRXz^ZqY(05 zq_lNFakXaaSD6d9VU8mYAbV|Nl;UJK)|DEJ!|vYt!Cb|G7GyimPE88B#OqgRjYghOxXCX~Oa zuM!L_ph#d1A!(0;2}hBl6DHvEE7sVd z#YNZe!ntj-3sg-$W)s@f;lWv9Ub@7IZY0U{ZkF`+im6CBo=w|NJ03=)5BndQ95~KN zV`M|;_l0{S@?Sb)Y-+_nGYTHO%1DzXGpi|eum5)SI5oloP01VYMvTVz(~d6%BGwqM z&p7?Oq(hf3dpyJW$5cWw?XD%I_`o;TzON#@@m~$CbCV)53GScMyPF<~6PUf4JYwgVd#q6R#LHm(*YWzy>+uKzD$g3Ok6_+Mo=Dd&DD>E**r z(f*&v5a4(?jiSm0Uw>%PI+(0hfR`+}NqDy=2oY5@UIxP((ibtA(WoQ>b4%WWi0*=< zG*qOWTzz)U=u=nVY4kkGx`$oNZA(^(R_c&u&t=Z6r zgh2-tfaJ;NjQ)%vJL=g-`xmjMIwU?FJGUoNqe^&d?sb-mr;YZZP4CZwPeT>adve#= zpJi=~iwv-IHW=JG^BjP~flw7N@QYVh6t7hJ*g~dh1 z@P=&K0Xr6M)WLppda?jPENNNK&#;F8GQveC;@>XE?X2w%#BG!Gbhu zgtkN)=@MSyHd2N#=e(4Hb-uMK0T09m=66IT{TtWQ*R4IbxSiQgt6)1{U6G`2S(ySB zCQp+#=4h=^H;uf=RpiM*t3{_IT_vpR5+e^ez29nSMt8}9l2x)SDSihcA|+v9nluN6 z6j{kZiR%x9l*wiALj^;uP<6)cCer{f8XlFa!LXaD`WnH8q-agd`(gAqLYcGNK&bPG zX>EG4Iy%@^$fWia!{ZF&H+T(L4TICa%xOERk!rdrI7^0u#}kkvVlQs54)!wmA ziFs6qDBX*g3enMvy4&409eN^1A(Vq!IQ*h$LP?DtJW5t&q1^*EotKz@O&YqST>-|G zNB8`C(Ri$3XDZ`pq0W&&E@Oma>&xFTDLnnrB}SbR^7V`fYQ)-ZtF_?Tj`wwe7>E^W z78smgBhiaE0;GiX$*da-Ss7vCp5eTEcjVvBLGX~BR;&6=ohc4i7FrwI^35PebK1sAZ?kQ$XHrrNx ziH}1~3U+EXcxEmS1qy}=SzHZzr2^U;>SIkm(!&vxysU6tbSJX1btw78Hh(y(ruXYL z?W=j4(5jWI;CfCof=f6@?{bxn>(cGe^I-CfJ3{s03OHfbrdD45$#4ZEq2=TNN=_{M zD<1Squ9sUoHk}XLSh4hJJ7Hsvn{iPGq<0`Z$LBtzR;(KDEo7mxkho6c*Wuy9mIZMt zzQ1Su;Ug(XQ<|-;l(nAr>#ZQUZ8q80e~fQS6k`B$q$(Kz1l>*7he z^H=rVQ$cYfQA+8oE!+O*;_qJ>4X4zybpUr8PhqoK*g8gA-@+T5)P zI1&nos3l{)&a(xzLMVh)~i{Rii)K)NeVT44={;7jLxBly|ZFS0kyn`R$CK?w6_S zsV&GZST2nFlZS*O8_8q)-?$B84*W*^65q*Q?H}&X>%JB|lFv6*8f>}NQq5NI(yLgn zEEr64M$b(zC-BEZShCZ*=DPk?6$;9ArTa0wd$cA9WXKo3pA>-U#No0eMGfv%fH7frWwvTM|_gTR;K4jmS+R zdhz0diJ4qXK^-59o96-6Xk>aqZ6E^@{Ri4J^R3pt^ZxU&N&G9RP5gK1QC=?j(=U?nLjp!aI?C zvAsw^KF2lF5c&6tRP&fGu9tN!v@TWS*_mIqjHNGIhV5nU_Eb-7aB6VXcOqm$ctXU% zAm5%Y^QtL7f!E3NvAQF3qI7;Xyv&Q74xFc-*V12%vFC2{?al5)-w!d9p#Lvq*#AQo z`z+}4#gK|Eocg>7hUnVe?%@7hD$CpFL(LKvO~>X@!|}yNixD6+KKz^5ALUX*1+N+& zRB)tQ?h*u}LeFK5In^NP1$l_2J57)lq;vo z4{%_3EYDjZo$1jOo_fw)+u#gnPon3L$o~RE|*4yDHoflkv)e)J0r_btfb<_ZIGfwJp2s&b=yAX?~nx!(@LT~4KUYz*^cZ7 z#L@1+W{F-quz@9R3fNGvXhxF^ZZIHE%WsyC2JJNZ1Pvt-$Sn%FsSY^%Ntet_l?^z_ zyGO-UTiai(E()T(<-z?-KZF>W%YZ{PdukCF<}4!Ax)l^&-_)|ax=9Ez z%8S80jh$!fMoN@nb1-aSEe8K=R3vLGuv_#_2bOCC&vO2ltCmeumvw_c}!ElVKEn-Q38{eO1>w z0s1);f1VaT*P<(U)$9imqYG_UAiH-Lq&DycYO1RcEfTft{X5vtL5GT zN@41Wu@`|2;lo>A`tlBjdJN+Xa|X7Rgw!?s`Q%<~r&8I79fEPhf`O?S2XftlFhY)X z2}A~!ytdLJ^)msP&5B`mS!uE+Wd99P#YZsh*2yu*gVdG7w&s;|5^>M zYL-ZpyD~Ff!)}#T1AUFhD-&lRx((~UC_Xka*ku`xnOe_XFwcd|4l+V*$8;obm0zUz ziza2V9=D$DBDofOZTXGtz;A~s)`Ii}10m@-pczydZ2L9%DqT^RzS6arEt*Sl$dM0A z311FA7~zO?or3RW%LH=F*g#S5hTY+LZmFBWo2(5N0S=mo*Klg2wL(x1yQ=%lzoD45 z8R?r1tKpT+b+@ycJ!`f7q2-#>nP>i0j55NjoUX;k(CHTllL~ZXe(y#_6s?R{+&8d+ zL_Tr~+50$I>O{Tiy+w^{6r;bg-p;kP8Iq5?XwvSv2)3<^NXQ%8E zmwLae79xoAR0dVqlIVryE*8;7L?u6epCD=|=`DEtV-QiM{D%mbUam%u8(tgj1u0h+ zA&sZ2)9bo69D?yz>u7Yh<{ao-Z{Lza|0GedB`@tA1GAFjDy$%!Ne-~-B2dPuRZ_dO z10w)p5tGG6fFddgJ)b9{Pve8Y{(S$KsA#-RR^e|X)KM5Fr3DfOK@kK2#M^qcOIerB z0nCx>%og3Pb^}OzFMx#yn~?_!Xn`wNo+;y9#3nF8S=`%1L{u14k1Wz3!&|QKrSO|P zx2!GSI2DKYVE8H0S7xv62}amb!uX=dVs&vhF*63^&_NtwHVVQDJ((1YScQ3CLZ=KJ z)R-=m*L0|d9w4O9qo_bLi-!QJn6O(^nI3XlFCYWWj9q8I=Ea#I`GoKViUPn`4e(Xq9Ygx(_L`x2rW0K-)d`Dla6>08;dS+ahn5$SB2j_3 zh4Q&qvfoOR3?xcR5}9aSf5OMq)_7DQrkIoh>b_d~-r3dTKPl-pP6NnU z;N>;fj{*insmI|A>gGnr#@ZUEuNi=q)0;s0Eas?sN_z~kQKar9>{vao;f-!CTFQk}_dZ_8&!R2~ zH~+nrk+ZVgDt&!lqx*4dv-`<~_0#@;7WYlFsfF$`RY^HC*;E@^qe&Dp<)FLKV%n>6 zT}|#TXXAru8OuVK1M{1X@ET<^AXHQOw0BE6hLxJD=V$vIc+hY#U2E~Y z)Z5S-9vU553`Yqs3AZC+vD5TLcYVC8I+{9C*k8Kq@v-=rTIXy*cCI>_Jd!^|Iv{D8 zTz6Qhs@15)sy(WstaCRrS(!Y)O0QzMGGj2EG$S0v9~EQi9G?I+fRaJ*CvzvfbMXM3 zm!y>LPCX&#Hhknl`hIIna*qOt;2C@D> z{OLc$-U6QjJ(w;dw<3IT+ECkm@DlN2j`c$s`_uV$fHr4C3teP~Qjq9EQ&v;_qWw@h z@O_zIG(nw~%(xMa;{FD2Za1g9c+pc|lb&`L=7!~_C-?ofVaa$-5(P;PQg?BZ_`u;U zctLuf_OHdMMED`hMa(eF4Bi zI$=5pyqEzz0Cy^s+fe&SU(Uy#^)eTQ@$59(NytOoBR>Q`Odr}Ox$F4t8AF-%#ZUW( z)ANB93k!v*P1ejzg0wF!#o)+(EwP>crKKi?NT#u#Tn;nSku5gUJgtjZTmTfDtQ>P( zU#!+uQ1~2Hu|bBP&r3gOV=n!qyZ*~ePjgj;9?xxt#kCupEU@vmY*Mt(BLqrggLg z&NvUt8M8RAr*D#J9UZMl7vb*=CE6;mUfQdMBrsezm$IghQ52mkmuw2E_}ruAlN+Uu zOX<1x6PucdV>e@0PJcjO4xM+X9sv?7uYYSV>NoKfuNjr;tTS;NUNvxaM9WXO%apj1 zZDlo{U}C>^m>~bDb0tOpahxJs*DJz*_&qu3vU26;?qD9)jABRew{XL09P`+Ir=Y@f zCi#_bqDStui)dy{f{rPUnf03b4N8hRRF@4iGD0&W0#zY2Zg$nln8m;yp?T`@=tb&1 zusZ$iGZt*mVZRjt87^J_VUE{RF8-`NSnVblSBw4&CaR5MOZ&qSnx9ieb8DEcAa$BCy+Z6-7D))N2N)b(71ixXOZs1IZE@JITWGIV#>l=<^YjvOGD!1{FU z;lul;SKt}_^(oTxCjARoaYX6qxnPQ$&!{V=vguA4B4|?zPAuwM}uL^toP78 zJr@7D#^ym5JNJ*M51 zEfk5%c9f6rkdOsA8xY8T=W*6A&G3}Qp>+%U=Uw+rNROU1IrEDaf_#UBlX1j!YTxl}&u16sIn{COmb@g7<^|(}hvd6Htz^2Tx!*wr-XxBHzK%7D~ zV-3x5>%_aJ*+r35bHr?Vx+6Q6+b%20WAq z_C5b@`}x~T)r4r}AMd;fL-8r0>=4wPoRiSZggVI6+tu=&uCd&80l4R8w0NytVW5qHn5`Et%e5NNBjmcWaDn}4lb+h3Lo z2!WfTkE!t`MVCCE{1dFBM$M3I$cGSquhO2PRBzvqs>lJ;(QZqd&#ytuYALh4_{4cY zrlu_A!`sJ!Zc;dbYyQNb)YLJq9<*YnvG)WBa0OT2a4u|`r?HWjU-FoyCH0<2nMc|_ zRp+<&uqp!s50kW-HPVJUgU%?3PFh90{Eeogz~Q&R#jsc5ZPJqTc&_Jwv(eP>Ui)Vj zoGufUV-)rk)?4??JNd`$D%onoSRzBd+1NTZzBBdD&&|#ysvd@~WRjS_a$CJpeysVz zp^P*t3~2;YNWtJlNY7{bLEtuq?a(26Z>MIW&6_L@63;CMtu&Uw|dVf4-S z2IVD-RM51*gHUjXWE5nn_I-UtbUZ0!Df~li&BQEbfkK9#`uI@2GvyYTof36FaXc!b z+!~p~rV6$Q`iiNn&A&bVSr@igv~Wz}$8j%c-j-MFm(=x-M&si-6n)>?7Zz1zXC$w> z`6P4&Lbi*?wIRlKs5d+Wp4~9B%>9u#m=48GF^^P|fVR%QU-lzML^@APZ#THWY@~4M zv`bq_bTssJIcr=Cve{!U2s`SgodY+)2Aq^?_vR4@2dfN%=~fsW&T24d`XMdfEWf}> z2(bp{H}*#s{z}Ym?12e{kYns?%fO}88#Qb4qcJ*H7G1AbSvRdC!})=s9ziRm+B1K_ zswr#Ak5CoACI?$T$UJ?DQ@0hK*K4Kg-WNd@UhwrSViZx_qV#@j3A4h7trGuz)iYP; z{=JRUxMI;W+L6)t*haFb{ELI3H=m>6hI|s$8l6={lfKasa#prxmd_!mB}o=~}@?(v_Hmk1qy+t&(eUe{5KJ$R8$tji7=m(an@{B0R%;>ueZ;5-=5 zUTFg1KqsG{bDj;S9q6tVa3!27qV10p)LupM3Y0E|iEavBHM2LoyU z0PdIJ1HWz|S0drl3BP2*GQeZ2>X{_LJemYHCJYX;mEH(jiJlIU0=&)yuQi&iXnjic zHu0WbSmg$;JOazeCGkzNzU11hR3=@n0OSzV3rCoTmeGQU%2BYamz3;+-gbE(lD5kU z5#TmhpRS0YPRN%M5%P%n99!=oX{06Zc>srUhO~Mgf5n$yM@Bx7YEOO=jCvw&OX$Ol zoG7og-#$)iM@&adZPWEa#PwPv!s`Y*M{PL+m zA(1RmYS4d*=eg>cF?I8?OVz9Iv;OOIGYe5z@BvW>6jZdP`~95xCRpK9`?XLdnV71< zATPlCyu5!$J4Df>ajD16rKJm7DMryjo1NH&QH4>3{+ab;zFQF4M4_Mc#cD0weKt1R zJr*=k+#uFkb85V)fkA zJNYEKCpqY{XMnZI*sktTS*_wx&BVXS=9lMaa$Z)0Jjf6e3O^NdkMS{LFy#=t!T+En zz1MJTxKTeK;MLjlZUIvQC)_5Uft>61FndrFc47^4u*?Lx@6$ z^@`da4;p^a>t3{9qI({p42XQ3uVr46K_fl9&{M_m!IX1>J&iqr(3jFjvrEvO*w@9G zvx3E&xKDKVyWQ!5?kbN+?Md2r9&{Ll*NA*^FLj@<9}TKx2eR+kZ_Lk1pOPbEBhl}S zAi>2?ncJR^`wz;5Sh9k5cZ$~+){&Yo%!1BtRVUGlz@>NF)mr~hWa{hWJ^!+M-qnRz zNma$wK>x(%S%(2A=a(w#6{vId)%HsL3ZkQMlNT*;Q`eYhCgq9s@}h(AbKz0^?c#~{ z#=(c^W%D*Wy6c!1rHfY|n0o`cxA|mHLu8EnBoIq*tB3iF!C~?^sNAV+JB|fx7|8JP zXV^Gy=>$E`Ht}nuR6exVfX-)4WRV#Nv|E?abZ~X7b-ePAmX0}R>iDlDw;}|ve5)-w zDA!#mUJUE2!te1TKh}LKUS-@Vcu+c~-&G&YAN4jOFT1eX+PZYwTc2!}Ec6`cIydf> zm(ZezFsA?;Yih^Z1_*R7ZOgm&FlR6McZrj-*+ddKXFmMSt#>~AlatUSg%N_Nr*Gc* zHFsV^Lj+s&?Ol#WJL|nQp%7oG6r6oqA&h`_EI;x$k0JS|%NXyzcq9c$8a|Ydl{M^- z_1C*Mi#vyjE~d|vNAatbXM(H9XXp;-j-L*hk4+D}M+J9FtNeFY4PC1rVz2RvY;5Sp z`H2_q4aKYBtN!V)#TT!Q3TvI|ZOaeFYm_cWYnxeXw%$&cH!b0ZmsY$i8n&|IkgF7j z8EdPDe%9}`_cCWT6$%XkvoAK+YugJ!k6< zJjp!Vul*FL({4k=iz0~3w=xNCb<^I<4$jZt>jqXLR|Z$k0+AuMn%W7uu$otIg^1BS zD$Ix-I}2x>T@KsQAIZuyYKEe9b5nyo%w2(!m{r*tN2#eZOtcLd_iXXux ziUZ}PJSV9>P@*-nt0Ap2X9y`h=6Vc@gzxzbK!gYlRc7bi!G{- zZ9%{gaX;Bh+BKB1XAW(Oj_7;pAWy*K5~`{zBW39&zyeLWaag^P^ia*!`HV)FFwmy0 zi{58il%cVs;_OOvORMcg%0T9U@!Vs+2o?66<#hkLn-$fg%x~)3}MJzRE z-a3bE9S6$h9%`fr$yeyO>sw`=1Hi>9*l%Fweb~R|&s!SGV&jS-%=I+iN^WcL#BI2O zbN{v-Sdo^VCtw#a9bc}6C3~U^$pWu4$C=^yn-`FdY)lmq_RF(ZMVritD%N=s)g8e` zMk|@DzrbBzaR?o_h`T6XA#0+{b1uOs2{78JIo}pFspCQS1fg|DS$3>x^rW!9nDZZi zzyU#AgEiq>^Ow9Quth;qsH=u+A&Y7`zc5hAAP(%hh2PjsGpo4@pVcU6ZIswk&_ zCc}x4Jx5pe7Mx$zl64uF2f6+X4*iw(M##0CBaXji)kjcNu!LRF8`oLvLC(<0Z+(cz zY=a=3-UBnp>2L%1k;SB#CwL4GQ6dIuDP_RUw0wFA(#NJI(5Lhj^1MVtBSD+Xz)qzm z{f)26Yq}?+_wCp1&qti3!Ku0-y1+HZF+yKPjX1h`aEY4WNP-3$s#LMOsR|B;KMA4E z^Gz^x77zgWL0ZG7Y|61kN3Qe2U#niE9q7)rk)p{DHcIvExUzd3*2f2d{{d+u9Tb}= zQr76IZ>RMKp(-v{e?^46d+8prFf{ zJ0<+ed(@Gd(H)_>0uJmaEpd?=y^1m;s8p)lb7QE2l}T>-lNglOh*6ANKUoZ|Nz$Z( z`UQj^oi!b39Fjvew&oE(1*g0UY~pu3M2fqyC2?T9c~7olQ2miltH{4IHL{!28i*ee zX_Nh>P%g`~y+2K^q!yn=O^5Up8W5AwAx8+=aBz^#*&OsG9U`N|>8mY4$`j)AO^erR zjG^zXI_9MXz0(}Cw2Fg+%2srVSh3j0T5!ln%OlY=DJaWm%J7x2XYhC8GPtdPw@yvc zz~zK+hNE+A6dnlF@m24_@9)HYsH(xGVaCHb4YJMLY;mRSp}e~ zkkjHQ~z-w5l8$f{_ zQ>2|Xfe6A(Qj=Uc4?K3VA+jdwdqHAK@$tmzOQ3Xg76<1ASwagnA=yL)p^bWHy3POn z>oh4%fC5`>Pds9fcMLX6iH)_*SKY*614d46@E3NYUne(Zq0t&*a@8zI7EGX=gu?g&#*{{~~GM9{&gEZS6LeSpk+s)!Gc*>bcNv%318Y-Wdr=E4o0oNxMDk$>v&5+g$x)c+#D!d3Tt^rCz%-%{|VLoOA=y z1T6*Dz|uCk9r=(QTfV!%-f`0lI-M4Bh>@Rtu0@%g%hTwWguILnm%QnIOCBNI0LN7r zinX0!I?nI-Nwc!Gs(!e>YfIi<+utmx4dTt#p1)h}b=qJi-TM0(S{Br}7tUYcnGX&p zt+09{7QnfbrI+*rz$T=v`=Hg8bh!gYyMObDw1^Y#Q!=mp<7xK~U@j+}w{bKiU!u|* zeX3@=oyE{ykB7)?vb7oTf<}5)xOUFuwkRWRWAZlA>AKJS_kFO9-@-;b&beQ}`km)D z^u0Ra>6|QI!O?KM7pN0z{UjLecZ%jrli+uG{tT-U^1Pr23H zsS=q|<@c3w;fi7>T!#4(E+Gd|7aXj|;P%npBly%M^@GQYcEzEI4@-U1l}WvEu808S zjh~l|GFGAPv^c5#fj*uiaW^^k8{rz~!T_&)X(YGc$` zQNy3ldV~MotAS%_A>{<72v$C!GmNi>2vrkkh})qlwUBX$QkMO24HnJ>0JW(Paupwo2%DI}2Y;q* zCO(M=W}>;NFqf=oEtm*M2(6wWKx)F>npmsks)H^z&O^3iwRS9nI;_m>RRJe@Y@)I_ zPHI`JMh?I1=FM^?{KaZhwOb(%bR z3%OBY|8C4ixM*Y&e$(GUxiEI%ClU(T73%sP}la`7R2ip-GrK{qC4TyowzfcWXS&q0w3v5H~rJCCL9b# zq&9+zt4X>xRJJnJw%8lKFq&z3)PKHBw>mHeax^+S7_G@I`z>6F9p))#mvj{jtb^@D zJ~=vjwyK1H@Im`I-1I2obJ%Me_^K7HyGQRdfp)YhpHSGGn7U(=kG~`ed1-;k-6QLB z(l}_pvwEjqiD|!=)R}$7fqxitKw!8_&nR;-i_%f)z|2Jw7ZnRGRcf_!qeqDjT~XgG zRCcW18u={aYa;KGrdcy^MVym~RKD4?Y*AgLZe=?H>qda1HdWkq-=_Q1fjqaw`PQ)l z{A_)1t+?@D1EN#1>D+8484?UqTnmarESGgG`WNRQqXX&s*F;0c}h2`FwYl9l>;X<7c9-WpR zFY19?8xy+A#5-nk_aBO=q&E@3xthP4=KR=o$XO3T)A4Zp@?RP?EeTL1C|#{061L0) zf6$+ai=&X#sqIZ@QWlzxhHrX5ol{ zg>bU`%ey}rlhw^=N+9tNd;oKjI2xM?nXDzTEY@J(JZW*-;KG8M9+?7z7VL~0YeoEg zA2NhvguY41kNs;Pfrx8a(PW23`>G91f6r>Dk4zVCn`kLY?u50ZYugtbXQZDJo2* zMNZ`y+`hm59C-{0->b7&hpe1H*}}qC*|AtNEy`Mh6-%F)WtqtgvD5ecF0YG`pm_~^ z+tCmGN#Q*XEKSx@GV|4Cu`kK1(Yc+@qko7@xXokD)CXeFGP^{H4rLC^YjUb&Qcb4B zs>H6%id!D7MnKndHglz$=^QSYD zTjYmv6r8SEw^D5V4T$Y8@Bx?Bfe(y8_z(zu&{(4@k;66#{+HASFtw%-o+rGIBTNr= zJu~GHm($Y0Vv24901jjXahT&QG)yo4-k6lWx8XFB@tnfTFp0g;koFs6PFI%4EWsfo zoR8twy@_nZw0K(GKDsg!9I5S1bFawH_ys7ZyMd6BWPsy&7>QyO2YCdPB}q95_UF11+IazQL<&+dWL!k_V>PWiSwDw z-`Y_v0|4REGPUl#bS^Jnq=PU34o7UYWYy{Jc_f1+q=*>j>U+yOpWo^kLkY zz2&;WSEB!$as8&h*4;LVJ8FUI6F6bdPxqGsJoo0yfPpl?^Gy3ov2PGgGL1EUP#C?oDhhaCcb+u0jh`|HlV4SqQ zms!q0)mny*;jwCCI#1I}Yk`EHXqT9+E$$iW@V)$7*)FN+hibis_k+;2bK=G0tp1bU zph!YE&u+yFpuc2{D^Qi+ln^k!)PLYbU-EkKk0Zpu;+QQT7$f5pWZ%~d{kCDZ3Qw&m ziZGT5Jab;8O~1v&U&MD*H=mO|L+J%)`Vq&s!;3b65oIyA2$3(MfuAHUlXO{e-0U!d z*h))c86>o8W6H>ulm6s+AHCqNVW7|7EpT1Mcr>I%2y{cIq0i!LNGqp?0&;PmM;BJgE z!E|$LTzyQtwLx8GX+oQHxt-0(6{S(|5D{0I>$)EJo){pXk;8gt22F^>)SNr)9iwgDjtayJY>d zI+Of(I+6u4VhUepzjmzUJ953)@5N5VP7P_&PUYTn-Y7iTJ(-{7ZcT43SrS0xpK`jg zy3)E*eDPcBO&Z;l7e^PXjGx*s-C z@2#MNS#Q}Y2L6Wn2EdZCy_Hv~)rkaX>^S*YcE)7}ZKhyG6+p6z{_gre090+hCJ)=2 zV`Ez|uy_hFlM{JYs2Tp%oi>6Gb1KJ6MVckQ=q9i3krs*>?3}Y{}sK; zMBr?GD5V*@e$FKP=D3CXpYnuBO~d#BOp(1`}X)7 zenyFwgS8i}je>1QfQjH`dQdcT-(OoTlLhQE7ztKU3mtUK z$;NK(VYSr8ZDIq=Sm{1;f&k8bM6~EH?H&e^sg#PlQx6<*p0V3L&628$OBRD1a4AdQ0H&w9z#rN#%VTPpfYN_x)4V z4D()?SoWiY>ucsI3JK3(;)kLb{T@J##2QveV4xMEE){l4@40ggb{J0YgUA$Z;TKKe z%g)GKdeifhYwdMUW-OW34`L3LK1TIkB?Cft17lcOMtURsZt}#d|3*8Bw0U9HSZ6wM z?RXtrA|t>FS~nvAb39ZgGhfy-!VA@c#agpkk1L9OgvkM>dHT-Ll;yxu2RHJ|gnL8X zL|Ab*PKKTZQnA3|#W{o^AG^)Nvn=2~OakcvEQXIs*oh$Q@1YW*|ynrZrpzF=zdr~U`1qP zWPUm47?{GDovb<3rOUg-K}ruSUM ztn6x-!?dTD^b4EjNOp6$&iEC$nsAN@_(ztaX~94B`Frl9(^YGp_fUEX2+MoDo8{J**xr9e;G44U83wNyoTpG{-vnO06Am0 zw&-uolPg}iB{7Q*Ngn*^$Tp{m`@Im*Eg``eJu3eV4E}+BBEb~aEJamoN9Y*vOLx?S z0ps({Z#3;U5+l#pu{<8sLd(5L!GD~&E+$r2stj% z`3c5Z1+0%EvJ!n>H0cL=DGAtJ%ARlS9Ts;y5sHX+B0p(Wdhb zG%vAj9J{6E@QhPJILz=xsv`wV(dn9w7on-)GBBJUJ>y8_&3gJ7`jqp>v>w$bJ{u59 zTTrWBUAP`Ln-Hh!Zo&=}qa(h?p&8fLxnoO`G@%npJHLulAI42F)T?c!Ho|3XQk#yI@ruCWl(8 zg5lsfTijGgW(Zr!Uy54zI@tnd411!M(STvd3Iy`!4-!E#LmfvhcyM>HLqrgDBnjs| ztK1lF#jb0sKGoTUGd8#cW|pB!c^&P?GL8qX#@8{&nnBXabNCYFP zzE`FC9w~AE0ShOMnIwcyS9ln+$=OiHO-wxp%eb*wLEGKvYVPkipxp?+5n4B}zrNjl z%I4AhDa)yK#Bo+#$TNTormnoI)sLI)wjii8vAH3rHxbZ}Fn;O!K`UXlG7gMnte1Vt z=wK@@sE~jsM%vUZ^1+@9Moq$gBt}TuY=*xW?MNj1VaR@-nUXxS5+d``W+ae@zWHAG zZGVqepbyCj7H_iIf40AW)zFNU!+c=VwfZFTtUrVd3p0%Dv{P<(3(aaud&mqSnYsXE-{VaLTc=r zNEou>sZv@7chdAc=|aFK|FwSdbK4ug;~+#hkcm$tuX2^s9$rW6X3}-`4di5T(Hko- z8`~(~rmrZ9xXg8gY!Wxo9-3kby4)Y zl480P)C|H_zH!}1GVE51be3Z#G3iIFtz(4JNmWCtP8fsB~c)jgDn0DO>pnF$(LP%O+8^$X-B>#Y1Yj3jqZ&WWHF! zKI{!}X&zrFaq~WCI6`c#iEqdlw3HFv+^lkv6Nuzun7NAP@B(roaBlU7i2B6TY`i@` zoy+iQHKgoY@KJYcWou5#DkiW;QcC3bf+|MF@{wb~Poc*}o2QRE9 z*QVFV@F{X>b})Dr9Mb3JKBo9-ad%QUt zKbKyK55h;l(~1xLQF^GlbUkVusvPR(*zxFG(7tg#{4NLU^&e{EbMZfT{5gM&dYIhU z{9R~RpAC94@F(J9>g#)*SDpA{{kY!kuB+$Ko#S>0+_S>N{bH}~)E=2GtL@&m7eF;$+a7@rRY{(_gVJWO`I z?hg*TflC8hljunXf%5`)-e*bTN#jY3CH3#Mex`@OztBHw$$KQa@;!QfmWP-lbHA^K zz9O54fWHbu=W^RfJ7z9g*@H_&#Wd(C_} zo!**ZkDa(e&Mmc^H@~O*_$IGjGwpw)%bi zpGV5Xv^dNAwYr);tS;)a_b&RFz8{@2FV3>=@atx6TRzM0Y~@(lWqmuIFIGw`15f*3 z`CFY|eBIpmJN}GUj=uPJ=x^-4zx3W~t_(HkZq5{_!+J4@ZD2Vl_DKqRpXj!w$kkOl z2xExbB#k$$#7;&$b!$GoLdb0-iG_XQ2B&TtMW69vu8M&M)G6g{86Pfh4Ia0e$e@~1 zf5}u|cg^@gmt$;#GAxq09|KSt*zUqCl<~1BjtR572Az?&PoU`7v_5t&SEz;8QM)_H zcgf6!7!iblVecbkkK^o^8{U#tM2cc>B)u>h1Sg64yHGMZ4$?KLYQy~wK`V>>y|m#J z)$nBnts~@M#TCo^q|_Y+K#%K}->6w}^@eltHqd0iZe>Tw*l9A0=A*DH!Br2k5%>#A31$f05jiX6PFcY!Pi=xmI#aPPNIu-B zNXVUyM8&oySmW%mZH?U$z9_4UO?BnNUO9;a21+b=Ke(E5)s@Rwa$PmtzHU_t#?vwznd9A+M51Ca-#_LyHZC(0p)(CWnY(M-Crc)xD!{Ilw*du%~QaoQPq)^ zAS(QoX*ZHnaq=#QXW@WPBlbFD5@JB7Y^KfG1gO6*)oh59vMFR%$i*1sG|ZX`8i6nb zcj)UDHv*fFt+NYZCL2D@(+g<@NyzhK^{AI(+6HWWk1C9%hi#M~a#DjlziGve+4yQc zE#6zPw2V$?36mzai7I7iwa8g1N}j4lHyVKJGz_tT?KA`#hwpsq>WEge!=Q67=6yyC z>$B*p@};X=RXt z4@A`_=dnsf;PEnm$Fay%d|`WwTPm<@#5Y7Z`<>b#f2Xc2?khM&%k5Q-`oSKNh~=BG zpsDw>&si?Y!KyBikyDX~eOhj#)cF;rfQ+eZtV5AEgflvmD+qy>0|f(SD$VFsKx+I0 zZdBynF9~Qzv%35{WTUd7J5p8rGe!^JU4L3fW66noRJ+yZ2a@WUR2K0Z^6D`Ph#RfKv}E*PB#$0VYYRji{~@R&_Z zA-;8d+d9iC#={w*#CH0#VRjes!M$S{h7V~AwuxZ?JBZYIv)Qaf70Yt=5qKQ{EHt=8 zbIW^sOYp>9LPU1jZR}i;e#FXpaXdxX`iw?d%#NnOl`t|Y5;PormTMsJl7+!@Y3Zm{ zXcZe;+}L$L2Wi<7ViFtnlF{g>Z>sY}GQ)ItH^#fu2|aTI1_xLKJDbjy`mf_q_v)@{ z>m?)6Zg$t61QT$Qvlxng(FYb87wOOH#GGBjz!<19{d>J#DgHDrKYo6z^2U_c_r4Wga{dC4K-4U2L zQw;%!x63+SgbEC}EYHcd*XB92leU5n2i?h1R)3yzS4q2l!DiYjKh7`F67Yhhz{2wR zsrod(M?2Vj=J!Zl%HHB~Nw*J{+L_q0MnpqqAdBJrA=R7Vn$x{xFYvrUlx@{W zKV`#d<+cn*8D%+9v0igpc86|)0;BVb(gX|9?BfAM6mS{Xh~8+ZffVp2*giIG*}@%- z-qu^nj~Y7#o??nK_ggiQ0?e5YYT&Kchl03;m`v+g+p6cTIlijX?f%&wN-$e=q#^*A zqM`^?UWqFiLGoi_M_1^QuiZ_9?N$VV*AyHj zhXG;%^EjPA)}(i?Z(dKe6?7Fwbf%Ch$BAV=5;$0xv-lL#CKO!`w9a8BRt;MyJ4-_! zJ}Q!`v&fwRhZscF{JOTV&2l|<;60|-SMIP!V7?R_%{oYFqT>lbW9l-6kV8EWfRTD# z4~4yXQ)*7PyqZgNNDd3r`KNd*$a!mH$YcY4@Pg@_Dihr@==Zf$CrRnEV_w7xO>WPS zTxaTEN1*$z#9uCZyCSPj6L=Vu2d+l(AlDG8cymFcIaA%}7nvpGV}KO-%mpc^-g$d2 zr6>xX35S-}tkgEJt$-Iv4#H)K!n-ieh#)sMl^Db=Y<6o()b1s9Pnhr(m(TJ<1@ZDc zKsuJrV_hiYL{yvt;VCa}HF0708;0T~2JM=*EGZn|hNi`kkJ_AaPH*~9jRfW&RvcBr z7eMRBk_1lDVl6OV=pq68spS~liG%FF_EvCFdM6NJe=^-tuN@rTC5%x*PXLq!Zg>6#yqX&T|U&7#+^y zV(f+L_20r&2q>`6#D;FeFwJllvP1w8Z(K%>0wxJ_60ct5vj&dsnCXaM%L|3TUhS8> zha83mpv+b1j!HTC%c*|zrFohdo`V?5u$Gg{&9k8qf%yE2o`)S;8O-*4P!gDNpg0a` zjFt7eH_j1MH|2}{)3J3m@?>0$_JnmuOG821)W-ss9YrQVc(tX?I6E0}TC)TB&(bMI++4efBBC!#+y!2B2GXob&?Bnc}yZ zawX0>#TMg27s?=n2KF#EG^iytJFXW!m>-9NMJA&+p73(XLGetWYWWKpTV?Evg0sp;sHS znpF&buuELF?wp=L^jL-|&ihX)QWW7nBh)HAwYd3D={;|O;w`^GgTX(ltCFz8VZ?z4 zEcLA?#wIZ4+bo2VB<|9tJ(haQSiodZ9@~bYN8>6+@jhP)>9X_`BJWPM5M|ru0-9K0 zu#4N#>gF)iiu@06Q3o#3zvoJrD83W*12nI`Ga~o0nRWYewaDb>p!7D& zsT%~1x-g?KkZFxgoQv_|)oO&hDuKpgt0rYCoX1>7?b=FCxb?#x3;?Fx!!#y$N)nDOK0MOlYc zQAgTZ^{2Qf4?>DdSreq_MaPJ4*dI&AyIQ;AD%fGg)fz1`LWlLn zA&Rn0KG5K!#fFuLDmaPm*biY6r`Ya&f6WUjV}jA@pq9zY6|3+?(@GPnI^qVoTF)cI zrAHyz66fFqx#PqK)dV&jc!x5NR`3&GX@&vIt_q@*rA4H5068axgB#(rnd;Y-QYcc^_8H?(}qTnZm|&+8X57ZR7brK`UGcf^@@ z_5X3_dFQ7r-$yoYlOsL-b#H~|P-Jm+>F9tm!CPiC$b4D%jfEkcWx!rE$T0Cd`>8=J zkvN)+EgP*(ZiV7r>}<%G2o}oPIF?E9u4XD5^vw@D*%2*THp2g@4(x;5>z)vU0$sXZ zLyCuXutOu*k4kL&AYK0B7LQenos0v{+`7 z7#VF$Wx1p+-ERH-ts)d*U3yR}LESK=}BU*te*X8BsamiN7>@Y3kgaw+aoV^aOI{8W9l9=+cI z%H`AnGk$GXPdT=9sXv%)Qk;H zylks%FNZhzpTwh!@A{*WBka&xXna&U^qyPzH+{NKzBiS<&hc3?cPMo-{d}K_+s8vM zLe&g98+kq!PZK6~%$&>YPxJkg$e98CH)~{da=rZD($>i*F|z(i?eR@^TZ%ttFCSvI z2a`XSzrCN)0CW9sfr=;V$MN&;-QNS_H|c}C&*%gHW4|fiL~izPzw5(S>~`-F_!#j~ z@saG&{K?$$t$gm3|K?3$f3SYoK4;g%k8WdquzW16B^#HI;$d*Jpy6U;cP2f>d@;U% zono(_G{&3Do83&;nDXPtnu#~4i|J$g+ASsf`ZO_lty(y;@NhA_i6>g% zcIA1pIg>pMj-@6dXL|qQ6I#!hiN(c7#PnqOv%kGxPxhG4q<-zWjJQ%SrM+0+1{tOH zC4AD|oS%BQ%*8NQVE`M+R;E&~B!#~46?OlJd`}EpD_pKetkL=xf zG20n`-MVXjh4xK+ZSM)>h~A0dU3ihbJXq`YwI8J$%8RRh5q-yt=0%|%4tpo6FYDRy z?!Cc{WF6(Z_;Njyx|!_Wp~=1Y@;FP}o|Jxh?CEM+!9K%1|5|(rzJcCJmM%}+vwXiybyco25}h+d21Z|CkxxEV56e{YLWq0b%cgJ-zc{5i#WwT9tfzYy`7!?EHmm=Vb_V=3dKckMA5S*^ zEz?cy)lRej!ngJ5*0a4(ZHqsWSId{oNBwj0Ri>N&2Y2Dj^QsX0KR+vPIv1`-gVV{D zmw$9ExUk)GZY|XaT=Qs9Tgz$|(9Lf`G%Rx~YtP;n?$!BC-&`+t7p=~GhkB-*ywP$Ai zmhbf@vuVlA^X=Q+pLHh9CcA0Dwc%#S%)T1u(`MFtfsJWxY1y-D+5Oi)m3Ie^tplT{ z@j1F%gy+uh3|24MP!c>ZOzVfve236 z*#Jn!2@@>F54V9Wa)i+#NJa?BYWeL2`KjT^5RmjUxfL3uYfeM?FkX|Dh#21 zqTEgJQ7oIhk|xTXKEq)XXsPYLm_VJ~0Dh9<>mhZL?Jm-NE*pwdRQabGwix|D!m7A?VkEXF4xC??c;IHXV3FewRF&gZ1r{d;@WlH%P&w*fvvM{q83j+zl zHy|j%bFaw|>dd1-fVvuD#G$gll~{tJ^ohFzBLkLF2i+<(el@f282~*PL7ncN%KB6mV))8-swQ(I(RT$ zR#IliQ+?nB*MBJ>gqy44{)7dB+ z$84KsU^saGX^+IfJ@Y-5i&URTVel}GQc^&fFQpRkp1!i+lD-41_wkAEeJ0HO$x?$w zsOm?a2GznPt3u;2$xb=6g*X7%GMog=7|PRUl5;p43^*j+A(r9Cts@8_XXSu~m+!tH z4l<`S50pnwa0(f7Zm=agMHBz-HUss6#RM%H?Lbf=*~(QW2k_g5NV=0EnRkh=sE109 zLsDl66m;>7+Fpn^-1{H5cVw?oA zj$85%V`uM8F-@lS+Q2K&9F)*(IfrZ?Y63M%-ewU=}T~ATkN(OvEwE z@Dc#S;FA2h-~~g?K{kw(7*QEyM7>4FU*R%D4}m2$f;=L8;0|_~E`^=PKG06Sk;gba zQJFB&l_Iv=4!;1dz|;r|HrOAai;9^8h2Z3{34mWc zC}T|h@ERO87kkN&{0y}8al67+%;2;2ZLsEk!3 z;P0mys_5<%OIq7nxTI~u@DCdS+kk;d1p6QF<2u3l(Rw+_3BG-LyVa`i2r3Si(dWB@ zVhwE06Qre&eF#b!MUvyWiZYcXkL&QMt1vK4O&Y+3PD1}$lrbb?@ll4*f&kIIEjjl_ zt2PW6xZwAKLJqIkbmTm;KLU-l#@5$iPzsf zKk9X~ntBBa*KJ@*7(}dMBe_J{{{HI|C85$^!Tw(xHEUN%IZ5A5j^Bp+3R(;TIRd}6w+ zS*f1T=L(DhnQB*HpGPMINX|SdP~Z%zW42^7m}nzaVeC+Atn&*@#de*H7oePkFm_2lIh_a?!4(M9 zQZ(}vF#u>iSuvtFIwm*Bm7wE8`(&yTH|(v}o088BXARpTENj2AZM|@awxyKwsDfaJ-Lgdgne^7 z=6F<~@j64C_dq&Uohp@EzhZmSh7))3zay6&`wZ?%GVzAy+)7Gm>9DjZo=Hs%3=C#_ zC`P)vtE^bVcr0yt4Af)N=g1e8XJtC?!ZMJ`N_E9QiHmD6LHXZq=kMEhR$iKl1G371*AEF zB7c*i%c)8qY}6%JDNujFGrWe&u?<7sLNNh^JLKL0vX5ZFAxX^t_3zdjX-Kec3QAeh z4`%h3;5Dm)J&G7d2^b7LfB?n#fyBeRAr;n=(19t48NV9|3LZ;xlv)A-w!U`tlX$56+O49CLe z5EIoJxksAX2w}8Q5UjXUT zrp>se{JtB0mJd-MxXKlv9XMtN9N9qTnz3cYK3;sRN&8fWLotiENj3y4CV}bLV+gmX zL?itvqNlX`#m1JXq|mM(GZqC7)Z0S2u_fgPpN17X0bI}K2ynuHCrXnY)*T;)Cxlwx z0TgeXu>|2aMuUgF*3Ho<_#FF>(CA?KHcw9lD6pFS$K;L78eJxyi_$Ij%fvon5UC&o zOW~h_l7xN6z);?qpbRFQvRw`(=!DVEZ{!xi5dZEl5OT;bb8Y!jSmBNtffIu}!6L2+ z0^9@R8Nw5Wk=Ce@biqt6$D%E`P*eK&P_U%9q^FH5Bak?HAR(H!!V*N(FIZeg5Wkhz zPtN@|6>4*mBW7dv*l-G5;$xj9p}7$Cg_+t7ZdG!J*~p#AJUGxd3K|SN zyg+dEKL_Cz?-V=`%cdf~MTql)U%<>ToWsPd0~mjwr62gYIu*Rnvh;h_#2{Bh4!a}} zHG30+gd8Ts1NZInC%C2V{$^JEz;5}69ihs>gu*K$-iDPFE-X_o>U+xGlr$5lCocBVD)+gElk(^2eYkHVn-ZN}(hV zGWt1@Y@ts0p$f6+l2fN2cF-IOU8}+ZnJ)1b&Ty{h>jmd9gXDub93N zu~~yCvU^L#hCMZ?qm5BFCovJS2dXTaSdf)$S!0i({W`cNLXR?d!QnX8Q?EIYg7Pys zU%52~C$zXx=7-&8c9_MaI;G83q0{tK?`lvTI%vgH-@_qOqQ9*Oupl7B_TdEG@T~n} znf@s)6mERjD~zmCp6|#uHb$()Mc`~&A1$UQQ=ktg!u z4dmu=Z=mmu7+fMEIkFLm7Qka={NxErrK;piNs0FH4Fk*l+UQgCQ(UZS*@%jRqR&(- z)D4Mb}2)HozH1Dj88UcLjsEL z2w1SAxynv6GPsLK-GB&HDH{4%*&775ZgtzDFb;f@Hznr3Y*Old9pO{mogHhUMTf`)3h#J4s9+CEOmH?O*u*y?Bd5n9!YY%A zUrlUq^*~N<#Z{&% zfrrC1Cpm%R>_8yav;IH@k}S13ZdO%_C(emD7ho3*3z!c}a6_)o5QIX0gsyfG6^YGo z-bZ9bj7HlsG4F@Kp~W5C1&)y6r90F(m5@u3zaVMgp&^bvPmV3|IdN&g7zD+z*PuZ# zyfK%cmO9_jiSvlLbDPTinMdqIVtYfVp&fBATc8>;Gq_K12KzmDoFru}oA*?0Io&Q7 zgez{k7eWc7y{ncly@5ws*5&(I>@N%I7-U@RU|gDLeT|K4@{C!4R6>?0Q5;!<*w)o9 za=lVb(-T1{A_}%mD}cv%E&?u`RB=f|Jr&`yM4**26V(W@NWl1zIpWNp3y+k-2i)E_ z?~|-vQg2e>2o+09`}zX34(poNd&>H(Xk^_G9AmWi0?vi#h=DMA|L{>Y$mzv zrGeS8dbbV3I6O65t!y(DrILKu08B>CarL+p#9gt-?4}8}L{EYB_xy=Kh2&~pGccI7 zBdm*Q(=!5%ENtP3jDxLo4EVBFJi!^Q7O+cjCIE5hht@EiU-G=#Uct1M-_-kLq*Xrf= zl(^if?)-Q0Z|+}qY5uexDqjt+;zyk`nKO~IyoEreCKeSdny;?=rwMUt`|7SHC$poW z30-Q@M9|&*{rF+}XnZt!RDHTUdONgu6zBi%wTL~cO|PZX*Yopyt30|o3QvQl5l;(G z!%p|7_R)TAawxqO+Z!HeA28eN83+_AVNkH3`b_&F`Vst|{(N|Iev|sX{?y{j!5?Ek z_Mi04|DbuXyKdf8ZeMH%ce-d-u_Ibd`QZIca3^2>TUT$8H*0NpF{^EV` zJzo6v=wfG{Fqtzuw0;YaWt`1?Cif(Mvwc~JiV zOJY;epQ{7cm%_`u2?18bcitb(p2n~9wX&`F`?q&8ERJ@g%~OY=oulzc_l^3-dGC67 zFvgt-o%oQrnCMLmPCP%RQNsCK0ELIvr}3$Cu-rQeeSpq}exb+F8l!nd{~e=;)rS8X zGw^#b{3t(7Jh(p$cktlCZ-;mrTmTPc{?zY_!L~-6V zjIQiJ>86L*Mtu45B~PhN7h(c64`*c@BmwmO)?esfdfLsf6k=4ktB7B3p864`_ z@TE>hyGW6AcC;m3?u5Sy>L`UNA31RD5m08U+GaOcx)Q6B2eVd}E+iP%8z!`*DMk9g zkn}s~fr1(;0tN0>uM=LP@#q3GVNFeej`zJ1`8nl zL0`d%gqCvXKLMzuM?aeNN7^x{x#rII%#VP z4!OELNKdyQD#1lWGCv&OJ9H1pGeI61SiruTQeU>*0y%G4q8#s_J z-xu1oFFs0Q^G!llv(FF+K<0MjUp$v$3f#qs?dnz`qs2%(yd5hQ+1~jFxg1go#qjQh zTt_n#*6*K<@-jA#P266>)}v-y@DS?Oj=KsbBkQDo6cWuC?42`KYJdZKV1g5Aqn1#T z{Ec-AEpyyVaI4(r1p>tw_K}O=St!Ammg7-7#xHmglE=7C>qHKc@kz zstrOFeN9dsS}DlM!o>rPgV^#==8ZWSZ3)KDbL6&=fh)=WVZyTyDEbdLM^z8Q2X7e^4bo`%#0a#GlGpN3<)*PZZM!+E~D@|oIK17 z*#tF1cNZfWU7!>o4r7gr=;)RyX#&XwC+ct#qk$>}U9tsLa)#7)ZW3icr07Ic5RJax zsmi~}YQ2qKrz8(TY|z#3(j~DrYC2hIl55p$A^E)m6QxK4>(Yz*qX$Co>eCX$^`~rCZM^?@=3U4U*Ud(KtPv7 zp5Y|a`mi~K-BM(YL@#PVX(*a02+hBeKC_3@&^OyYO`QfG*jz%Vc_M?Z8r20V_aJ94 z#MpCb1pT51Gn)V82}6=*1!X{bcPQ~2k&|lITFC4O#t+iLVIuu7CPkSL>wI)s20P3NA|KL zJuJI9OoI8}5uDgVC3|CxgM9~t2!Em_LEnjTg+AtW!RWop&W$-y?uBg|JOCmb>v(1A zcw`eH39>9%dQzon=+g@Naeps*WS?P`0TKK@=1hb;7`WrK5Xsj=6G{X@3t?u}6w1n+ z+~jbk4DnGt0i$U3poffrkb|p)*@(Tvd42mJAPAoQ7~(8t^=n5to@t5MlHS zlh!=IbU-OQL-t9Eq&f1yW{a|g7*EZrKtsIg5OgV_40|HL>P=y(%K(D1lt-=5W7B7( zxLUs>C>^)dhTI)5%B}^MInmA-2POH#4@&_KTNmveLe7qwF%6-L40bhRSxWIR>-!Ny z25Iig#?&7PAqX>Q?~Y6ps^4O4k|k3zik?Qq-n!%;lC3bX0>~$95|LWR;3ULsCP}DR zawJEqdQWZ01PXs{OKJW_0fHrRRC$2NkSMjUyAqGQr0Y;e3mKZC6Um)GzFn`=mopiE zqq?alpk~oc385+VaJ1b!(#(FnGdU$gqu0#v)Yvei5M}Suk(Zzpcdpv;? zfZ}&ZM$-1M&8cYAwhBZf>~4INxO<*r1)H&e7LbcjX{xdWz65E&W;jU1 z+k=6Y36k$at7AQuGLOE!@7L$+%TLQ|Ry*hXrT9|#X?oPWs=pP!Migq;lyAK^o1=jV zUut0LSZekteRO^*J*plJpDqnbJj${tWKsFM@+126TAJRL&)0k3N#aR;^cple)L#gx zUIofE%^}nxsQuo*Pwls99~S-;^EZ5F{FCj8?i_EW&0YSX+|~Nc+4YwO^Tx_cjWknx zlb&&(g!T+i#s`t(!YQpXMrT%E$}gjLn~tO&=3I&RNbdLi(wY)I*<9t?ZE)q>%HE`W zQhiyz3>Z>h$+p-Zwf)i|)^FQi$(Z(C`o8=Sek6V*`9ASo8OP>; zKmD`9r2-9Nb_DCFpJqpki|g6K!h^JVnvgWV=uUJGo-dadjTc2$M4Tw^{0@38t)8Zj z>)qM#^l&G7boBRd*|0wG?e≪t8=i(I}Dl@4-nSL==uD1icH|Kcp8@A5tHJAC>P{ zD(1t01N##Cllf!vef9Qn^7XL&#xJ}RQWHiO=9m3pcYVHD+KzrQGo72$=}pJ>;(lx7 z!2K#vhXMnd9n3%L$K>t!{CPRP&TX#TINhiZmIsLgO9BKO*lX64|7LQ0KPAolqwjqK zggxqb?Tr>L?8Je?C!)A?OX$c^L=j0;nYriidaM9HxpMG|!N;dJ$j&b3Tt*qknqz2| zl~gelxH@JQOz`{lVS@-JOvCzghp|Hh;n*8sgt1#69=_upn#J1e!Zw8Mu>uW_$;Q~k zB^qP!Y3r-Hy1ULMlwERk_I&>!MIWliU)TJpsrggeQ_JUWay6DK4u}6{{8{h8=f{~B z2|uEb|2=%WIJkrPy->&cS-CeksE^skS;xk}or8CZ?w?hy3C76Z?H){PI-$ z$dAdr?N95*_-v$4@hr1*6OMQD;k9G_;m6yw;ofueR#@M2-9sMU!({^NYjk>bIx%-w zs9xM(h%c7U=ksveT9hAOgI6QMkJ^nPD}FFvbNUb5-}CkZoE?4!KL)?cqwQe#Uw7}- zG>%_AyWeYP_jGvvfARI!L2b3qAL!e+Z%Ye>mQtX_Da9qlTeQVJxJz*f?oM0Wi+gZ) z2^In^?iNBI5GYP?OK`Z{xik0o-}lU%nSEx@oH@_gGiP^pKf52XvrtKs1K8|!{rr6KCb0kE-G4e7{!+0O`h)fX-ShjO|Nr=X-D`~IC%khCHd#eH->4<8sWMQo zUE2nq7?2jOPl+C<8x$ARW6+fhMU^t4l_b4jN*@YSf;6%U|NO-un*1{ysy9hGlw8M$ zhry>Mcb$C~-u<}Z2u69jPEv_QK!H9jqNDJKY01@={`p0v7fn~n_|E#DOH!;SY^8V{ zFVw56qzy_TmdZuWC7je?H~f>7Zhj5Uzwrx<>RhSph8K1B9N+0C9tg7Lagn@%JU2c= z1p7YIg<<~XeQh%|IWg4JQ#wG@n<{3~QsnG9=WHt8PUti0jSSzw}t>`2lSiXneh5GUOoPw_4p2X~LSSMMkV@= zpMn>2E^@4QMtaD1MzAE;?rkjF#7??Xt&XY-q#HGuLX_T8*&BjlcI~!xn0iR~n=xf` zs0GjMixDz<>rURyBBk0&?bL+|-I?e?G>^4rO1ObOq8nw@PM??$bs z{z;%T*k#DYb1_R}=CD<4^t9lmoT1@m(=2oOoBe}8L)qD#gFz1q<(WF8>gJp7X1l|% zDH3?XmOlpZe1G($=G~Ct#be z{sCpb{Y`$)CV@fE^qf1iXT8;%9~+U?d2Er*J$MN9W^y>si%oBWkNF41lKyEKJbtI7%U)SN%4j z3l&R9&tud0;}0i2fyQiEXg>D4VDP?q6@v&PY(;W4k8VDbclB4E<&wZe^ttfr7`sd^ ziSL@W?wDY`Tb(vpdo04A`SICv=kwnhhk@0qoT`VVEzB_SYIweTTS{YCrVr`DQ4 ze*PXlA^Q8&Sa?Y4PN{8vYMrWh)#X2@dvP;8XQ#T|hQ@3eseSGv1XpGe#HpVD;yZ=* zBO0WQ$hAr?N=7;n@U2;l35L{ZE0q1^7uwZO1;9AZ{iKmvFsCOqbM-G z^Pgi>D?iNU3l-u^I3*RS16B!kC^0AcXPm)_!px(Kdo?+$2Mdodl2P-(tH6s>6KT&P z)>55uM;i^Obc|#4Gdx!l89>fntEY#a#T+t^mqY^%^*il+&o3Zu<$C-A)h8YlqW_5; zj|dVlF28M$RjkcmtZW!Fj}CD1e3`_4`sZKjhP7SOZN~gHhUuLw53r!=7HUJmw)z{{ zDPLg*))?xJ&I8#%$cvTLh7AdVQtlh3#pq8atBY{+dTo z*_8y%Xz^9XzoI^xn#h&$JdmFX9FO{X#@Le6?$eiUSDI7tWRe|-#(i^GEEC;(@N5B6 zk+4f{9v4 z<{s$649y_nf{^1{3vnQ{w>BW$j-*pl&F_`;LaBO1D)3PCUk)CKhTOum3cYeNNC)aZE1wAnK!S;mdNy5eC6DkY)?ur65g&+DuP($|A z>Y0uBZJdfJ2D4cuHYdgsZ_!1Tz&D%Iv9S$KCyN#R6Z<#&1cTkJ>o=L|`U zhx#rcbI3Pu1n``0odc&!|iujwPg^A?*+O%!q`>MdP(2!&kI;D5iEep>1 zg8;))VS=dcd4QGZ%%oK>-kC_2*p^~B|D4!XFk|?(W^cp%^l!T_N`mOs{Gat%Ro5!z zh8ubvg<|Cb>G+kOU*;g?w`ypKOWoDd{OSc-#`8EH)i&h_(fdA1y1Q@y!@8TBu>PX9 z1eRGi#jLDd`QRDE)0=a8j&D-?o|h86+TOm;9eCZ&VY;>Ur^td<{iSi?i#4p}> z*cZXHlRl2dOkVG=Z=z?&IWbNtFD#7}pC)eFXeEfTa&NU;ZDgu7uryokj_;1Gdl-DI z(NuyPyL+6bnJTrM{&lLR$Aee<5(*2J<^CNz>?_lhwwTC{RveL5t1r+%UQKenv(g?$ zM$1G}Oxrp+TGq39p_Ko~j-BbI>~8SH)6kr6Y7J9<76W&2rRp@HS4@RR8H$A$>IUvc z4yfV$QK9@|Lgcsf(#E(Z^TKb(TYBh4B7QJ5cf`8kl>_z^a< zfCPa~!8uHheW+=+17mx;hx9u_cy_!@sG8^p3R735uWog3lNg3$#9`?T*{bm*(~|n- z?@BVNNmZwR1^Ey3f~4*83U8$4Bi^Xk@3X)H*gOk$0U|7`a01W7F5S}JhRG)HgmJmX z7lD9EQZ+(-8a!zl`s3G!b5qW*k0M&_U=uBR8qnty8SYMZaFbv~GG z+_1^tsJ@7s`wd|&f7>-aD3GWBbMI}aUWNh5wY>3aVQlOo#o-f6>#1E>)H5A39lKie zcA*2;SoBjm!-7u`ixY}=?T@v7R?%;Sy|j8~?LD@|`Vf3UFFHKlAus=#tg%I}x**D? zvz9i7{`>+8?GELubC~F48kT@uEhApH7n^Bnv-k7Yz*x|WtNO1?Qa?0lR+1WIP8Aod zmx?gjBr9wzYTbdRlO!?toFZYNV{M|2ycayrARXU>7H0(AwbDxL{Xr>6yyTYa3S7-v7H{uvOa*>lM3grZqKHZtZlk*uF zLbZ6zU2TY?jTvm+L4ulsUohIQ;B zQH#TtyPjK-_Us4!ue}Te>bv<~xl0kBoo-}&HLN;#xC>oCjqZI#^|>?N7`F%k=F!xJ zYtC8@GDkL&1L+ASnYsscw#zd zK*V-;&osX{nLeOyPu5y9ogOkC~ccPqu(}Lzha22W6I{A`fTDuF| z(l|k-&evaR1Cy8Lzn>D3ML*%zdIsOJUrZnqgK5oKK-en3e@bo3(Lr)^pNu}cmD8?r zR=LtZcG!>YsBSr`VYu_7Y1a@gS(804uiyjQaCZqbIlqg3aEX<2t??+Wu)eSYQRE6) ztS-rTx~3vA&9^FTlxrKf$;U+0RwGE%_DD+9cCA%oAXBqdEVGu}?4A5 zwNeucI{_NvCzF7%y(E}<&XbQ6LraT3--|}t(r*MnDVNKPiJquzLs$22lJUT8_=HC-@=dTW?YUNEO#4g#AgBdO@sBG!epVD_S zeRvS}Rgy2tKaGifvoO)Jw*;&KDp(ut98nH##EagKGUKWiivG>VkW|42fMrnTtqjfz zH#X;wE1qktPuN?l2lNRw^v&g_nHb&8pCHqi&CfYy%}+=E%pHcUj&xD^T9uh!9_P2D zIlh3ko*{~XoJNMj909J|tE#>%oo|ez2w{-7R9EN~9dZ+)o12wG>buKTlIWMT#$r@L zH`qSGmjRx~8qr=HOk_~9Wxb2Bd?b0G=QcUPvK|Xp43~&&nYHQZB=bunF}@&{x{t19 z*OOVNwJ*g)FV=6y{>W~L3+K{`h_?#mzTKmJLD6iDbFKqnG({(Ja$mRsRt z(Z^kx1TRFy&*v7|Z(oipk7Fn5+sLniX6~=&mq!Dk_YaIsqj%o8`uT~88Z7NW7pS={ z$P*XaNq%kCxO_-3CWbn1>ogf(Ts{xY7~wq)2)|y6>!hO2y_7$z-%cwiwh_8g*~E+< zzU*{&=6AE+WX8+=bdhDd`}bu@7yoT}@Jp5D&jH8ZcD^0{IpY()fG*9Q)%s?Y-lqq$ z-X@*7!GLFC8G(Gah$a8_8NW}rS2F=HfC}e1&EY=g_IO4Zuag&h^2E!yBjFI=1d+%qpN; z-aF{qgc*Y&6XxZ)d;gQ`A*T-}i}yXZ4=cHo(`&`+(;>g}+CAEZd#P1W*qnEiw7b*h}Nu)^TfXA;}>Dv0e4eZi1_Etwk?AJ#7E z)oVf%XnJwov-~!&bYt@fb(GUn3mGL6U+VtnpKo|I$EH!D1L(i{wH{*m8|IJnMLFio z-*Exnn&k{(-_op3vl}vb;bOckAi@Ia>Q2+Tp1K@}+E2n5O$u(uVnVtm-gudUriixg zbY2pzW2#e%>DO(-`$FE{VkVu(-+W-2mhH@v3K=BBA_3NRF2X>N|BC76ffZY7%f%~I zG1g-7(gA>uvWDnPIzwgpk~AcO;v7CxZ-nu?>MJvGRG>a9iBV_sBPUWWBb(uE6GLUU z=5Q*VckK=`$l6&tC5DpYekFn`)!g_3cRJhtLiYBG?V;!wN|*<^%C%Vi9Zd$O}r zAp54D!8_bf(!=IpWz|nk69s2u9BP*G{t0mS2|OAmU2cvdZUo1FQb)>x-I}&mSpz{-49M=T~m_v}Yzp_JzyVItgWsT3xrg;k?Uq zkDuuizS9^6j{V$g1>7p^i={J};y(U%O2aXZmfzx8zwb=uX0u^?m6>lybR{8~DXyx6 zl4t?$zCT^UF-vB>L?j7>?rgj%(wl5(Z*Pn<3m2FcdmWFMM7-9yRLr9G^)bh3)ubj{ zGVqr?_N4Vd&}~%dCD(>JS~5wbBgNYa3G+$`77izZ4b1f4nd|T1jbt_XeG=J9hy+ir zS>OI~1*b}rVNbj#t2E{Ph!Gr2LTyc&Y*iU49{D{fTh4QyoS}9pnpXVK6^c*~Qz%$IR#ghkq_9tL-M*qsA>ud&LZmm`#_H%h12xc zS1py@$7iJZ8_exVycG{*C$0PYkcOineZAiZ>hQLh-FPX;AyT3GH~W;kCPV_S)M`>B zZLFe}dDlI4wrAC-L^Py{`ERm5Ihf?or9Z2)KZ2g6nP3Wxlr;0|ZkpqUvW$|CKjWu& z*o^dbBfjQ%s+j{;-9IR}8|5+HGOVsDEYzNDuGJfQZ%~KI zP-#LT?V)Pve9y98`KI#$u1=?~e&+&X#2&r*SVn;@Vtb7hS(+nu!B;6#k&EzU+i^RX zV|9Rk_GyIBf3<7ko1}oi!*_b|RmJ6Mvr%w(uMI)yksOp}`h#hOfA#sV>Dga?9<0yR zf`v=Kx(n>|lEH5ZY?FLoWXe;BDOGy&fl*|@6a5hNBAu$>sZwX9{eY1O@~;%!gijzM zs2=e3Mq+AVpnRkA=DX<+e;l=(tg+sYCIV$pBrN?BXCi5PcRv+( zpe48(=C^*s=8BBl+|RXGofW;-oQ+-d{Sgz=jh9~O2W_xaJ%4RRqj>O2gAz*-ma(8M zuZ`j$FRSMF?iABu>mU26S1*`ngX(@3>VSe87!n2u4!J!^s@-(bsw^myPHO6cdzQV< zZ2|2nWCX32H41ID$+Ts+0BFn)xc*6Wdduj?5WDTLic5;l@%M-k-M|~a?a&JbzgZP! zA|@paDA|UuWjlY|i5$7PRv1)mQEs{*G2`%5d2z#0N;T5gq1RwglgM$N zHB7W?9aTxD>u_3;6|42IPonC)o@aDpPls#Zm_&-e7|-nkJ)$zotv~6r?4#C(Fql@m zkpakRsmfW4lwT$Q?_I8G`B=%<_$F1>=~3-%S7wA;hTy0|T-Rc1-9VN3wk#ASjL7Q~ zm`NznP1_Uf&Q+v-Eap1)@k1Dg;wo|W`Epi+#)8A`16dsmhgkL0dHDPm+e@PIr2B9VPdO&8tjN zk`m*zkv!+q!11($)6&S-M18FZ(&}%GA;nq)N3zJ331#w&#tFsR@kpUO@H&liL zH+_o*Thh;B#Y#>}D(#l^EMl}Pl2wde7kM_3PJM`dqr?eJU*2}hX}<7tXsCz$slPdQPOWaZ&_D8l6^P>b`P@=~@(&6Nnx>OA07&qy!Z8*}wbN z+4!N4Ci_X(2XCmjr8(f#yZx6sH@3)1ov1=~J%Dpxx%w@oe~EjN53cYbAIF$#eq_aT zwH3Zg*tyaCt?kl2LXdO5Dpgw6UA~&W{H)$r__tqZcM8Yme#&Q3VY^z$RJ2Nu#Pqp| z#nql#-=6a_6{CyDqTedy?22)$^d_<2EiiIUElbU73@>QOBf~MCP{cODHlPnjC>hGl zplxFV{UbA(q!=M~n5r`?@^@B~GR=WZ!Sbpjdc4pkUQ+2^th>&-vZqepIO*CTO&zT1 zVXdlcCahVS8BD!~0F!CVolb+x2a+UX$RaRe;aY#PQ!+3vLM+dljuVoKam7L=zHWsaqcujAQJ!22A8FPrXS!d+ zQ|R$qZ5eRrdNf<{M`UntTB$?;tm(HSX@$0YXy`!&n6p3nKEGsC2S6K-643`fGW6&A zgBj9rmgilvGcI%2%cI@e9);PpjjRFLI7F*?UsoF$uH`1kyB$f0V>RC zj&GD6hKkfig;}J(UaGb8l=LNZ&GqWdXbqn+b8FeQ@uOa7l(ZYlupd*YApSO7uV|<` z0&ICM`-mLPpG_#a7Zo`@B26f|F&OGuS`z*2h^OgXl`x{(on+O`#$;1qzfFoL=N4>D z+Te)>rX@<38D!POE+m$v@Eet-JXrvp+BfAGU9PWc{yNZab9OHho1p z>Fjp!rDSv5h^a7q{mu6lzfE>lradQ$!`tWmVHCB_j=vorVeCys^W{q5O1a#Dabqn0 zzKXAQMvhF!WuMK8+t8`Kimt3ibrmcZu?_u+6N&^b$swb-TUyvsN3wVkjr}+{> zuA<#xd_R(~MTB@e^3o&g@-HMY{rhSYzuL2pswX$1(el#L0AwJx%<(?otm)yp z1(n&Phm5oQ<9uKB+p}#?&y>;JmSVAVpR+$az3lsm6nshN7L{goMLuEA7EEbaktsuH z7)pLgq@)eT8dKkMAO}q<%6!BPujD_-=bU5HQwoC6nw@1@z_=-b&X+G$Pp(003xmK3lc=BiL z?^|WmVl7~$|B?;5Ms=D=u)eQey}W9}Cb;sh%ydjE$!sTyY^H>Q&Vzpi{pztJ4oUr? zVK%huY3&rPFt4y2t)`_$FlzK%bL9uD`o9ubImJ!%3&Qcm4EzQY()kFMUyR4CX*o*< zc6ya|hZ7-zKy`L~UeY3cbZKgUOm9{8)dsgyMdtn}VS!V81nfXl%_5A2Ji@nw)%=W_ z!n9#RbZq@C!+I>-HQfiYtFY9nOR2-cALqTQPz?{^BBAJi>X|RY^z!~}NCLbor}r%s zWBf>8H{U-;;DKZsV2mp6Kn1-q$8o)_RjCSU-&}6?uEu5Z4$knCggu=C zxmR2cf1P?fX%)h{(Z1JURKpXX?BjXF$JavN=)WJERxjL?V#8{id9FwyeDnHv9xGSm zIDlhUqa&5mdm&*{*09>Y#O#r$Bj?ghZeed(w$Oai@2h8fiV>}|t}ROOpA+{>L5CVL z?`W&gmQRSV;ZEhCJx}W0H>*CXi*Z*d`aP2djId@DQ!=nsF=rUM=+!MqOGQs9U~H0| zv;59^iW0yBjhk}nXuss-2;oT;f~~6&&Xe4FroCfKTdrzKtE1kT<;`G#25w&8WPn8; z$?Y6ad0Rc(`P*=zDolB)zQ5fC&2j_?oN0862YD7hE&GFgj+CEPuE|%W0nYstq{TO< zSx)L$kBuGM@%{Qj4Vr-zoNhP8cUbC}4Xd?3HV_nWHJpNS_D6#oJ1omMMGH@e+f9M| zGC6iHqBh0_jI{z*H=4<5c`{2|76QPLJw=b2*W;*R1DonA?FB$~=Hd;@hTI07`=X}; zRscNEA&k0YV!}ZWlaxJ%8Z!5IAUB3XiTFtbulTj;4flau zF!OCk*#fC)g&bh*$>FxoSS$UK)K$~s%3u`2m+KpgmK>_u6}FZg66}Wg*kIseLrH4VBfS$*;NkXaJlg*DF*bU99)soYRJPt15Uk*; zv{eJiOnfcop;kMHNDFN{@^=`{?6WO@ieHYjm(v`5e%!#A(}!7*((1KytMu6@D%Bb` zkJDMDRuL~Owncc6*q)ROUAeA!%B2378JnGYpOBkOU)&=!PFYbGJOkM-(NYKUq7%%I8>8PbHH zdR-XKz2PlTXfF!zh1bj+v_6X*YY48uykvKJ*{H5KTU;DF%|=W}yodHX&^VrqD49x3 zwA%6Bi%e$zFj>pCM>S-%|Ee<14K1)_|MD5$de8Vp#8ois3a_=X&Esu(v|s)>CVeZ4 zED5ADBaMDAKt8!Z(JnnUbx@{IB3ZMBjty-)#`Nd=cJE%m>0^!(qM*-xPR@G!$*}Dz zj|dSVBctz$O>x(ZW(V7dso6I7q@C_|gloI~qvqOUpD7(7u}>CU_4>?*^oUahFVpA) z!+sTwQwqVzu|8sBX7||W%s620;hyTtzy1S79UIO1da50^Ml^bb`#Rt86|&%SD~~D` zL-pJ4&tg?%mpQr$N7tZmgtVXy?>{vr8$zPo8l<~Q#>y_q%}=NCoD9+P1zV9)+2hXD z9DK&?AUmwck6+L6=o}y}^Hs9VUk{$21$^ijb4(v@qeB~y#<99r@>_B3+YvL;3mrz| z7LjpX>Jv`Vy4k=PW`CJJMDsJBz(4PHUKW6<%=kX#5&!LAW7xV$%VRdSrdD=#qpo&g zAdJ#pY(+PRr2~!}Pgq}Qad{}1d)m%r&Qq|n`Dll`|6&jggzrjd%(RL-Pe(wfmVO`Z?mkz3#|o0cVp0)ag-`Z;Iq=*1XEKs99XU_*=zCQZlaPytABM z$6-VQVjSi7mMkiqVr=vIokX#MN$+cYrViU-T&!}C&!NdpUTs(wMh9*LMehMX{=4Y( zg~|oCiu7K}pYhJjlkd*3nf(rTv^HOBzx+a6{3dusxDjK|K{Ch>NrxUG^k}FN?e}^7gvJUYd;EUrnB#if7-?=O2y++1^3J#rSD+ zg3J#T!`Z?Q#Lq_BthYJ^&r{mm!kJ#8(+x8G{Enc(hvS{xilu|KgiE3qt9v(p;f;mC z@nd2ik5V zOQa;85?FV5(5Yn7aX59>gTaGmxQkIJhQqsZeO-1iKjq>jt9T~z9@PFV=bLzmNtgApLr=9(Erz8MVu8LUdjs8g%>~cx- z&}ay}4vj>T0GI-^FH}}nq{OBcu4jL^MVioh<9>fzh2rV1LNvt1JobP8{gLKJBaD4m z?fVpLO@S#(wSfUO)|WHem6;?`AJEQ)jEVXnki_(ETCQWam!a%fA4$iautB zn!LNWIwyw)i5c&7F5Y5~;|^MIu#Q8wwTVMN=bQ0k=5xmW>w&KN!9Xj_|VR@p|MDs<<+7ilgiG%Iv~x#+J4P&LNoVC3AZXm$@oL!e!9C`ZYn} zGHpQJ?bK9O;dygy?9%7|Nr6MqJ5 zl%ygSIrV$Jq*cyAmtd?w&UOoYH{D-D+S>5NH`;6Q1FHD94y?~Yy9HN};=G2>O6&+% zSjp65jodJb*RkSCfn&E_YLi7!?7@C~W_y}P+g#|I*yc`#i*kexJU&!?rP9mxv4`Mv4%qike6Y zkboESu^K64O%}Ve2AqDCB@fcI&5+&QNiDY0vr8$i_4CpFy?;fs`>|0jUm+}9qHw#p z=h;iv6q_e;SHC>5XKL5Bqp`{^-(9opVwwp;Uz}FiwQ(;Tlra-p@5oESutl*n0Lo@f zn-37u!H8$9nd*KEJ&PVJBcs*qk2SY)DjZEnvu>}NcD06>(rYo{28+#z?qUyMO0ocJ zk>d^tMgF@n%4YL@cK9rZ2=!DVO%@l5QuHn0OyJ>$~laH*X))U*b zF9;qsomxd+4Ky(Lm!E~h2Lp}WGSv0#ctsZJlK3FW&mI!rOYISZ*>-&;#6$+EtH+D= z7merpe2sSbV1p-i&gu_P;T#sWmm5lv)q-1yG^OIWCz`y%)ojk zAw-vqV>PrijL&J*ef)}6qppbh?eecM)vj^(3OWs(jg{R@w<{l$7Qh|>@F!Yx)9z!D zmDoC1rcjNZjYCoHq1Z;LpVWL@ku+3@VadeW$Y_%+n)~u#jC*+)^5P}6<*1QQ3J>u{ zaw%g5W6x_qGHC1f@)KTwKwCJZ#fr9abN2FAe|wwSLu_n7+awBeP$|<{8e1pkr))E9 zYf#H0A#XBnj1Et<^+GJ!{C2M$AD913cmQsFmX=!CfhEWAIvYd9sNDe$b3N?7wwm|_ zZb)!Zw8gFr<3x%_-+S!icR(>7rN|V4#8PfS_RV%@#35+D+03=e6&|ShS!_5eNq?uL z=CefwD zrs`9Yj&QC-$<8a!v<8_>dF@h~Og-IwB@bCbm$B1VoEg?q1Ind8mJa4-DsM8;B%f1? zFdZh-&T62wtdtXoD}rXaNRx7XT4KrnWjv2*{n%GK-}uu~<5$a9r?@iBiQ@hyNJf{O zXtym?q(sZdbQ&%N0t(x1VqK-|%jovs?>_`YXyuC1SR3fHyN``YfRAD-w^!K8aB$Ig zViTV>H$b;QwEs*47g7JkGwKAZph&@32h(rG*TE=0j-`z^FCba$e_82Ctwv<2eTu*7 zdI*M3dc%O*(KaD0vLAe@KeJ{Db?7u~#JXWO9_>^1?Ed-dD|8xVIw7$KkZm(_VNG&- zREmz~U1=goU;+C`d;6zPyKy>ENb@II$wHtkCY%(zmMedA+6A!SkaqwcJrVv|`K&n^ z7peWbj8~-{!-9g9_FaW{B4*9x|Ly#-G3fbR5}(nO9(y|qIddsmy)4sF;q}_oFxKD_ zC^%V~IbC-R%OF`&+%|Y;u3iuDG?N0YO0j0_&o3|Kd4zGF$GWU}{!+PaG7fgsy}rA$ z)wikhlfC7Ol{j72U?3k?mD>tXFgkR-?T|LL;biMoFfbF8@xyZ)_@zMBP`c3Bs>}&# z_-kKXb*x&=@z_NI)w zK8-~PeSCg8O{t3Si&}ksR{9*{Fq$CX1;Wx7xw5$b^jRm3W*=1?KI`TLCW5wxQCnM) zjN~hO^d%ByS5_GZk_vtJ#p%LaWQlpAT+P61R?>NMw~ORCgDe0 zUoWx_Fl@;DIUFodcKiMQI-H@xmP>d^D%*X^H@JrD&RMo^=uA-sSiJhcEPhU?&9n0H zjK|IE3A+yizw-iXFo5GrY|@e%(sAlJNO#iG$^~&WT7#NuR0_ zJZ7`h0Bez*&(sBYngt)+yY|d6A=T`-XC1XD-MuMti3az3Yqs*ZXw<7I+OX9Xwt8BY zJgbXoYV+EiA`rAr&al_N86Hv4X?vi-MP|p`hyD5wt&X~`<|zYL+wE#bTO%~9v9Uzk zmm~cC!YQ4J6Udv5JvCvy#!w6YD9TBkrc7(h1fOL*Rmq>d)nnE_q>AVa88t17lU-(q*2TC#ok6V5xZab}^ zHkGalZDJ{paf&t4keno4_X6YyE?kN;$C$A`o#ec)1~YmdO?a{H7AlY(eVJ}jIcb`k zy?!R?DHkH7N5L7zJJ9Z*|CM4vZ9`+C1oBZ(#vCYGP8AKT$m-B;jIHaL*G01lY!g)^ z4J`I-a(!yfK-vkA%48oZWX)B{?D{t4I_f(x#rSB52HgS&R(UZcSnN`^z zsxLR}jCa3ZJr&|uRF=<@%AJ*JjX$(F2(+SLL~hFp0k0Vo(oLrI8J>gB6d&r!TLtaX zvO4*QwMW)?O(H@%Ds%&&P1wmdHFc)mn>NG@C)Q}qf3W$zRUdf{AULVOGQEwLRTgCr zb#=$a>Dwe#8&?cfyEGg7o3`+406MKe3;BHfJK6dIm3>}@-hA8UuF;DS{c$EV3C_TJ zN($k*`qy9l@?*}{N}v(;wzisWhW)`U^bxM07SN&xJK{SR7l}0FFYWxKnITC+#Za#e zMBi9xe|5AwEdVX;B_yjxs|L?Yii@+=p0mm+M-cYe6VjoZ9k7_=Hpc<`l9%JzCKN^F zkUs#@$DyA~_hQHFYB;FQY_EEV%8^4Uiqn#cv~FH#o!h$%XpXJPBsV3+Uo7QmVB-o z-R&lH{#1-`Na_Di3$mHt$FFd3jEuATg2~H9AMCYmUB)anZ<>vZc0F+7iSPtkZXhJy z)>KS|PH)e|)T}>$j@ReXTqxnnD>mw`r)1k2kDP%^opg9Q;9>wTeIJ`pT& zmTuH`2Ze+FMY3;txPFoGrL&uVtb(Z}0I(4jj_wnptFH;kMJ{XMUx#+G-Dazh$aW!# zj1X2?JF=Q|>CMkUBHc}qvXa?t0V1CP>n7f8Rs)xH`_Wl0QzAzeI^LhXBBTexD6*_v7Brnw*|mriDN{3`L0{~^cAEmO zUdol}xpG>|F}m>A>=vv*RA#Y5`C=Yv^CO6m=Zs6jrwQQ(MSHgGp?s}t8C1ioAuEXo z6TrTVKwLIVj5C~LcF&~LnJC_P;G03laAR!l#==x!McVw}ZeMGq*5yGqyeaY=h*nCc zl@UaK+da%j$adQ^oFca3!eiE0XomUl+2J>!FuVPGPvL;c!*8SPvF_XfG}T2vth%Ix zA9toWII0meV#gRHDnLzk_M7FTa-1}mW-0NxHHZ+%LXvMOtxFGT9FZoNV^sy#u1=`w zQWaOG(-Mz8if1-Fm<@~6YZ!MO$OSt603_d7Mdq&M`e|?9Xbwzvawl$Pao( zXci;`Ri&QpxQlQdOV$+QD89YYyB3{1jhfoPjBs47fis<_hSXa{IdhJap(M4iuN(ER zs-A<4s&bj$n9)EbMLFaiuh!`M^f=#Mn3b)K7PtA1wcUCp<|t*KSZ?y0x`))E?* zvnO})qHKDmOq(>|bZs%(=v3TcoYxYmt<(!g(fmVQZIBbGs){)}k4D4<7WX7*0c8GRj1;rshPfQM_RXwX^39HWUB3pK<+f*7!_Oc}5b| z8|H6Q?--}?Z;Dh+m}YGJjYk!^5O9bGDx7TMKTvlj>a2NnIJgPCQeWiJ^KZdCid|eG zSKB<8m5_8H)a^gin6$pOUjOM*vPTE;f1wqn8hQq_#ExbIq37Cdvvm9pTR8!>Y^86p z$mWE5#(t`M6E?#arTdbx7{$FSko&4TlV0pyeadXc`0TJj+0o86nY3)G>VD7EB4Yk@ z_Jc}6lHH|ReVVKzd57{v{#)#pY9p-ptB=d>o8en?GmX*gQ`~2a5SqA+$w80zcBNClJ0r$=_1^4XyV_4+&8LUFc0;Q% zEcstWbKiQro3U~5Qga8+__RaawML0{1VCq*32uS%BQ9M1Lgw zK>oII!sTTpv_!pVcAeW0KicJVtR(0(?7>~>`8I!C8QPgh05Hm3vFHRl`t?U6A9yWA`G+X3`48iFPQebd$AMau9Q$ zmGx$QJcgmjgd+pNlY@N&IVZ949Z2yNYhPF4q}cMP5cP6hUQ^rnc#}NG?IKe}W#ZcT zh)p)XBtNSVItl!)v66!`#eJ>qSbGTAeI6PW`Lx*y+F|m-X&Y>occW%Ule?U-Mc=R6 ze+t2ZZowdPd@$(?W4ukmHgN<^5+^RsUpG#K8kb{F4&b)Ha%5*>j_ZU&`cjyx`SsI1#QzlV5->xCpsx zzJuSRE@Qg*a;=jkE+UGyZO+(L86Q|dwrKl#mvLQ@IPm2A`#5J}zN1P}@U0szbdPsl zup!>sWSHlFzKZ`-ddo!xEzE_f)#EdsGR^PizHVT$T7MVaAdf|!QC$M$BD>r{_aXR# zgu~Vn8(E$;ufC!8TyZ>AcaQ$>9{)q~@AZ+mS@^Af`uysvp_P%s+i$TPgPFR#AXEC; z;qIzfX0UIgTvzQ~MvB4UEmB4fdfqR@QTRo^j~@@y3OD z>b-xVgIveZwdeN#-tZI`>!juCb8+7K+ge_lH8WqgSw8(A&;Ng_duiAB3qiTkMm!^i zV^h-)OL;Lp4sd5%GL@3Q(|-*z{S+k+V?G`mzbQdm%m#qb^rLOa#Tx_k%|J5o)i~3( zd-3DFJRTwD;8Ws(NkfW^(q!AqY;*O%RP;DkncHR`Y0m*EbcQtRQVVr?XfY@D`?RP- zCGxryYO)d}TrRvgv#;XQbmUhAjXo8ompp%C@E0jQayTJR?hJ=SxHrr^)%|I*@h>1r zO_0lNElhgxi&4@wx1cmM>wN*5G-K!d82SsF_{s_kY%PCJOO5x>yHbaz7l^>-nWWqC zlZn*T5dfg;m*xNM_kT%_8Nz*CfzcN|nHxcJy4i}m-VK~-0(TZ&DrD>OsoJS)r}has zmRr;W$59VZ|138%pxBdkPzQc1ZkL&Q;@@^eI}-Yv5#9AAc)9EQKJ~Y*x-@3mN6ev| zH;L`{BkhbqlTJgxpwLTdn)P$fkY!}+kwRTB+&1-Quh4v`Ruoi))5`+ge3S#c;8h{O zKkCTM2(ZFoH-ytP9$E)zjSaK!Pi`QgCLA%G9o*;JhUe3#^Ip~yqoUbkfQQmgaIq6W zy!B%5{p0_&$Nx_^Dfh8K-`?`l)e+Rgp{K=4!~&P)2|GIx+_7#H41Wf-+N^T5>@L$B zUa*h9sg&5{{^Xi0sKj`jI@;#Vj37%!efQNQeAFKSb2$v@jd z_9{>tZr=~Da35*N&TSr$M?*JSg=~0xS>yGj{b=&HTV^r&<*tlqMB$IhiNWDkiz53m zk)Tla+$>xdTxL`5FlamnogFN-uJAe6{n2EU$UHKJ^dE%r;;@(ovLZY}+4zJf3_%CPtPS z;LMbhUu>$>fCL(8qq?W9(cj@%oxglK<=>Lm_I+19?6^vAH>EEBPJ|IL3pjsY(vw{d2ruscj~#Z6GnJQLdTEeIXi^_tT0zYEmQ5>Jv;D(1M)T>t`J zDy3+D>O1?G{=sSr=)L}>mh!JD?*TJ+X9rVvHr~0fJq^Z}cbx+W^Urti4u67fbLSf_ zrm0vbj{So{bxf0$Y^AD5{dadjk7c+sBRLL|rHFh9oHOW!%~tNadt*HsigChuHM)<- zW(%CBQr?7C)%^t}*~$rV2*Pe#AVx~H3qM~e(tQ_{|FUfzL#HHWT zc?Ty~g>COeZC(nww-^Jq0F+eanQ;SBnK4T!u+{Aol)d&OkBEe_2E& zg$w-uQ2MbpS&nU7|D}%=7_CxA%yos-O=dYldh1-()^$zDU=+Cwym(@u)w06J?DS?s zW`1+!El;<8P{!A!J^%(n7N}d)Da4JWI>gvvJe;{Y6Kd35hX@N<{MT@?5)MPN+!wRM zZtaZ4a*Q3i=h-CFIp@kOTzH;&U5V89AIz2}Vnl{rsXhOvj?MVSEZbtosp~J|vI@}a zf+b7j%ofgZU68+=RGG%VCU=lB`qfa?2_a3{R$qn+dIiyBUnR6OA~=AjM4K5dH*G?Q z`+w$7UZ!sz(W{ZG6xl0cqWfE4-Hto!!n*5{w{eK_RUYJjPRsjtWvWPuvnKn&Qmq*Q zFEUx0p{azoR5PSs(bpHHv)iX91qpe-&s-Oh5&!bl&}&1+3!;BywQ&27jSlZZ%tn{60LR`@hLqlf?drdu2ZvSzD{`X1!amTpNdPnf<&CXj-TiL1p6GDam}7pGr&xdf zUKEF3iYR;7U4|_4l1QD6@F}+>$3xd-xt28~OFg}dD&mr|{H!yMp!ydO8XZm=4TZ(S z)EdI~AeV+?Ckl|o*5rGpaKvD7LXb2rJ0nMRa+yt42uy5{gyXxEO5CsR-PUW`E0;sC|XE?2_`VQel?@;QU^SpcR7?Y+hQ6InZGsl}? z#;l;2JoA#Pa5sdnvc9e_7+g67HhW(<;w{?as5g9EEBV2S$1*YyRM7CImnXfHm64s5 zg)v>v4a><8xnPygX46BX8BBs!+aA4nILeQ?MI(@#gqMrJiI%)G5(`k{c@>$_8zi`E zNQks9ddTcO+N$Z8n*t{DULk^1JT(<0t(vu+b_&Mw33tOkGXA{hT=Up6`MpO9K2k|n zd?y#>8$|vtpjhKShzkEEI#78{w|#q|TM&3d z#r9&dXbsEa2N2P{d!na-AEnhQ{=diC$$jy+<^O>A`!|?D(z?WSoA;xriBDvhfp(~{ zd&}RiX%f=}??-DDx$IEk^%?bAkBLuAm_Zi*VN(4@qxbg&n1Nu3@LYrUcLvNrgeD;| zr+%Z}`_V#0u3)IJFij#NXyQ{}MXnLQ44120>-}h^A~zRi;1D8wVNN~h(Zr_@FoWI@ zVc%)>TJMQZRdQeGM_k|&=51%<1=d_d5V!%oj01wy4pjA(b5oO2uC{~gw_`r`LF15~ z|Csk@Y08o$v*KRdIxJrI;r^Ge18_^O&-Su}YY)QZuNBAW$`iC_hjvEQN^DDMoifSy z7$CS~8Vrq~q8iSeNgu5!J4;D;0KP@kDw{}%!-l~@B;;~NN z`T`WUx`?ACSvJhvj#5za{lc|zlv-+qEkG3CC-Q`%NNW_CE4*k{lfKT{8VEu|8ASFi zIg9xuO)@0U9bn1$MK&1SbuGYk8W74fK>o?4G5(+Jk&jO6_VVK7AO8b%hzt_C62UX+&(R`Xh(dD+k1o>F#N<5fRXCQw^Xg%_`X<6@=#-!iw$Yur!M9ou?~ zoR&~>?tN)TOZ=9Q1sKin*Y~kq)8rd*tv?rG2meOhKmIn3Aj>Uk;xO5n(?vbKyJ^SG-ehNi zW!DPKpMs<}BJv+Lg+rI@+RfYgx84)NyQaF}H+RseNmbdT6yx_{h|?>Vpp9Y!BwRhb zZ`;rnA*~f{hK$ftRmT~dA{v!an{X=)5P7kZwU#?3GApO7PLC-?M-0rSSm=vtCK^|F z41_<2j|K(?7N3?;JiA4|U@{vRzU4PsWSQ7=$0d_BS>m!Z?$(4%&@4Em8`@+&dqza6 z#i?rCEiOt)OtG@N$g@jt$@463Ud+X=I@>}awS2PV>ehVti)VK|wIR#uHYYJAJyKru z0Q<+WUfuW26e}{c@BxD)oZ?Eo2I-VK86!Tn5j7$**}A1rIm(K9ie2S`OD(TgCdZ$p+J6&Lp$0{$sVdGu+U~;0le4FUhuVwn{O06gs#gQ>u6`0?LbEsO zU%FpbdA4bb-TJa9>thSy-?MVOX+2Lj&L7uvI5ac7E%BT0>~O zzM)vg>Z((W)D`{I`}=7QIoKn^56!OenlBM(-b==>c9}gT7V2)>O=sOf=x-P9AynK; zSMsoBj=4%|fsthwB47O^a2!$VW@$l*Rji?{T3b4D$#?jLr$zQ}Cd_L#de8dGt;Ed` z2y$qMG?zwW-}B!!-3&fs@b(H|2iJQ9;cV4G8o#xlA%YReo7L%mQLZ|a2X1)ip~-Fy zt?3RG>!X&%A`$cmUR{m_(hrdo?KsD|`g|LXRH4gBeV5>KIF_K)^^FF!gjV7cSyAF{ z@>7Y&MNco{BxYTegXmF?05+lM%yQU|QEijUmVtq25Iz!P#0Oh7)6o|{2(n9VUrb3R z;iInDsJ-+UxPeU(tr+eS9#GTNpngDGf$&ky4faYa6`Lod0t`-N*4{QiDrbV@PnNku z!c?=Zbo(h{e*I5X=oRoMx_-^?oZe05f5c2O(3|O^LF`uMuoi{*Oemc&`7JczZs<_7 zus8A~eGFAsSR`$bXWlgMV=AZ53Ii2$;4T~5md_1pU~0Bf2~(=WXJ?;d@Rhph#nBnl zN}-KbCmn|CO${0K&V8d>xO@?o-{YNRACt#)_a00iTx~QmGr*LQ0irr!17y__z?-@& zSqf>WW=}KWL|0XWvrU_`NVVqdj&K3u?_5y<-Bup0&9P)hj8KV9)HaVxE5hEG%6Qsa_7GTZw<-oZ>^bk+9_BV^G{J2q*C3aQz~gwRsk&wlw+ znm5-X51nyeS*APl7F~xwHiaJq<>}RM{O6rrwC0MG3JZJk$EcK1DL&c%?9H(6jSBWs zOU_;f

cO=OG1S!rnH1sm4vNKr+Z!3;$?&*k|Tcua5)I)4@es^MeYq&RlxeW-O%| z3R7`-jca;7X&JkvlUb;}8*WKwU2k@)73sY{b5RNaF^f3t$M!gzwx^y|x+UgUv`$$>3E++aK8&5++53)A=E%wRHH(WyP7##`!+)~cZf5`hoS;P& zrIUF;R<-xiB~_EIk?6#Te3ShA3F{X(;d%O#&0jxtppI)_9#4YICv{6j09nyAqq(8F-O(hlbH7Lam#?;exB6}s)Wh9hjQ z$o=r1z2w8w&I$cB5EUMQ7TK<+$pkSk+8TGHzI~9 z*6$$cf1OQ_ud8G($rsTz&qc`;OBzOmY#T(j7Djuq8iRw4(g$kzkaErwC_|K0$SAX1~jtE9Xld_E>TeoXj}|HIBi4_)}hP9^fc4GsQ)g2h*&;N&wr z*bwEv40bP&ag_G&aiA2%^6LD}#bQgJU`Pnn9^rGzVRzg>d-egjq(Gz0dv1H}2F~pI z_`5~;I;Ahd=23^Jgn7Gw#GXGpC;PBYzKZD4-BbU@$f9PFr)lhdr@UjO_(W@>)q3Zc%L$W(4q1Ik&+OFK*K^!5hMdCo5f zFd9+M_#q5AUyr8X&dj3BG#h@%;fFedpkS@+TlSx9xFpS`SVtneQMkSK&=uZo^g9K5?k?9Pl}kYUqFu|)6|5%)InP|$Y)unaJwKh*uZ7|0%6_>a>-N4|`uB(Amq>gc^~2z-_2diWP6sc^95VZ8 zOXSNFYe32PMXD&&$?UGCF^KH_mXmy}!{IN)+b_bvCBl4?U u(v { marker: std::marker::PhantomData, } -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -pub enum DocumentFetchKind { - PerDocumentId { - retrieve_vectors: bool, - sort: bool, - }, - Normal { - with_filter: bool, - limit: usize, - offset: usize, - retrieve_vectors: bool, - sort: bool, - ids: usize, - }, -} - -impl DocumentsFetchAggregator { - pub fn from_query(query: &DocumentFetchKind) -> Self { - let (limit, offset, retrieve_vectors, sort) = match query { - DocumentFetchKind::PerDocumentId { retrieve_vectors, sort } => { - (1, 0, *retrieve_vectors, *sort) - } - DocumentFetchKind::Normal { limit, offset, retrieve_vectors, sort, .. } => { - (*limit, *offset, *retrieve_vectors, *sort) - } - }; - - let ids = match query { - DocumentFetchKind::Normal { ids, .. } => *ids, - DocumentFetchKind::PerDocumentId { .. } => 0, - }; - - Self { - per_document_id: matches!(query, DocumentFetchKind::PerDocumentId { .. }), - per_filter: matches!(query, DocumentFetchKind::Normal { with_filter, .. } if *with_filter), - max_limit: limit, - max_offset: offset, - sort, - retrieve_vectors, - max_document_ids: ids, - - marker: PhantomData, - } - } -} - impl Aggregate for DocumentsFetchAggregator { fn event_name(&self) -> &'static str { Method::event_name() @@ -1573,16 +1527,19 @@ fn retrieve_documents>( })? } - let facet_sort; let (it, number_of_documents) = if let Some(sort) = sort_criteria { let number_of_documents = candidates.len(); - facet_sort = recursive_sort(index, &rtxn, sort, &candidates)?; + let facet_sort = recursive_sort(index, &rtxn, sort, &candidates)?; let iter = facet_sort.iter()?; + let mut documents = Vec::with_capacity(limit); + for result in iter.skip(offset).take(limit) { + documents.push(result?); + } ( itertools::Either::Left(some_documents( index, &rtxn, - iter.map(|d| d.unwrap()).skip(offset).take(limit), + documents.into_iter(), retrieve_vectors, )?), number_of_documents, diff --git a/crates/milli/src/documents/sort.rs b/crates/milli/src/documents/sort.rs index 59858caad..3866d9e27 100644 --- a/crates/milli/src/documents/sort.rs +++ b/crates/milli/src/documents/sort.rs @@ -72,6 +72,10 @@ impl Iterator for SortedDocumentsIterator<'_> { /// The default implementation of `nth` would iterate over all children, which is inefficient for large datasets. /// This implementation will jump over whole chunks of children until it gets close. fn nth(&mut self, n: usize) -> Option { + if n == 0 { + return self.next(); + } + // If it's at the leaf level, just forward the call to the values iterator let (current_child, next_children, next_children_size) = match self { SortedDocumentsIterator::Leaf { values, size } => { @@ -189,41 +193,54 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { fn build(self) -> crate::Result> { let size = self.candidates.len() as usize; - // There is no point sorting a 1-element array - if size <= 1 { - return Ok(SortedDocumentsIterator::Leaf { - size, - values: Box::new(self.candidates.into_iter()), - }); - } - - match self.fields.first().copied() { - Some(AscDescId::Facet { field_id, ascending }) => self.build_facet(field_id, ascending), - Some(AscDescId::Geo { field_ids, target_point, ascending }) => { - self.build_geo(field_ids, target_point, ascending) - } - None => Ok(SortedDocumentsIterator::Leaf { + match self.fields { + [] => Ok(SortedDocumentsIterator::Leaf { size, values: Box::new(self.candidates.into_iter()), }), + [AscDescId::Facet { field_id, ascending }, next_fields @ ..] => { + SortedDocumentsIteratorBuilder::build_facet( + self.index, + self.rtxn, + self.number_db, + self.string_db, + next_fields, + self.candidates, + self.geo_candidates, + *field_id, + *ascending, + ) + } + [AscDescId::Geo { field_ids, target_point, ascending }, next_fields @ ..] => { + SortedDocumentsIteratorBuilder::build_geo( + self.index, + self.rtxn, + self.number_db, + self.string_db, + next_fields, + self.candidates, + self.geo_candidates, + *field_ids, + *target_point, + *ascending, + ) + } } } /// Builds a [`SortedDocumentsIterator`] based on the results of a facet sort. + #[allow(clippy::too_many_arguments)] fn build_facet( - self, + index: &'ctx crate::Index, + rtxn: &'ctx heed::RoTxn<'ctx>, + number_db: Database, FacetGroupValueCodec>, + string_db: Database, FacetGroupValueCodec>, + next_fields: &'ctx [AscDescId], + candidates: RoaringBitmap, + geo_candidates: &'ctx RoaringBitmap, field_id: u16, ascending: bool, ) -> crate::Result> { - let SortedDocumentsIteratorBuilder { - index, - rtxn, - number_db, - string_db, - fields, - candidates, - geo_candidates, - } = self; let size = candidates.len() as usize; // Perform the sort on the first field @@ -248,7 +265,7 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { rtxn, number_db, string_db, - fields: &fields[1..], + fields: next_fields, candidates: r?, geo_candidates, }) @@ -262,22 +279,19 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { } /// Builds a [`SortedDocumentsIterator`] based on the (lazy) results of a geo sort. + #[allow(clippy::too_many_arguments)] fn build_geo( - self, + index: &'ctx crate::Index, + rtxn: &'ctx heed::RoTxn<'ctx>, + number_db: Database, FacetGroupValueCodec>, + string_db: Database, FacetGroupValueCodec>, + next_fields: &'ctx [AscDescId], + candidates: RoaringBitmap, + geo_candidates: &'ctx RoaringBitmap, field_ids: [u16; 2], target_point: [f64; 2], ascending: bool, ) -> crate::Result> { - let SortedDocumentsIteratorBuilder { - index, - rtxn, - number_db, - string_db, - fields, - candidates, - geo_candidates, - } = self; - let mut cache = VecDeque::new(); let mut rtree = None; let size = candidates.len() as usize; @@ -307,7 +321,7 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { rtxn, number_db, string_db, - fields: &fields[1..], + fields: next_fields, candidates: docids, geo_candidates, })); @@ -322,7 +336,7 @@ impl<'ctx> SortedDocumentsIteratorBuilder<'ctx> { rtxn, number_db, string_db, - fields: &fields[1..], + fields: next_fields, candidates: not_geo_candidates, geo_candidates, })); From 1bc30cb4c8c5f89fa75e80d83b427ed54fe40e29 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 15 Jul 2025 17:34:04 +0200 Subject: [PATCH 139/312] Restore old benchmark names --- crates/benchmarks/benches/utils.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/crates/benchmarks/benches/utils.rs b/crates/benchmarks/benches/utils.rs index 93fa7506f..8eb3c344a 100644 --- a/crates/benchmarks/benches/utils.rs +++ b/crates/benchmarks/benches/utils.rs @@ -160,11 +160,9 @@ pub fn run_benches(c: &mut criterion::Criterion, confs: &[Conf]) { for &query in conf.queries { for offset in conf.offsets { - let parameter = match (query.is_empty(), offset) { - (true, None) => String::from("placeholder"), - (true, Some((offset, limit))) => format!("placeholder[{offset}:{limit}]"), - (false, None) => query.to_string(), - (false, Some((offset, limit))) => format!("{query}[{offset}:{limit}]"), + let parameter = match offset { + None => query.to_string(), + Some((offset, limit)) => format!("{query}[{offset}:{limit}]"), }; group.bench_with_input( BenchmarkId::from_parameter(parameter), From d6bd60d569d4a07578c4891542756bd0cf38705a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 15 Jul 2025 18:00:37 +0200 Subject: [PATCH 140/312] Apply review suggestions Co-Authored-By: Louis Dureuil --- crates/milli/src/search/new/bucket_sort.rs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/crates/milli/src/search/new/bucket_sort.rs b/crates/milli/src/search/new/bucket_sort.rs index 298983091..645d36e16 100644 --- a/crates/milli/src/search/new/bucket_sort.rs +++ b/crates/milli/src/search/new/bucket_sort.rs @@ -161,12 +161,13 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>( }; } - let max_total_hits = max_total_hits.unwrap_or(usize::MAX); - while valid_docids.len() < length - || (exhaustive_number_hits - && ranking_score_threshold.is_some() - && valid_docids.len() < max_total_hits) - { + let max_len_to_evaluate = + match (max_total_hits, exhaustive_number_hits && ranking_score_threshold.is_some()) { + (Some(max_total_hits), true) => max_total_hits, + _ => length, + }; + + while valid_docids.len() < max_len_to_evaluate { if time_budget.exceeded() { loop { let bucket = std::mem::take(&mut ranking_rule_universes[cur_ranking_rule_index]); From 191ea340ed7ba4deea3538e7a716b7c6052c1689 Mon Sep 17 00:00:00 2001 From: Thomas Gerbet Date: Wed, 23 Apr 2025 11:50:36 +0200 Subject: [PATCH 141/312] Sign container image using Cosign in keyless mode Cosign keyless mode makes possible to sign the container image using the OIDC Identity Tokens provided by GitHub Actions [0][1]. The signature is published to the registry storing the image and to the public Rekor transparency log instance [2]. Cosign keyless mode has already been adopted by some major projects like Kubernetes [3]. The image signature can be manually verified using: ``` $ cosign verify \ --certificate-oidc-issuer='https://token.actions.githubusercontent.com' \ --certificate-identity-regexp='^https://github.com/meilisearch/meilisearch/.github/workflows/publish-docker-images.yaml' \ ``` See #2179. Note that a similar approach can be used to sign the release binaries. [0] https://docs.github.com/en/actions/deployment/security-hardening-your-deployments/about-security-hardening-with-openid-connect [1] https://docs.sigstore.dev/cosign/signing/signing_with_containers/ [2] https://docs.sigstore.dev/rekor/overview [3] https://kubernetes.io/docs/tasks/administer-cluster/verify-signed-artifacts/#verifying-image-signatures --- .github/workflows/publish-docker-images.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml index 74384e670..6d2ce2248 100644 --- a/.github/workflows/publish-docker-images.yml +++ b/.github/workflows/publish-docker-images.yml @@ -16,6 +16,8 @@ on: jobs: docker: runs-on: docker + permissions: + id-token: write # This is needed to use Cosign in keyless mode steps: - uses: actions/checkout@v3 @@ -62,6 +64,9 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + - name: Install cosign + uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # tag=v3.8.2 + - name: Login to Docker Hub uses: docker/login-action@v3 with: @@ -85,6 +90,7 @@ jobs: - name: Build and push uses: docker/build-push-action@v6 + id: build-and-push with: push: true platforms: linux/amd64,linux/arm64 @@ -94,6 +100,17 @@ jobs: COMMIT_DATE=${{ steps.build-metadata.outputs.date }} GIT_TAG=${{ github.ref_name }} + - name: Sign the images with GitHub OIDC Token + env: + DIGEST: ${{ steps.build-and-push.outputs.digest }} + TAGS: ${{ steps.meta.outputs.tags }} + run: | + images="" + for tag in ${TAGS}; do + images+="${tag}@${DIGEST} " + done + cosign sign --yes ${images} + # /!\ Don't touch this without checking with Cloud team - name: Send CI information to Cloud team # Do not send if nightly build (i.e. 'schedule' or 'workflow_dispatch' event) From a683faa882ae5c996650ada5d58822eb3f71e226 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 16 Jul 2025 11:03:24 +0200 Subject: [PATCH 142/312] Apply review suggestions --- crates/meilisearch/tests/common/mod.rs | 173 +++++++++++++++++- crates/meilisearch/tests/common/server.rs | 4 +- crates/meilisearch/tests/vector/fragments.rs | 175 +------------------ 3 files changed, 177 insertions(+), 175 deletions(-) diff --git a/crates/meilisearch/tests/common/mod.rs b/crates/meilisearch/tests/common/mod.rs index 1a73a7532..1345fa197 100644 --- a/crates/meilisearch/tests/common/mod.rs +++ b/crates/meilisearch/tests/common/mod.rs @@ -3,8 +3,12 @@ pub mod index; pub mod server; pub mod service; -use std::fmt::{self, Display}; +use std::{ + collections::BTreeMap, + fmt::{self, Display}, +}; +use actix_http::StatusCode; #[allow(unused)] pub use index::GetAllDocumentsOptions; use meili_snap::json_string; @@ -13,6 +17,10 @@ use serde::{Deserialize, Serialize}; #[allow(unused)] pub use server::{default_settings, Server}; use tokio::sync::OnceCell; +use wiremock::{ + matchers::{method, path}, + Mock, MockServer, Request, ResponseTemplate, +}; use crate::common::index::Index; @@ -508,3 +516,166 @@ pub async fn shared_index_with_geo_documents() -> &'static Index<'static, Shared }) .await } + +pub async fn shared_index_for_fragments() -> Index<'static, Shared> { + static INDEX: OnceCell<(Server, String)> = OnceCell::const_new(); + let (server, uid) = INDEX + .get_or_init(|| async { + let (server, uid, _) = init_fragments_index().await; + (server.into_shared(), uid) + }) + .await; + server._index(uid).to_shared() +} + +async fn fragment_mock_server() -> String { + let text_to_embedding: BTreeMap<_, _> = vec![ + ("kefir", [0.5, -0.5, 0.0]), + ("intel", [1.0, 1.0, 0.0]), + ("dustin", [-0.5, 0.5, 0.0]), + ("bulldog", [0.0, 0.0, 1.0]), + ("labrador", [0.0, 0.0, -1.0]), + ("{{ doc.", [-9999.0, -9999.0, -9999.0]), // If a template didn't render + ] + .into_iter() + .collect(); + + let mock_server = Box::leak(Box::new(MockServer::start().await)); + + Mock::given(method("POST")) + .and(path("/")) + .respond_with(move |req: &Request| { + let text = String::from_utf8_lossy(&req.body).to_string(); + + let mut data = [0.0, 0.0, 0.0]; + for (inner_text, inner_data) in &text_to_embedding { + if text.contains(inner_text) { + for (i, &value) in inner_data.iter().enumerate() { + data[i] += value; + } + } + } + ResponseTemplate::new(200).set_body_json(json!({ "data": data })) + }) + .mount(mock_server) + .await; + + mock_server.uri() +} + +pub async fn init_fragments_index() -> (Server, String, crate::common::Value) { + let url = fragment_mock_server().await; + let server = Server::new().await; + let index = server.unique_index(); + + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + assert_eq!(code, StatusCode::OK); + + // Configure the index to use our mock embedder + let settings = json!({ + "embedders": { + "rest": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + }, + "searchFragments": { + "justBreed": {"value": "It's a {{ media.breed }}"}, + "justName": {"value": "{{ media.name }} is a dog"}, + "query": {"value": "Some pre-prompt for query {{ q }}"}, + } + }, + }, + }); + let (response, code) = index.update_settings(settings.clone()).await; + assert_eq!(code, StatusCode::ACCEPTED); + + server.wait_task(response.uid()).await.succeeded(); + + // Send documents + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel", "breed": "labrador"}, + {"id": 3, "name": "dustin", "breed": "bulldog"}, + ]); + let (value, code) = index.add_documents(documents, None).await; + assert_eq!(code, StatusCode::ACCEPTED); + + let _task = index.wait_task(value.uid()).await.succeeded(); + + let uid = index.uid.clone(); + (server, uid, settings) +} + +pub async fn init_fragments_index_composite() -> (Server, String, crate::common::Value) { + let url = fragment_mock_server().await; + let server = Server::new().await; + let index = server.unique_index(); + + let (_response, code) = server.set_features(json!({"multimodal": true})).await; + assert_eq!(code, StatusCode::OK); + + let (_response, code) = server.set_features(json!({"compositeEmbedders": true})).await; + assert_eq!(code, StatusCode::OK); + + // Configure the index to use our mock embedder + let settings = json!({ + "embedders": { + "rest": { + "source": "composite", + "searchEmbedder": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "searchFragments": { + "query": {"value": "Some pre-prompt for query {{ q }}"}, + } + }, + "indexingEmbedder": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{fragment}}", + "response": { + "data": "{{embedding}}" + }, + "indexingFragments": { + "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, + "basic": {"value": "{{ doc.name }} is a dog"}, + } + }, + }, + }, + }); + let (response, code) = index.update_settings(settings.clone()).await; + assert_eq!(code, StatusCode::ACCEPTED); + + server.wait_task(response.uid()).await.succeeded(); + + // Send documents + let documents = json!([ + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel", "breed": "labrador"}, + {"id": 3, "name": "dustin", "breed": "bulldog"}, + ]); + let (value, code) = index.add_documents(documents, None).await; + assert_eq!(code, StatusCode::ACCEPTED); + + index.wait_task(value.uid()).await.succeeded(); + + let uid = index.uid.clone(); + (server, uid, settings) +} diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index e3839855b..390652340 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -35,7 +35,7 @@ pub struct Server { pub static TEST_TEMP_DIR: Lazy = Lazy::new(|| TempDir::new().unwrap()); impl Server { - pub fn into_shared(self) -> Server { + pub(super) fn into_shared(self) -> Server { Server { service: self.service, _dir: self._dir, _marker: PhantomData } } @@ -327,7 +327,7 @@ impl Server { self.service.get(url).await } - pub fn _index(&self, uid: impl AsRef) -> Index<'_> { + pub(super) fn _index(&self, uid: impl AsRef) -> Index<'_> { Index { uid: uid.as_ref().to_string(), service: &self.service, diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index 4fe2bddb6..a994eb64c 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -1,180 +1,11 @@ -use std::collections::BTreeMap; - use meili_snap::{json_string, snapshot}; -use tokio::sync::OnceCell; -use wiremock::matchers::{method, path}; -use wiremock::{Mock, MockServer, Request, ResponseTemplate}; -use crate::common::index::Index; -use crate::common::{Owned, Shared}; +use crate::common::{ + init_fragments_index, init_fragments_index_composite, shared_index_for_fragments, +}; use crate::json; use crate::vector::{GetAllDocumentsOptions, Server}; -async fn shared_index_for_fragments() -> Index<'static, Shared> { - static INDEX: OnceCell<(Server, String)> = OnceCell::const_new(); - let (server, uid) = INDEX - .get_or_init(|| async { - let (server, uid, _) = init_fragments_index().await; - (server.into_shared(), uid) - }) - .await; - server._index(uid).to_shared() -} - -async fn fragment_mock_server() -> String { - let text_to_embedding: BTreeMap<_, _> = vec![ - ("kefir", [0.5, -0.5, 0.0]), - ("intel", [1.0, 1.0, 0.0]), - ("dustin", [-0.5, 0.5, 0.0]), - ("bulldog", [0.0, 0.0, 1.0]), - ("labrador", [0.0, 0.0, -1.0]), - ("{{ doc.", [-9999.0, -9999.0, -9999.0]), // If a template didn't render - ] - .into_iter() - .collect(); - - let mock_server = Box::leak(Box::new(MockServer::start().await)); - - Mock::given(method("POST")) - .and(path("/")) - .respond_with(move |req: &Request| { - let text = String::from_utf8_lossy(&req.body).to_string(); - - let mut data = [0.0, 0.0, 0.0]; - for (inner_text, inner_data) in &text_to_embedding { - if text.contains(inner_text) { - for (i, &value) in inner_data.iter().enumerate() { - data[i] += value; - } - } - } - ResponseTemplate::new(200).set_body_json(json!({ "data": data })) - }) - .mount(mock_server) - .await; - - mock_server.uri() -} - -pub async fn init_fragments_index() -> (Server, String, crate::common::Value) { - let url = fragment_mock_server().await; - let server = Server::new().await; - let index = server.unique_index(); - - let (_response, code) = server.set_features(json!({"multimodal": true})).await; - snapshot!(code, @"200 OK"); - - // Configure the index to use our mock embedder - let settings = json!({ - "embedders": { - "rest": { - "source": "rest", - "url": url, - "dimensions": 3, - "request": "{{fragment}}", - "response": { - "data": "{{embedding}}" - }, - "indexingFragments": { - "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, - "basic": {"value": "{{ doc.name }} is a dog"}, - }, - "searchFragments": { - "justBreed": {"value": "It's a {{ media.breed }}"}, - "justName": {"value": "{{ media.name }} is a dog"}, - "query": {"value": "Some pre-prompt for query {{ q }}"}, - } - }, - }, - }); - let (response, code) = index.update_settings(settings.clone()).await; - snapshot!(code, @"202 Accepted"); - - server.wait_task(response.uid()).await.succeeded(); - - // Send documents - let documents = json!([ - {"id": 0, "name": "kefir"}, - {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, - {"id": 2, "name": "intel", "breed": "labrador"}, - {"id": 3, "name": "dustin", "breed": "bulldog"}, - ]); - let (value, code) = index.add_documents(documents, None).await; - snapshot!(code, @"202 Accepted"); - - let task = index.wait_task(value.uid()).await; - snapshot!(task["status"], @r###""succeeded""###); - - let uid = index.uid.clone(); - (server, uid, settings) -} - -pub async fn init_fragments_index_composite() -> (Server, String, crate::common::Value) { - let url = fragment_mock_server().await; - let server = Server::new().await; - let index = server.unique_index(); - - let (_response, code) = server.set_features(json!({"multimodal": true})).await; - snapshot!(code, @"200 OK"); - - let (_response, code) = server.set_features(json!({"compositeEmbedders": true})).await; - snapshot!(code, @"200 OK"); - - // Configure the index to use our mock embedder - let settings = json!({ - "embedders": { - "rest": { - "source": "composite", - "searchEmbedder": { - "source": "rest", - "url": url, - "dimensions": 3, - "request": "{{fragment}}", - "response": { - "data": "{{embedding}}" - }, - "searchFragments": { - "query": {"value": "Some pre-prompt for query {{ q }}"}, - } - }, - "indexingEmbedder": { - "source": "rest", - "url": url, - "dimensions": 3, - "request": "{{fragment}}", - "response": { - "data": "{{embedding}}" - }, - "indexingFragments": { - "withBreed": {"value": "{{ doc.name }} is a {{ doc.breed }}"}, - "basic": {"value": "{{ doc.name }} is a dog"}, - } - }, - }, - }, - }); - let (response, code) = index.update_settings(settings.clone()).await; - println!("Update settings response: {:?}", response); - snapshot!(code, @"202 Accepted"); - - server.wait_task(response.uid()).await.succeeded(); - - // Send documents - let documents = json!([ - {"id": 0, "name": "kefir"}, - {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, - {"id": 2, "name": "intel", "breed": "labrador"}, - {"id": 3, "name": "dustin", "breed": "bulldog"}, - ]); - let (value, code) = index.add_documents(documents, None).await; - snapshot!(code, @"202 Accepted"); - - index.wait_task(value.uid()).await.succeeded(); - - let uid = index.uid.clone(); - (server, uid, settings) -} - #[actix_rt::test] async fn experimental_feature_not_enabled() { let server = Server::new().await; From a005a062da374e945b840cccf1938713362f405f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 17 Jul 2025 15:27:53 +0200 Subject: [PATCH 143/312] Add security if chat settings parameters are missing --- crates/meilisearch-types/src/features.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/meilisearch-types/src/features.rs b/crates/meilisearch-types/src/features.rs index 8878a8281..44a0071e4 100644 --- a/crates/meilisearch-types/src/features.rs +++ b/crates/meilisearch-types/src/features.rs @@ -162,10 +162,14 @@ impl ChatCompletionSource { #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "camelCase")] pub struct ChatCompletionPrompts { + #[serde(default)] pub system: String, + #[serde(default)] pub search_description: String, + #[serde(default)] pub search_q_param: String, pub search_filter_param: String, + #[serde(default)] pub search_index_uid_param: String, } From f1d92bfeadd3d4ac36227633754987786ec6163e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 17 Jul 2025 15:28:18 +0200 Subject: [PATCH 144/312] Make sure the new filter chat setting is set to it's default value if missing --- crates/meilisearch-types/src/features.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/meilisearch-types/src/features.rs b/crates/meilisearch-types/src/features.rs index 44a0071e4..ddffb107c 100644 --- a/crates/meilisearch-types/src/features.rs +++ b/crates/meilisearch-types/src/features.rs @@ -168,11 +168,18 @@ pub struct ChatCompletionPrompts { pub search_description: String, #[serde(default)] pub search_q_param: String, + #[serde(default = "default_search_filter_param")] pub search_filter_param: String, #[serde(default)] pub search_index_uid_param: String, } +/// This function is used for when the search_filter_param is +/// not provided and this can happen when the database is in v1.15. +fn default_search_filter_param() -> String { + DEFAULT_CHAT_SEARCH_FILTER_PARAM_PROMPT.to_string() +} + impl Default for ChatCompletionPrompts { fn default() -> Self { Self { From fe15e11c9de7ef9866c04fcd2b07d1d2644aad6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 17 Jul 2025 16:12:23 +0200 Subject: [PATCH 145/312] Introduce a new CLI and env var to use the old document indexer when importing dumps --- crates/index-scheduler/src/lib.rs | 12 +++ .../src/analytics/segment_analytics.rs | 3 + crates/meilisearch/src/lib.rs | 79 ++++++++++--------- crates/meilisearch/src/option.rs | 16 ++++ 4 files changed, 73 insertions(+), 37 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index b2f27d66b..f91e45914 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -139,6 +139,8 @@ pub struct IndexSchedulerOptions { pub embedding_cache_cap: usize, /// Snapshot compaction status. pub experimental_no_snapshot_compaction: bool, + /// Whether dump import use the old document indexer or the new one. + pub experimental_no_edition_2024_for_dumps: bool, } /// Structure which holds meilisearch's indexes and schedules the tasks @@ -168,6 +170,9 @@ pub struct IndexScheduler { /// Whether we should automatically cleanup the task queue or not. pub(crate) cleanup_enabled: bool, + /// Whether we should use the old document indexer or the new one. + pub(crate) experimental_no_edition_2024_for_dumps: bool, + /// The webhook url we should send tasks to after processing every batches. pub(crate) webhook_url: Option, /// The Authorization header to send to the webhook URL. @@ -210,6 +215,7 @@ impl IndexScheduler { index_mapper: self.index_mapper.clone(), cleanup_enabled: self.cleanup_enabled, + experimental_no_edition_2024_for_dumps: self.experimental_no_edition_2024_for_dumps, webhook_url: self.webhook_url.clone(), webhook_authorization_header: self.webhook_authorization_header.clone(), embedders: self.embedders.clone(), @@ -296,6 +302,7 @@ impl IndexScheduler { index_mapper, env, cleanup_enabled: options.cleanup_enabled, + experimental_no_edition_2024_for_dumps: options.experimental_no_edition_2024_for_dumps, webhook_url: options.webhook_url, webhook_authorization_header: options.webhook_authorization_header, embedders: Default::default(), @@ -594,6 +601,11 @@ impl IndexScheduler { Ok(nbr_index_processing_tasks > 0) } + /// Whether the index should use the old document indexer. + pub fn no_edition_2024_for_dumps(&self) -> bool { + self.experimental_no_edition_2024_for_dumps + } + /// Return the tasks matching the query from the user's point of view along /// with the total number of tasks matching the query, ignoring from and limit. /// diff --git a/crates/meilisearch/src/analytics/segment_analytics.rs b/crates/meilisearch/src/analytics/segment_analytics.rs index 0abc5c817..a96ddf068 100644 --- a/crates/meilisearch/src/analytics/segment_analytics.rs +++ b/crates/meilisearch/src/analytics/segment_analytics.rs @@ -203,6 +203,7 @@ struct Infos { experimental_composite_embedders: bool, experimental_embedding_cache_entries: usize, experimental_no_snapshot_compaction: bool, + experimental_no_edition_2024_for_dumps: bool, experimental_no_edition_2024_for_settings: bool, gpu_enabled: bool, db_path: bool, @@ -253,6 +254,7 @@ impl Infos { experimental_limit_batched_tasks_total_size, experimental_embedding_cache_entries, experimental_no_snapshot_compaction, + experimental_no_edition_2024_for_dumps, http_addr, master_key: _, env, @@ -329,6 +331,7 @@ impl Infos { experimental_composite_embedders: composite_embedders, experimental_embedding_cache_entries, experimental_no_snapshot_compaction, + experimental_no_edition_2024_for_dumps, gpu_enabled: meilisearch_types::milli::vector::is_cuda_enabled(), db_path: db_path != PathBuf::from("./data.ms"), import_dump: import_dump.is_some(), diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 43d7afe0e..8907a5632 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -238,6 +238,7 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< auto_upgrade: opt.experimental_dumpless_upgrade, embedding_cache_cap: opt.experimental_embedding_cache_entries, experimental_no_snapshot_compaction: opt.experimental_no_snapshot_compaction, + experimental_no_edition_2024_for_dumps: opt.experimental_no_edition_2024_for_dumps, }; let binary_version = (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH); @@ -553,47 +554,51 @@ fn import_dump( let embedder_stats: Arc = Default::default(); builder.execute(&|| false, &progress, embedder_stats.clone())?; - // 5.3 Import the documents. - // 5.3.1 We need to recreate the grenad+obkv format accepted by the index. - tracing::info!("Importing the documents."); - let file = tempfile::tempfile()?; - let mut builder = DocumentsBatchBuilder::new(BufWriter::new(file)); - for document in index_reader.documents()? { - builder.append_json_object(&document?)?; + if index_scheduler.no_edition_2024_for_dumps() { + // 5.3 Import the documents. + // 5.3.1 We need to recreate the grenad+obkv format accepted by the index. + tracing::info!("Importing the documents."); + let file = tempfile::tempfile()?; + let mut builder = DocumentsBatchBuilder::new(BufWriter::new(file)); + for document in index_reader.documents()? { + builder.append_json_object(&document?)?; + } + + // This flush the content of the batch builder. + let file = builder.into_inner()?.into_inner()?; + + // 5.3.2 We feed it to the milli index. + let reader = BufReader::new(file); + let reader = DocumentsBatchReader::from_reader(reader)?; + + let embedder_configs = index.embedding_configs().embedding_configs(&wtxn)?; + let embedders = index_scheduler.embedders(uid.to_string(), embedder_configs)?; + + let builder = milli::update::IndexDocuments::new( + &mut wtxn, + &index, + indexer_config, + IndexDocumentsConfig { + update_method: IndexDocumentsMethod::ReplaceDocuments, + ..Default::default() + }, + |indexing_step| tracing::trace!("update: {:?}", indexing_step), + || false, + &embedder_stats, + )?; + + let builder = builder.with_embedders(embedders); + + let (builder, user_result) = builder.add_documents(reader)?; + let user_result = user_result?; + tracing::info!(documents_found = user_result, "{} documents found.", user_result); + builder.execute()?; + } else { + unimplemented!("new document indexer when importing dumps"); } - // This flush the content of the batch builder. - let file = builder.into_inner()?.into_inner()?; - - // 5.3.2 We feed it to the milli index. - let reader = BufReader::new(file); - let reader = DocumentsBatchReader::from_reader(reader)?; - - let embedder_configs = index.embedding_configs().embedding_configs(&wtxn)?; - let embedders = index_scheduler.embedders(uid.to_string(), embedder_configs)?; - - let builder = milli::update::IndexDocuments::new( - &mut wtxn, - &index, - indexer_config, - IndexDocumentsConfig { - update_method: IndexDocumentsMethod::ReplaceDocuments, - ..Default::default() - }, - |indexing_step| tracing::trace!("update: {:?}", indexing_step), - || false, - &embedder_stats, - )?; - - let builder = builder.with_embedders(embedders); - - let (builder, user_result) = builder.add_documents(reader)?; - let user_result = user_result?; - tracing::info!(documents_found = user_result, "{} documents found.", user_result); - builder.execute()?; wtxn.commit()?; tracing::info!("All documents successfully imported."); - index_scheduler.refresh_index_stats(&uid)?; } diff --git a/crates/meilisearch/src/option.rs b/crates/meilisearch/src/option.rs index 9658352c8..77106d362 100644 --- a/crates/meilisearch/src/option.rs +++ b/crates/meilisearch/src/option.rs @@ -68,6 +68,8 @@ const MEILI_EXPERIMENTAL_LIMIT_BATCHED_TASKS_TOTAL_SIZE: &str = const MEILI_EXPERIMENTAL_EMBEDDING_CACHE_ENTRIES: &str = "MEILI_EXPERIMENTAL_EMBEDDING_CACHE_ENTRIES"; const MEILI_EXPERIMENTAL_NO_SNAPSHOT_COMPACTION: &str = "MEILI_EXPERIMENTAL_NO_SNAPSHOT_COMPACTION"; +const MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_DUMPS: &str = + "MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_DUMPS"; const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml"; const DEFAULT_DB_PATH: &str = "./data.ms"; const DEFAULT_HTTP_ADDR: &str = "localhost:7700"; @@ -467,6 +469,15 @@ pub struct Opt { #[serde(default)] pub experimental_no_snapshot_compaction: bool, + /// Experimental make dump imports use the old document indexer. + /// + /// When enabled, Meilisearch will use the old document indexer when importing dumps. + /// + /// For more information, see . + #[clap(long, env = MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_DUMPS)] + #[serde(default)] + pub experimental_no_edition_2024_for_dumps: bool, + #[serde(flatten)] #[clap(flatten)] pub indexer_options: IndexerOpts, @@ -572,6 +583,7 @@ impl Opt { experimental_limit_batched_tasks_total_size, experimental_embedding_cache_entries, experimental_no_snapshot_compaction, + experimental_no_edition_2024_for_dumps, } = self; export_to_env_if_not_present(MEILI_DB_PATH, db_path); export_to_env_if_not_present(MEILI_HTTP_ADDR, http_addr); @@ -672,6 +684,10 @@ impl Opt { MEILI_EXPERIMENTAL_NO_SNAPSHOT_COMPACTION, experimental_no_snapshot_compaction.to_string(), ); + export_to_env_if_not_present( + MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_DUMPS, + experimental_no_edition_2024_for_dumps.to_string(), + ); indexer_options.export_to_env(); } From 338806283b1303691843150d6530904ca34fb717 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 17 Jul 2025 16:13:00 +0200 Subject: [PATCH 146/312] Do not track meilisearch databases --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 764447352..44cfa8f75 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ **/*.json_lines **/*.rs.bk /*.mdb -/data.ms +/*.ms /snapshots /dumps /bench From 760ccffdbd2a1bb51b95de3c024e77e09f46f475 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 17 Jul 2025 17:12:18 +0200 Subject: [PATCH 147/312] Expose the documents files from the dumps --- crates/dump/src/reader/compat/v1_to_v2.rs | 5 +++++ crates/dump/src/reader/compat/v2_to_v3.rs | 8 ++++++++ crates/dump/src/reader/compat/v3_to_v4.rs | 9 +++++++++ crates/dump/src/reader/compat/v4_to_v5.rs | 9 +++++++++ crates/dump/src/reader/compat/v5_to_v6.rs | 8 ++++++++ crates/dump/src/reader/mod.rs | 7 +++++++ crates/dump/src/reader/v1/mod.rs | 4 ++++ crates/dump/src/reader/v2/mod.rs | 4 ++++ crates/dump/src/reader/v3/mod.rs | 4 ++++ crates/dump/src/reader/v4/mod.rs | 4 ++++ crates/dump/src/reader/v5/mod.rs | 4 ++++ crates/dump/src/reader/v6/mod.rs | 4 ++++ 12 files changed, 70 insertions(+) diff --git a/crates/dump/src/reader/compat/v1_to_v2.rs b/crates/dump/src/reader/compat/v1_to_v2.rs index 0d050497b..35d369c3a 100644 --- a/crates/dump/src/reader/compat/v1_to_v2.rs +++ b/crates/dump/src/reader/compat/v1_to_v2.rs @@ -1,3 +1,4 @@ +use std::fs::File; use std::str::FromStr; use super::v2_to_v3::CompatV2ToV3; @@ -94,6 +95,10 @@ impl CompatIndexV1ToV2 { self.from.documents().map(|it| Box::new(it) as Box>) } + pub fn documents_file(&self) -> &File { + self.from.documents_file() + } + pub fn settings(&mut self) -> Result> { Ok(v2::settings::Settings::::from(self.from.settings()?).check()) } diff --git a/crates/dump/src/reader/compat/v2_to_v3.rs b/crates/dump/src/reader/compat/v2_to_v3.rs index e7516e708..62326040e 100644 --- a/crates/dump/src/reader/compat/v2_to_v3.rs +++ b/crates/dump/src/reader/compat/v2_to_v3.rs @@ -1,3 +1,4 @@ +use std::fs::File; use std::str::FromStr; use time::OffsetDateTime; @@ -122,6 +123,13 @@ impl CompatIndexV2ToV3 { } } + pub fn documents_file(&self) -> &File { + match self { + CompatIndexV2ToV3::V2(v2) => v2.documents_file(), + CompatIndexV2ToV3::Compat(compat) => compat.documents_file(), + } + } + pub fn settings(&mut self) -> Result> { let settings = match self { CompatIndexV2ToV3::V2(from) => from.settings()?, diff --git a/crates/dump/src/reader/compat/v3_to_v4.rs b/crates/dump/src/reader/compat/v3_to_v4.rs index 5bb70e9b2..1dba37771 100644 --- a/crates/dump/src/reader/compat/v3_to_v4.rs +++ b/crates/dump/src/reader/compat/v3_to_v4.rs @@ -1,3 +1,5 @@ +use std::fs::File; + use super::v2_to_v3::{CompatIndexV2ToV3, CompatV2ToV3}; use super::v4_to_v5::CompatV4ToV5; use crate::reader::{v3, v4, UpdateFile}; @@ -252,6 +254,13 @@ impl CompatIndexV3ToV4 { } } + pub fn documents_file(&self) -> &File { + match self { + CompatIndexV3ToV4::V3(v3) => v3.documents_file(), + CompatIndexV3ToV4::Compat(compat) => compat.documents_file(), + } + } + pub fn settings(&mut self) -> Result> { Ok(match self { CompatIndexV3ToV4::V3(v3) => { diff --git a/crates/dump/src/reader/compat/v4_to_v5.rs b/crates/dump/src/reader/compat/v4_to_v5.rs index e52acb176..3f47b5b48 100644 --- a/crates/dump/src/reader/compat/v4_to_v5.rs +++ b/crates/dump/src/reader/compat/v4_to_v5.rs @@ -1,3 +1,5 @@ +use std::fs::File; + use super::v3_to_v4::{CompatIndexV3ToV4, CompatV3ToV4}; use super::v5_to_v6::CompatV5ToV6; use crate::reader::{v4, v5, Document}; @@ -241,6 +243,13 @@ impl CompatIndexV4ToV5 { } } + pub fn documents_file(&self) -> &File { + match self { + CompatIndexV4ToV5::V4(v4) => v4.documents_file(), + CompatIndexV4ToV5::Compat(compat) => compat.documents_file(), + } + } + pub fn settings(&mut self) -> Result> { match self { CompatIndexV4ToV5::V4(v4) => Ok(v5::Settings::from(v4.settings()?).check()), diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index f7bda81c6..f173bb6bd 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -1,3 +1,4 @@ +use std::fs::File; use std::num::NonZeroUsize; use std::str::FromStr; @@ -243,6 +244,13 @@ impl CompatIndexV5ToV6 { } } + pub fn documents_file(&self) -> &File { + match self { + CompatIndexV5ToV6::V5(v5) => v5.documents_file(), + CompatIndexV5ToV6::Compat(compat) => compat.documents_file(), + } + } + pub fn settings(&mut self) -> Result> { match self { CompatIndexV5ToV6::V5(v5) => Ok(v6::Settings::from(v5.settings()?).check()), diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index 23e7eec9e..91c6d5880 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -192,6 +192,13 @@ impl DumpIndexReader { } } + pub fn documents_file(&self) -> &File { + match self { + DumpIndexReader::Current(v6) => v6.documents_file(), + DumpIndexReader::Compat(compat) => compat.documents_file(), + } + } + pub fn settings(&mut self) -> Result> { match self { DumpIndexReader::Current(v6) => v6.settings(), diff --git a/crates/dump/src/reader/v1/mod.rs b/crates/dump/src/reader/v1/mod.rs index ac7324d9a..d86ede62c 100644 --- a/crates/dump/src/reader/v1/mod.rs +++ b/crates/dump/src/reader/v1/mod.rs @@ -72,6 +72,10 @@ impl V1IndexReader { .map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })) } + pub fn documents_file(&self) -> &File { + self.documents.get_ref() + } + pub fn settings(&mut self) -> Result { Ok(serde_json::from_reader(&mut self.settings)?) } diff --git a/crates/dump/src/reader/v2/mod.rs b/crates/dump/src/reader/v2/mod.rs index 14a643c2d..a74687381 100644 --- a/crates/dump/src/reader/v2/mod.rs +++ b/crates/dump/src/reader/v2/mod.rs @@ -203,6 +203,10 @@ impl V2IndexReader { .map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })) } + pub fn documents_file(&self) -> &File { + self.documents.get_ref() + } + pub fn settings(&mut self) -> Result> { Ok(self.settings.clone()) } diff --git a/crates/dump/src/reader/v3/mod.rs b/crates/dump/src/reader/v3/mod.rs index 920e1dc6e..5f89eb861 100644 --- a/crates/dump/src/reader/v3/mod.rs +++ b/crates/dump/src/reader/v3/mod.rs @@ -215,6 +215,10 @@ impl V3IndexReader { .map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })) } + pub fn documents_file(&self) -> &File { + self.documents.get_ref() + } + pub fn settings(&mut self) -> Result> { Ok(self.settings.clone()) } diff --git a/crates/dump/src/reader/v4/mod.rs b/crates/dump/src/reader/v4/mod.rs index 585786ae4..16a1e27c2 100644 --- a/crates/dump/src/reader/v4/mod.rs +++ b/crates/dump/src/reader/v4/mod.rs @@ -210,6 +210,10 @@ impl V4IndexReader { .map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })) } + pub fn documents_file(&self) -> &File { + self.documents.get_ref() + } + pub fn settings(&mut self) -> Result> { Ok(self.settings.clone()) } diff --git a/crates/dump/src/reader/v5/mod.rs b/crates/dump/src/reader/v5/mod.rs index dfbc6346c..0123db433 100644 --- a/crates/dump/src/reader/v5/mod.rs +++ b/crates/dump/src/reader/v5/mod.rs @@ -247,6 +247,10 @@ impl V5IndexReader { .map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })) } + pub fn documents_file(&self) -> &File { + self.documents.get_ref() + } + pub fn settings(&mut self) -> Result> { Ok(self.settings.clone()) } diff --git a/crates/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs index 449a7e5fe..08d4700e5 100644 --- a/crates/dump/src/reader/v6/mod.rs +++ b/crates/dump/src/reader/v6/mod.rs @@ -284,6 +284,10 @@ impl V6IndexReader { .map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })) } + pub fn documents_file(&self) -> &File { + self.documents.get_ref() + } + pub fn settings(&mut self) -> Result> { let mut settings: Settings = serde_json::from_reader(&mut self.settings)?; patch_embedders(&mut settings); From d67db6e3c2a97acb956c55ccab4f38b1dd988212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 17 Jul 2025 17:12:42 +0200 Subject: [PATCH 148/312] Use the edition 2024 documents indexer in the dumps --- Cargo.lock | 5 ++-- crates/meilisearch/Cargo.toml | 1 + crates/meilisearch/src/lib.rs | 54 +++++++++++++++++++++++++++++++++-- 3 files changed, 56 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ceec0a05e..8413b3d14 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3775,6 +3775,7 @@ dependencies = [ "meili-snap", "meilisearch-auth", "meilisearch-types", + "memmap2", "mimalloc", "mime", "mopa-maintained", @@ -3908,9 +3909,9 @@ checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" [[package]] name = "memmap2" -version = "0.9.5" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +checksum = "483758ad303d734cec05e5c12b41d7e93e6a6390c5e9dae6bdeb7c1259012d28" dependencies = [ "libc", "stable_deref_trait", diff --git a/crates/meilisearch/Cargo.toml b/crates/meilisearch/Cargo.toml index 83eb439d9..21f6b58e5 100644 --- a/crates/meilisearch/Cargo.toml +++ b/crates/meilisearch/Cargo.toml @@ -50,6 +50,7 @@ jsonwebtoken = "9.3.1" lazy_static = "1.5.0" meilisearch-auth = { path = "../meilisearch-auth" } meilisearch-types = { path = "../meilisearch-types" } +memmap2 = "0.9.7" mimalloc = { version = "0.1.47", default-features = false } mime = "0.3.17" num_cpus = "1.17.0" diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 8907a5632..57a20a633 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -30,6 +30,7 @@ use actix_web::web::Data; use actix_web::{web, HttpRequest}; use analytics::Analytics; use anyhow::bail; +use bumpalo::Bump; use error::PayloadError; use extractors::payload::PayloadConfig; use index_scheduler::versioning::Versioning; @@ -38,6 +39,7 @@ use meilisearch_auth::{open_auth_store_env, AuthController}; use meilisearch_types::milli::constants::VERSION_MAJOR; use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader}; use meilisearch_types::milli::progress::{EmbedderStats, Progress}; +use meilisearch_types::milli::update::new::indexer; use meilisearch_types::milli::update::{ default_thread_pool_and_threads, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, }; @@ -534,7 +536,7 @@ fn import_dump( let mut index_reader = index_reader?; let metadata = index_reader.metadata(); let uid = metadata.uid.clone(); - tracing::info!("Importing index `{}`.", metadata.uid); + tracing::info!("Importing index `{uid}`."); let date = Some((metadata.created_at, metadata.updated_at)); let index = index_scheduler.create_raw_index(&metadata.uid, date)?; @@ -553,6 +555,10 @@ fn import_dump( apply_settings_to_builder(&settings, &mut builder); let embedder_stats: Arc = Default::default(); builder.execute(&|| false, &progress, embedder_stats.clone())?; + wtxn.commit()?; + + let mut wtxn = index.write_txn()?; + let rtxn = index.read_txn()?; if index_scheduler.no_edition_2024_for_dumps() { // 5.3 Import the documents. @@ -594,7 +600,51 @@ fn import_dump( tracing::info!(documents_found = user_result, "{} documents found.", user_result); builder.execute()?; } else { - unimplemented!("new document indexer when importing dumps"); + let db_fields_ids_map = index.fields_ids_map(&rtxn)?; + let primary_key = index.primary_key(&rtxn)?; + let mut new_fields_ids_map = db_fields_ids_map.clone(); + + let mut indexer = indexer::DocumentOperation::new(); + let embedders = index.embedding_configs().embedding_configs(&mut wtxn)?; + let embedders = index_scheduler.embedders(uid.clone(), embedders)?; + + let mmap = unsafe { memmap2::Mmap::map(index_reader.documents_file())? }; + + indexer.replace_documents(&mmap)?; + + let indexer_config = index_scheduler.indexer_config(); + let pool = &indexer_config.thread_pool; + + let indexer_alloc = Bump::new(); + let (document_changes, mut operation_stats, primary_key) = indexer.into_changes( + &indexer_alloc, + &index, + &rtxn, + primary_key, + &mut new_fields_ids_map, + &|| false, // never stop processing a dump + progress.clone(), + )?; + + let operation_stats = operation_stats.pop().unwrap(); + if let Some(error) = operation_stats.error { + return Err(error.into()); + } + + let _congestion = indexer::index( + &mut wtxn, + &index, + pool, + indexer_config.grenad_parameters(), + &db_fields_ids_map, + new_fields_ids_map, + primary_key, + &document_changes, + embedders, + &|| false, // never stop processing a dump + &progress, + &embedder_stats, + )?; } wtxn.commit()?; From a1b42c10e2fceadd07f39ca1cac547a99053a8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 17 Jul 2025 17:21:03 +0200 Subject: [PATCH 149/312] Make clippy happy --- crates/index-scheduler/src/insta_snapshot.rs | 1 + crates/index-scheduler/src/test_utils.rs | 1 + crates/meilisearch/src/lib.rs | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index 0cbbb2514..32ce131b5 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -20,6 +20,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { let IndexScheduler { cleanup_enabled: _, + experimental_no_edition_2024_for_dumps: _, processing_tasks, env, version, diff --git a/crates/index-scheduler/src/test_utils.rs b/crates/index-scheduler/src/test_utils.rs index bfed7f53a..0a705b6c7 100644 --- a/crates/index-scheduler/src/test_utils.rs +++ b/crates/index-scheduler/src/test_utils.rs @@ -115,6 +115,7 @@ impl IndexScheduler { auto_upgrade: true, // Don't cost much and will ensure the happy path works embedding_cache_cap: 10, experimental_no_snapshot_compaction: false, + experimental_no_edition_2024_for_dumps: false, }; let version = configuration(&mut options).unwrap_or({ (versioning::VERSION_MAJOR, versioning::VERSION_MINOR, versioning::VERSION_PATCH) diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 57a20a633..13d2eb789 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -605,7 +605,7 @@ fn import_dump( let mut new_fields_ids_map = db_fields_ids_map.clone(); let mut indexer = indexer::DocumentOperation::new(); - let embedders = index.embedding_configs().embedding_configs(&mut wtxn)?; + let embedders = index.embedding_configs().embedding_configs(&rtxn)?; let embedders = index_scheduler.embedders(uid.clone(), embedders)?; let mmap = unsafe { memmap2::Mmap::map(index_reader.documents_file())? }; From 1b476b8a35655283840cca3f37ce1af691d3feb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 17 Jul 2025 17:26:41 +0200 Subject: [PATCH 150/312] Add documentation to the new documents_file dump reader method Co-authored-by: Louis Dureuil --- crates/dump/src/reader/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index 91c6d5880..c894c255f 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -192,6 +192,7 @@ impl DumpIndexReader { } } + /// A reference to a file in the NDJSON format containing all the documents of the index pub fn documents_file(&self) -> &File { match self { DumpIndexReader::Current(v6) => v6.documents_file(), From 626be0ef28fef7e71ad8628d95f0b76d44509b61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 17 Jul 2025 17:27:00 +0200 Subject: [PATCH 151/312] Small typo fix Co-authored-by: Louis Dureuil --- crates/index-scheduler/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index f91e45914..8715bc100 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -139,7 +139,7 @@ pub struct IndexSchedulerOptions { pub embedding_cache_cap: usize, /// Snapshot compaction status. pub experimental_no_snapshot_compaction: bool, - /// Whether dump import use the old document indexer or the new one. + /// Whether dump import uses the old document indexer or the new one. pub experimental_no_edition_2024_for_dumps: bool, } From b85657de1eb075a5b5160dfcbec721237856feeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Thu, 17 Jul 2025 17:29:59 +0200 Subject: [PATCH 152/312] Update memmap2 version everywhere --- crates/benchmarks/Cargo.toml | 3 +-- crates/index-scheduler/Cargo.toml | 2 +- crates/meilisearch-types/Cargo.toml | 2 +- crates/milli/Cargo.toml | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/crates/benchmarks/Cargo.toml b/crates/benchmarks/Cargo.toml index 68ed5aff4..f60f0979c 100644 --- a/crates/benchmarks/Cargo.toml +++ b/crates/benchmarks/Cargo.toml @@ -14,7 +14,7 @@ license.workspace = true anyhow = "1.0.98" bumpalo = "3.18.1" csv = "1.3.1" -memmap2 = "0.9.5" +memmap2 = "0.9.7" milli = { path = "../milli" } mimalloc = { version = "0.1.47", default-features = false } serde_json = { version = "1.0.140", features = ["preserve_order"] } @@ -55,4 +55,3 @@ harness = false [[bench]] name = "sort" harness = false - diff --git a/crates/index-scheduler/Cargo.toml b/crates/index-scheduler/Cargo.toml index de0d01935..20cc49686 100644 --- a/crates/index-scheduler/Cargo.toml +++ b/crates/index-scheduler/Cargo.toml @@ -26,7 +26,7 @@ flate2 = "1.1.2" indexmap = "2.9.0" meilisearch-auth = { path = "../meilisearch-auth" } meilisearch-types = { path = "../meilisearch-types" } -memmap2 = "0.9.5" +memmap2 = "0.9.7" page_size = "0.6.0" rayon = "1.10.0" roaring = { version = "0.10.12", features = ["serde"] } diff --git a/crates/meilisearch-types/Cargo.toml b/crates/meilisearch-types/Cargo.toml index faf59643f..f3279a094 100644 --- a/crates/meilisearch-types/Cargo.toml +++ b/crates/meilisearch-types/Cargo.toml @@ -24,7 +24,7 @@ enum-iterator = "2.1.0" file-store = { path = "../file-store" } flate2 = "1.1.2" fst = "0.4.7" -memmap2 = "0.9.5" +memmap2 = "0.9.7" milli = { path = "../milli" } roaring = { version = "0.10.12", features = ["serde"] } rustc-hash = "2.1.1" diff --git a/crates/milli/Cargo.toml b/crates/milli/Cargo.toml index 3d08252ac..d94a4d4e1 100644 --- a/crates/milli/Cargo.toml +++ b/crates/milli/Cargo.toml @@ -40,7 +40,7 @@ indexmap = { version = "2.9.0", features = ["serde"] } json-depth-checker = { path = "../json-depth-checker" } levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] } memchr = "2.7.5" -memmap2 = "0.9.5" +memmap2 = "0.9.7" obkv = "0.3.0" once_cell = "1.21.3" ordered-float = "5.0.0" From c2c82be5563215cfee2f3a93bde6f96b227f5428 Mon Sep 17 00:00:00 2001 From: curquiza Date: Sun, 20 Jul 2025 22:07:23 +0200 Subject: [PATCH 153/312] Update documentation --- CONTRIBUTING.md | 38 ++++++++---- README.md | 2 +- documentation/experimental-features.md | 83 ++++++++++++++++++++++++++ documentation/prototypes.md | 74 +++++++++++++++++++++++ documentation/release.md | 75 +++++++++++++++++++++++ documentation/versioning-policy.md | 83 ++++++++++++++++++++++++++ 6 files changed, 344 insertions(+), 11 deletions(-) create mode 100644 documentation/experimental-features.md create mode 100644 documentation/prototypes.md create mode 100644 documentation/release.md create mode 100644 documentation/versioning-policy.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 57d52116e..72a91a765 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -106,7 +106,13 @@ Run `cargo xtask --help` from the root of the repository to find out what is ava #### Update the openAPI file if the API changed To update the openAPI file in the code, see [sprint_issue.md](https://github.com/meilisearch/meilisearch/blob/main/.github/ISSUE_TEMPLATE/sprint_issue.md#reminders-when-modifying-the-api). -If you want to update the openAPI file on the [open-api repository](https://github.com/meilisearch/open-api), see [update-openapi-issue.md](https://github.com/meilisearch/engine-team/blob/main/issue-templates/update-openapi-issue.md). + +If you want to update the openAPI file on the [open-api repository](https://github.com/meilisearch/open-api): +- Pull the latest version of the latest rc of Meilisearch `git checkout release-vX.Y.Z; git pull` +- Starts Meilisearch with the `swagger` feature flag: `cargo run --features swagger` +- On a browser, open the following URL: http://localhost:7700/scalar +- Click the « Download openAPI file » +- Open a PR replacing [this file](https://github.com/meilisearch/open-api/blob/main/open-api.json) with the one downloaded ### Logging @@ -160,25 +166,37 @@ Some notes on GitHub PRs: The draft PRs are recommended when you want to show that you are working on something and make your work visible. - The branch related to the PR must be **up-to-date with `main`** before merging. Fortunately, this project uses [GitHub Merge Queues](https://github.blog/news-insights/product-news/github-merge-queue-is-generally-available/) to automatically enforce this requirement without the PR author having to rebase manually. -## Release Process (for internal team only) - -Meilisearch tools follow the [Semantic Versioning Convention](https://semver.org/). - -### Automation to rebase and Merge the PRs +## Merging PRs This project uses GitHub Merge Queues that helps us manage pull requests merging. -### How to Publish a new Release +Before merging a PR, the maintainer should ensure the following requirements are met +- Automated tests have been added. +- If some tests cannot be automated, manual rigorous tests should be applied. +- ⚠️ If there is an change in the DB: it's mandatory to manually test the `--experimental-dumpless-upgrade` on a DB of the previous Meilisearch minor version (e.g. v1.13 for the v1.14 release). +- If necessary, the feature have been tested in the Cloud production environment (with [prototypes](./documentation/prototypes.md)) and the Cloud UI is ready. +- If necessary, the [documentation](https://github.com/meilisearch/documentation) related to the implemented feature in the PR is ready. +- If necessary, the [integrations](https://github.com/meilisearch/integration-guides) related to the implemented feature in the PR are ready. -The full Meilisearch release process is described in [this guide](https://github.com/meilisearch/engine-team/blob/main/resources/meilisearch-release.md). Please follow it carefully before doing any release. +## Publish Process (for internal team only) + +Meilisearch tools follow the [Semantic Versioning Convention](https://semver.org/). + +### How to publish a new release + +The full Meilisearch release process is described in [this guide](./documentation/release.md). ### How to publish a prototype Depending on the developed feature, you might need to provide a prototyped version of Meilisearch to make it easier to test by the users. This happens in two steps: -- [Release the prototype](https://github.com/meilisearch/engine-team/blob/main/resources/prototypes.md#how-to-publish-a-prototype) -- [Communicate about it](https://github.com/meilisearch/engine-team/blob/main/resources/prototypes.md#communication) +- [Release the prototype](./documentation/prototypes.md#how-to-publish-a-prototype) +- [Communicate about it](./documentation/prototypes.md#communication) + +### How to implement and publish an experimental feature + +Here is our [guidelines and process](./documentation/experimental-features.md) to implement and publish an experimental feature. ### Release assets diff --git a/README.md b/README.md index 77eecde25..40833a0d6 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,6 @@ Meilisearch is, and will always be, open-source! If you want to contribute to th Meilisearch releases and their associated binaries are available on the project's [releases page](https://github.com/meilisearch/meilisearch/releases). -The binaries are versioned following [SemVer conventions](https://semver.org/). To know more, read our [versioning policy](https://github.com/meilisearch/engine-team/blob/main/resources/versioning-policy.md). +The binaries are versioned following [SemVer conventions](https://semver.org/). To know more, read our [versioning policy](./documentation/versioning-policy.md). Differently from the binaries, crates in this repository are not currently available on [crates.io](https://crates.io/) and do not follow [SemVer conventions](https://semver.org). diff --git a/documentation/experimental-features.md b/documentation/experimental-features.md new file mode 100644 index 000000000..64e884ee1 --- /dev/null +++ b/documentation/experimental-features.md @@ -0,0 +1,83 @@ +# Experimental features: description and process + +## Quick definition of experimental features + +An experimental feature is a feature present in the final Meilisearch binary that is not considered stable. This means the API might become incompatible between two Meilisearch releases. + +Experimental features must be explicitly enabled by a user. + +> ⚠️ Experimental features are NOT [prototypes](./prototypes.md). All experimental features are thoroughly tested before release and follow the same quality standards as other features. + +## Motivation + +Since the release of v1, Meilisearch is considered a stable binary and its API cannot break between minor and patch versions. This means it is impossible to make breaking changes to a feature without releasing a major version. + +This limitation, which guarantees our users Meilisearch is a stable and reliable product, also applies to new features. If we introduce a new feature in one release, any breaking changes will require a new major release. + +To prevent frequently releasing new major versions but still continue to develop new features, we will first provide these features as "experimental". This allows users to test them, report implementation issues, and give us important feedback. + +## When is a feature considered experimental? + +Not all new features need to go through the experimental feature process. + +We will treat features as experimental when: + +- New features we are considering adding to the search engine, but need user feedback before making our final decision and/or committing to a specific implementation. Example: a new API route or CLI flag +- Improvements to existing functionality the engine team is not comfortable releasing as stable immediately. Example: changes to search relevancy or performance improvements +- New features that would introduce breaking changes and cannot be integrated as stable before a new major version +- New features that will NEVER be stable. These features are useful to provide quick temporary fixes to critical issues. Example: an option to disable auto-batching + +## How to enable experimental features? + +Users must explicitly enable experimental features with a CLI flag. Experimental features will always be disabled by default. + +Example CLI flags: `--experimental-disable-soft-delete`, `--experimental-multi-index-search`. + +⚠️ To ensure users understand a feature is experimental, flags must contain the `experimental` prefix. + +## Rules and expectations + +- The API and behavior of an experimental feature can break between two minor versions of Meilisearch +- The experimental feature process described here can significantly change between 2 minor versions of Meilisearch +- Providing a feature as “experimental” does not guarantee it will be stable one day: the newly introduced experimental features or improvements may be removed in a future release +- While experimental features are supposed to be unstable regarding usage and compatibility between versions, users should not expect any more bugs or issues than with any other Meilisearch feature. Experimental features should follow the same quality standards of stable features, including thorough testing suites and in-depth code reviews. That said, certain experimental features might be inherently more prone to bugs and downgrades + +## Communication with users + +For each new experimental feature, we must: +- GitHub: open a dedicated GitHub discussion in the [product repository](https://github.com/meilisearch/product/discussions). This discussion should never become stale and be updated regularly. Users need to understand they can interact with us and get quick answers. The discussion should inform users about: + - Our motivations: why this feature is unstable? + - Usage: how to activate this feature? Do we need to do a migration with a dump? + - Planning: what are the conditions to make this feature stable? When do we expect it become stable? +- Meilisearch CLI: update the `--help` command in the Meilisearch binary so it redirects users to the related GitHub discussion and warns them about the unstable state of the features +- Documentation: create a small dedicated page about the purpose of the experimental feature. This page should contain no usage instructions and redirect users to the related GitHub discussion for more information + +## Usage warnings + +- API can break between 2 versions of Meilisearch. People using the experimental feature in production should pay extra attention to it. +- Some experimental features might require re-indexing. In these cases, users will have to use a dump to activate and deactivate an experimental feature. Users will be clearly informed about this in the related GitHub discussion + +> ⚠️ Since this process is not mature yet, users might experience issues with their DB when deactivating these features even when using a dump.
+> We recommend users always save their data (with snapshots and/or dumps) before activating experimental features. + +## Technical details + +### Why does Meilisearch need to be restarted when activating an experimental feature? + +Meilisearch uses LMDB to store both documents and internal application data, such as Meilisearch tasks. Altering these internal data structures requires closing and re-opening the LMDB environment. + +If an experimental feature implementation involves a modification of internal data structures, users must restart Meilisearch. This cannot be done via HTTP routes. + +Unfortunately, this might impact most experimental features. However, this might change in the future, or adapted to the context of a specific new feature. + +### Why will some features require migrating data with dumps? + +Under some circumstances, Meilisearch might have issues when reading a database generated by a different Meilisearch release. This might cause an instance to crash or work with faulty data. + +This is already a possibility when migrating between minor Meilisearch versions, and is more likely to happen when activating a new experimental feature. The opposite operation—migrating a database with experimental features activated to a database where those features are not active—is currently riskier. As we develop and improve the development of experimental features, this procedure will become safer and more reliable. + +### Restarting Meilisearch and migrating databases with dumps to activate an experimental feature is inconvenient. Will this improve in the future? + +We understand the situation is inconvenient and less than ideal. We will only ask users to use dumps when activating experimental features when it’s strictly necessary. + +Avoiding restarts is more difficult, especially for features that currently require database migrations with dumps. We are not currently working on this, but the situation might change in the future. diff --git a/documentation/prototypes.md b/documentation/prototypes.md new file mode 100644 index 000000000..047f22e7b --- /dev/null +++ b/documentation/prototypes.md @@ -0,0 +1,74 @@ +# Prototype process + +## What is a prototype? + +A prototype is an alternative version of Meilisearch (provided in a Docker image) containing a new feature or an improvement the engine team provides to the users. + +## Why providing a prototype? + +For some features or improvements we want to introduce in Meilisearch, we also have to make the users test them first before releasing them for many reasons: +- to ensure we solve the first use case defined during the discovery +- to ensure the API does not have major issues of usages +- identify/remove concrete technical roadblocks by working on an implementation as soon as possible, like performance issues +- to get any other feedback from the users regarding their usage + +These make us iterate fast before stabilizing it for the current release. + +> ⚠️ Prototypes are NOT [experimental features](./experimental-features.md). All experimental features are thoroughly tested before release and follow the same quality standards as other features. This is not the case with prototypes which are the equivalent of a first draft of a new feature. + +## How to publish a prototype? + +### Release steps + +The prototype name must follow this convention: `prototype-X-Y` where +- `X` is the feature name formatted in `kebab-case`. It should not end with a single number. +- `Y` is the version of the prototype, starting from `0`. + +✅ Example: `prototype-auto-resize-0`.
+❌ Bad example: `auto-resize-0`: lacks the `prototype` prefix.
+❌ Bad example: `prototype-auto-resize`: lacks the version suffix.
+❌ Bad example: `prototype-auto-resize-0-0`: feature name ends with a single number. + +Steps to create a prototype: + +1. In your terminal, go to the last commit of your branch (the one you want to provide as a prototype). +2. Create a tag following the convention: `git tag prototype-X-Y` +3. Run Meilisearch and check that its launch summary features a line: `Prototype: prototype-X-Y` (you may need to switch branches and back after tagging for this to work). +3. Push the tag: `git push origin prototype-X-Y` +4. Check the [Docker CI](https://github.com/meilisearch/meilisearch/actions/workflows/publish-docker-images.yml) is now running. + +🐳 Once the CI has finished to run (~1h30), a Docker image named `prototype-X-Y` will be available on [DockerHub](https://hub.docker.com/repository/docker/getmeili/meilisearch/general). People can use it with the following command: `docker run -p 7700:7700 -v $(pwd)/meili_data:/meili_data getmeili/meilisearch:prototype-X-Y`.
+More information about [how to run Meilisearch with Docker](https://docs.meilisearch.com/learn/cookbooks/docker.html#download-meilisearch-with-docker). + +⚠️ However, no binaries will be created. If the users do not use Docker, they can go to the `prototype-X-Y` tag in the Meilisearch repository and compile it from the source code. + +### Communication + +When sharing a prototype with users, it's important to +- remind them not to use it in production. Prototypes are solely for test purposes. +- explain how to run the prototype +- explain how to use the new feature +- encourage users to let their feedback + +The prototype should be shared at least in the related issue and/or the related product discussion. It's the developer and the PM to decide to add more communication, like sharing it on Discord or Twitter. + +Here is an example of messages to share on GitHub: + +> Hello everyone, +> +> Here is the current prototype you can use to test the new XXX feature: +> +> How to run the prototype? +> You need to start from a fresh new database (remove the previous used `data.ms`) and use the following Docker image: +> ```bash +> docker run -it --rm -p 7700:7700 -v $(pwd)/meili_data:/meili_data getmeili/meilisearch:prototype-X-Y +> ``` +> +> You can use the feature this way: +> ```bash +> ... +> ``` +> +> ⚠️ We do NOT recommend using this prototype in production. This is only for test purposes. +> +> Everyone is more than welcome to give feedback and to report any issue or bug you might encounter when using this prototype. Thanks in advance for your involvement. It means a lot to us ❤️ diff --git a/documentation/release.md b/documentation/release.md new file mode 100644 index 000000000..4d8bb0016 --- /dev/null +++ b/documentation/release.md @@ -0,0 +1,75 @@ +# Meilisearch release process + +This guide is to describe how to make releases for the current repository. + +## 📅 Weekly Meilisearch release + +1. A weekly meeting is done every Monday to define the release and check the following information +

 + +1. Go to the GitHub interface, in the [`Release` section](https://github.com/meilisearch/meilisearch/releases). + +2. Select the already drafted release or click on the `Draft a new release` button if you want to start a blank one, and fill the form with the appropriate information. +⚠️ Publish on `main` + +⚙️ The CIs will be triggered to: +- [Upload binaries](https://github.com/meilisearch/meilisearch/actions/workflows/publish-binaries.yml) to the associated GitHub release. +- [Publish the Docker images](https://github.com/meilisearch/meilisearch/actions/workflows/publish-docker-images.yml) (`latest`, `vX`, `vX.Y` and `vX.Y.Z`) to DockerHub -> check the "Docker meta" steps in the CI to check the right tags are created +- [Publish binaries for Homebrew and APT](https://github.com/meilisearch/meilisearch/actions/workflows/publish-apt-brew-pkg.yml) +- [Move the `latest` git tag to the release commit](https://github.com/meilisearch/meilisearch/actions/workflows/latest-git-tag.yml). + + +### 🔥 How to do a patch release for an hotfix + +It happens some releases come with impactful bugs in production (e.g. indexation or search issues): we obviously don't wait for the next cycle to fix them and we release a patched version of Meilisearch. + +1. Create a new release branch starting from the latest stable Meilisearch release (`latest` git tag or the corresponding `vX.Y.Z` tag). + +```bash +# Ensure you get all the current tags of the repository +git fetch origin --tags --force + +# Create the branch +git checkout vX.Y.Z # The latest release you want to patch +git checkout -b release-vX.Y.Z+1 # Increase the Z here +git push -u origin release-vX.Y.Z+1 +``` + +2. Change the [version in `Cargo.toml` file](https://github.com/meilisearch/meilisearch/blob/e9b62aacb38f2c7a777adfda55293d407e0d6254/Cargo.toml#L21). You can use [our automation](https://github.com/meilisearch/meilisearch/actions/workflows/update-cargo-toml-version.yml) -> click on `Run workflow` -> Fill the appropriate version and run it on the newly created branch `release-vX.Y.Z` -> Click on "Run workflow". A PR updating the version in the `Cargo.toml` and `Cargo.lock` files will be created. + +3. Open and merge the PRs (fixing your bugs): they should point to `release-vX.Y.Z+1` branch. + +4. Go to the GitHub interface, in the [`Release` section](https://github.com/meilisearch/meilisearch/releases) and click on `Draft a new release` + ⚠️⚠️⚠️ Publish on `release-vX.Y.Z+1` branch, not on `main`! + +⚠️ If doing a patch release that should NOT be the `latest` release: + +- Do NOT check `Set as the latest release` when creating the GitHub release. If you did, quickly interrupt all CIs and delete the GitHub release! +- Once the release is created, you don't have to care about Homebrew, APT and Docker CIs: they will not consider this new release as the latest; the CIs are already adapted for this situation. +- However, the [CI updating the `latest` git tag](https://github.com/meilisearch/meilisearch/actions/workflows/latest-git-tag.yml) is not working for this situation currently and will attach the `latest` git tag to the just-created release, which is something we don't want! If you don't succeed in stopping the CI on time, don't worry, you just have to re-run the [old CI](https://github.com/meilisearch/meilisearch/actions/workflows/latest-git-tag.yml) corresponding to the real latest release, and the `latest` git tag will be attached back to the right commit. + +5. Bring the new commits back from `release-vX.Y.Z+1` to `main` by merging a PR originating `release-vX.Y.Z+1` and pointing to `main`. + +⚠️ If you encounter any merge conflicts, please do NOT fix the git conflicts directly on the `release-vX.Y.Z` branch. It would bring the changes present in `main` into `release-vX.Y.Z`, which would break a potential future patched release. + +![GitHub interface showing merge conflicts](../assets/merge-conflicts.png) + +Instead: +- Create a new branch originating `release-vX.Y.Z+1`, like `tmp-release-vX.Y.Z+1` +- Create a PR from the `tmp-release-vX.Y.Z+1` branch and pointing to `main` +- Fix the git conflicts on this new branch + - By either fixing the git conflict via the GitHub interface + - By pulling the `main` branch into `tmp-release-vX.Y.Z+1` and fixing them on your machine. +- Merge this new PR into `main` diff --git a/documentation/versioning-policy.md b/documentation/versioning-policy.md new file mode 100644 index 000000000..eda02137a --- /dev/null +++ b/documentation/versioning-policy.md @@ -0,0 +1,83 @@ +# Versioning policy + +This page describes the versioning rules Meilisearch will follow once v1.0.0 is released and how/when we should increase the MAJOR, MINOR, and PATCH of the versions. + +## 🤖 Basic rules + +Meilisearch engine releases follow the [SemVer rules](https://semver.org/), including the following basic ones: + +> 🔥 Given a version number MAJOR.MINOR.PATCH, increment the: +> +> 1. MAJOR version when you make incompatible API changes +> 2. MINOR version when you add functionality in a backwards compatible +> manner +> 3. PATCH version when you make backwards compatible bug fixes + +**Changes that MAY lead the Meilisearch users (developers) to change their code are considered API incompatibility and will make us increase the MAJOR version of Meilisearch.** + +**In other terms, if the users MAY have to do more steps than just downloading the new Meilisearch instance and running it, a new MAJOR is needed.** + +Examples of changes making the code break and then, involving increasing the MAJOR: + +- Name change of a route or a field in the request/response body +- Change a default value of a parameter or a setting. +- Any API behavior change: the users expect in their code the engine to behave this way, but it does not. +Examples: + - Make a synchronous error asynchronous or the contrary + - `displayableAttributes` impact now the `/documents` route: the users expect to retrieve all the fields, so specific fields, in their code but cannot. +- Change a final value type. +Ex: `/stats` now return floats instead of integers. This can impact strongly typed languages. + +⚠️ This guide only applies to the Meilisearch binary. Additional tools like SDKs and Docker images are out of the scope of this guide. However, we will ensure the changelogs are clear enough to inform users of the changes and their impacts. + +## ✋ Exceptions related to Meilisearch’s specificities + +Meilisearch is a search engine working with an internal database. It means some parts of the project can be really problematic to consider as breaking (and then leading to an increase of the MAJOR) without slowing down innovation. + +Here is the list of the following exceptions of changes that will not lead to an increase in the MAJOR in Meilisearch release. + +### DB incompatibilities: force using a dump + +A DB breaking leads to a failure when starting Meilisearch: you need to use a dump. + +We know this kind of failure requiring an additional step is the definition of “breaking” on the user side, but it’s really complicated to consider increasing a MAJOR for this. Indeed, since we don’t want to release a major version every two months and we also want to keep innovating simultaneously, increasing the MINOR is the best solution. + +People would need to use dump sometimes between two MAJOR versions; for instance, this is something [PostgreSQL does](https://www.postgresql.org/support/versioning/) by asking their users to perform some manual actions between two MINOR releases. + +### Search relevancy and algorithm improvements + +Relevancy is the engine team job; we need to improve it every day, like performance. It will be really hard to improve the engine without allowing the team to change the relevancy algorithm. Same as for DB breaking, considering relevancy changes as breaking can really slow down innovation. + +This way, changing the search relevancy, not the API behavior or fields, but the final relevancy result (like cropping algorithm, search algorithm, placeholder behavior, highlight behavior…) is not considered as a breaking change. Indeed, changing the relevancy behavior is not supposed to make the code fail since the final results of Meilisearch are only displayed, no matter the matched documents. + +This kind of change will lead us to increase the MINOR to let the people know about the change and avoid non-expected changes when pulling the latest patched version of Meilisearch. Indeed, increasing the MINOR (instead of the PATCH) will prevent users from downloading the new patched version without noticing the changes. + +🚨 Any change about the relevancy that is related to API usage, and thus, that may impact users to change their code (for instance changing the default `matchingStrategy` value) is not related to this specific section and would lead us to increase the MAJOR. + +### New "variant" type addition + +We don't consider breaking to add a new type to an already existing list of variant. For example, adding a new type of `task`, or a new type of error `code`. + +We are aware some strongly typed language code bases could be impacted, and our recommendation is to handle the possibility of having an unknown type when deserializing Meilisearch's response. + +### Human-readability purposes + +- Changing the value of `message` or `link` in error object will only increase the PATCH. The users should not refer to this field in their code since `code` and `type` exist in the same object. +- Any error message sent to the terminal that changed will increase the PATCH. People should not rely on them since these messages are for human debugging. +- Updating the logs format will increase the MINOR: this is supposed to be used by humans for debugging, but we are aware some people can plug some tools at the top of them. But since it’s not the main purpose of our logs, we don’t want to increase the MAJOR for a log format change. However, we will increase the MINOR to let the people know better about the change and avoid bad surprises when pulling the latest patched version of Meilisearch. + +### Integrated web-interface + +Any changes done to the integrated web interface are not considered breaking. The interface is considered an additional tool for test purposes, not for production. + +## 📝 About the Meilisearch changelogs + +All the changes, no matter if they are considered as breaking or not, if they are related to an algorithm change or not, will be announced in the changelogs. + +The details of the change will depend on the impact on the users. For instance, giving too many details on really deep tech improvements can lead to some confusion on the user side. + +## 👀 Some precisions + +- Updating a dependence requirement of Meilisearch is NOT considered as breaking by SemVer guide and will lead, in our case, to increasing the MINOR. Indeed, increasing the MINOR (instead of the PATCH) will prevent users from downloading the new patched version without noticing the changes. +See the [related rule](https://semver.org/#what-should-i-do-if-i-update-my-own-dependencies-without-changing-the-public-api). +- Fixing a CVE (Common Vulnerabilities and Exposures) will not increase the MAJOR; depending on the CVE, it will be a PATCH or a MINOR upgrade. From 4f8382b159ded27446334fb2cdca7b4d5ad2c1a9 Mon Sep 17 00:00:00 2001 From: curquiza Date: Sun, 20 Jul 2025 22:07:59 +0200 Subject: [PATCH 154/312] Remove useless automation --- .github/ISSUE_TEMPLATE/sprint_issue.md | 58 ----- .github/workflows/check-valid-milestone.yml | 100 --------- .github/workflows/milestone-workflow.yml | 224 -------------------- 3 files changed, 382 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/sprint_issue.md delete mode 100644 .github/workflows/check-valid-milestone.yml delete mode 100644 .github/workflows/milestone-workflow.yml diff --git a/.github/ISSUE_TEMPLATE/sprint_issue.md b/.github/ISSUE_TEMPLATE/sprint_issue.md deleted file mode 100644 index 30b5e16ff..000000000 --- a/.github/ISSUE_TEMPLATE/sprint_issue.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -name: New sprint issue -about: ⚠️ Should only be used by the engine team ⚠️ -title: '' -labels: 'missing usage in PRD, impacts docs' -assignees: '' - ---- - -Related product team resources: [PRD]() (_internal only_) -Related product discussion: - -## Motivation - - - -## Usage - - - -## TODO - - - -### Are you modifying a database? -- [ ] If not, add the `no db change` label to your PR, and you're good to merge. -- [ ] If yes, add the `db change` label to your PR. You'll receive a message explaining you what to do. - -### Reminders when modifying the API - -- [ ] Update the openAPI file with utoipa: - - [ ] If a new module has been introduced, create a new structure deriving [the OpenAPI proc-macro](https://docs.rs/utoipa/latest/utoipa/derive.OpenApi.html) and nest it in the main [openAPI structure](https://github.com/meilisearch/meilisearch/blob/f2185438eed60fa32d25b15480c5ee064f6fba4a/crates/meilisearch/src/routes/mod.rs#L64-L78). - - [ ] If a new route has been introduced, add the [path decorator](https://docs.rs/utoipa/latest/utoipa/attr.path.html) to it and add the route at the top of the file in its openAPI structure. - - [ ] If a structure which is deserialized or serialized in the API has been introduced or modified, it must derive the [`schema`](https://docs.rs/utoipa/latest/utoipa/macro.schema.html) or the [`IntoParams`](https://docs.rs/utoipa/latest/utoipa/derive.IntoParams.html) proc-macro. - If it's a **new** structure you must also add it to the big list of structures [in the main `OpenApi` structure](https://github.com/meilisearch/meilisearch/blob/f2185438eed60fa32d25b15480c5ee064f6fba4a/crates/meilisearch/src/routes/mod.rs#L88). - - [ ] Once everything is done, start Meilisearch with the swagger flag: `cargo run --features swagger`, open `http://localhost:7700/scalar` on your browser, and ensure everything works as expected. - - For more info, refer to [this presentation](https://pitch.com/v/generating-the-openapi-file-jrn3nh). - -### Reminders when modifying the Setting API - - - -- [ ] Ensure the new setting route is at least tested by the [`test_setting_routes` macro](https://github.com/meilisearch/meilisearch/blob/5204c0b60b384cbc79621b6b2176fca086069e8e/meilisearch/tests/settings/get_settings.rs#L276) -- [ ] Ensure Analytics are fully implemented - - [ ] `/settings/my-new-setting` configurated in the [`make_setting_routes` macro](https://github.com/meilisearch/meilisearch/blob/5204c0b60b384cbc79621b6b2176fca086069e8e/meilisearch/src/routes/indexes/settings.rs#L141-L165) - - [ ] global `/settings` route configurated in the [`update_all` function](https://github.com/meilisearch/meilisearch/blob/5204c0b60b384cbc79621b6b2176fca086069e8e/meilisearch/src/routes/indexes/settings.rs#L655-L751) -- [ ] Ensure the dump serializing is consistent with the `/settings` route serializing, e.g., enums case can be different (`camelCase` in route and `PascalCase` in the dump) - -#### Special cases when adding a setting for an experimental feature - -- [ ] ⚠️ API stability: The setting does not appear on the main settings route when the feature has never been enabled (e.g. mark it `Unset` when returned from the index in this situation. See [an example](https://github.com/meilisearch/meilisearch/blob/7a89abd2a025606a42f8b219e539117eb2eb029f/meilisearch-types/src/settings.rs#L608)) -- [ ] The setting cannot be set when the feature is disabled, either by the main settings route or the subroute (see [`validate_settings` function](https://github.com/meilisearch/meilisearch/blob/7a89abd2a025606a42f8b219e539117eb2eb029f/meilisearch/src/routes/indexes/settings.rs#L811)) -- [ ] If possible, the setting is reset when the feature is disabled (hard if it requires reindexing) - -## Impacted teams - - - diff --git a/.github/workflows/check-valid-milestone.yml b/.github/workflows/check-valid-milestone.yml deleted file mode 100644 index 91d2daa8e..000000000 --- a/.github/workflows/check-valid-milestone.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: PR Milestone Check - -on: - pull_request: - types: [opened, reopened, edited, synchronize, milestoned, demilestoned] - branches: - - "main" - - "release-v*.*.*" - -jobs: - check-milestone: - name: Check PR Milestone - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v3 - - - name: Validate PR milestone - uses: actions/github-script@v7 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - // Get PR number directly from the event payload - const prNumber = context.payload.pull_request.number; - - // Get PR details - const { data: prData } = await github.rest.pulls.get({ - owner: 'meilisearch', - repo: 'meilisearch', - pull_number: prNumber - }); - - // Get base branch name - const baseBranch = prData.base.ref; - console.log(`Base branch: ${baseBranch}`); - - // Get PR milestone - const prMilestone = prData.milestone; - if (!prMilestone) { - core.setFailed('PR must have a milestone assigned'); - return; - } - console.log(`PR milestone: ${prMilestone.title}`); - - // Validate milestone format: vx.y.z - const milestoneRegex = /^v\d+\.\d+\.\d+$/; - if (!milestoneRegex.test(prMilestone.title)) { - core.setFailed(`Milestone "${prMilestone.title}" does not follow the required format vx.y.z`); - return; - } - - // For main branch PRs, check if the milestone is the highest one - if (baseBranch === 'main') { - // Get all milestones - const { data: milestones } = await github.rest.issues.listMilestones({ - owner: 'meilisearch', - repo: 'meilisearch', - state: 'open', - sort: 'due_on', - direction: 'desc' - }); - - // Sort milestones by version number (vx.y.z) - const sortedMilestones = milestones - .filter(m => milestoneRegex.test(m.title)) - .sort((a, b) => { - const versionA = a.title.substring(1).split('.').map(Number); - const versionB = b.title.substring(1).split('.').map(Number); - - // Compare major version - if (versionA[0] !== versionB[0]) return versionB[0] - versionA[0]; - // Compare minor version - if (versionA[1] !== versionB[1]) return versionB[1] - versionA[1]; - // Compare patch version - return versionB[2] - versionA[2]; - }); - - if (sortedMilestones.length === 0) { - core.setFailed('No valid milestones found in the repository. Please create at least one milestone with the format vx.y.z'); - return; - } - - const highestMilestone = sortedMilestones[0]; - console.log(`Highest milestone: ${highestMilestone.title}`); - - if (prMilestone.title !== highestMilestone.title) { - core.setFailed(`PRs targeting the main branch must use the highest milestone (${highestMilestone.title}), but this PR uses ${prMilestone.title}`); - return; - } - } else { - // For release branches, the milestone should match the branch version - const branchVersion = baseBranch.substring(8); // remove 'release-' - if (prMilestone.title !== branchVersion) { - core.setFailed(`PRs targeting release branch "${baseBranch}" must use the matching milestone "${branchVersion}", but this PR uses "${prMilestone.title}"`); - return; - } - } - - console.log('PR milestone validation passed!'); diff --git a/.github/workflows/milestone-workflow.yml b/.github/workflows/milestone-workflow.yml deleted file mode 100644 index f2841c97e..000000000 --- a/.github/workflows/milestone-workflow.yml +++ /dev/null @@ -1,224 +0,0 @@ -name: Milestone's workflow - -# /!\ No git flow are handled here - -# For each Milestone created (not opened!), and if the release is NOT a patch release (only the patch changed) -# - the roadmap issue is created, see https://github.com/meilisearch/engine-team/blob/main/issue-templates/roadmap-issue.md -# - the changelog issue is created, see https://github.com/meilisearch/engine-team/blob/main/issue-templates/changelog-issue.md -# - update the ruleset to add the current release version to the list of allowed versions and be able to use the merge queue. - -# For each Milestone closed -# - the `release_version` label is created -# - this label is applied to all issues/PRs in the Milestone - -on: - milestone: - types: [created, closed] - -env: - MILESTONE_VERSION: ${{ github.event.milestone.title }} - MILESTONE_URL: ${{ github.event.milestone.html_url }} - MILESTONE_DUE_ON: ${{ github.event.milestone.due_on }} - GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }} - -jobs: - # ----------------- - # MILESTONE CREATED - # ----------------- - - get-release-version: - if: github.event.action == 'created' - runs-on: ubuntu-latest - outputs: - is-patch: ${{ steps.check-patch.outputs.is-patch }} - steps: - - uses: actions/checkout@v3 - - name: Check if this release is a patch release only - id: check-patch - run: | - echo version: $MILESTONE_VERSION - if [[ $MILESTONE_VERSION =~ ^v[0-9]+\.[0-9]+\.0$ ]]; then - echo 'This is NOT a patch release' - echo "is-patch=false" >> $GITHUB_OUTPUT - elif [[ $MILESTONE_VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - echo 'This is a patch release' - echo "is-patch=true" >> $GITHUB_OUTPUT - else - echo "Not a valid format of release, check the Milestone's title." - echo 'Should be vX.Y.Z' - exit 1 - fi - - create-roadmap-issue: - needs: get-release-version - # Create the roadmap issue if the release is not only a patch release - if: github.event.action == 'created' && needs.get-release-version.outputs.is-patch == 'false' - runs-on: ubuntu-latest - env: - ISSUE_TEMPLATE: issue-template.md - steps: - - uses: actions/checkout@v3 - - name: Download the issue template - run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/roadmap-issue.md > $ISSUE_TEMPLATE - - name: Replace all empty occurrences in the templates - run: | - # Replace all <> occurrences - sed -i "s/<>/$MILESTONE_VERSION/g" $ISSUE_TEMPLATE - - # Replace all <> occurrences - milestone_id=$(echo $MILESTONE_URL | cut -d '/' -f 7) - sed -i "s/<>/$milestone_id/g" $ISSUE_TEMPLATE - - # Replace release date if exists - if [[ ! -z $MILESTONE_DUE_ON ]]; then - date=$(echo $MILESTONE_DUE_ON | cut -d 'T' -f 1) - sed -i "s/Release date\: 20XX-XX-XX/Release date\: $date/g" $ISSUE_TEMPLATE - fi - - name: Create the issue - run: | - gh issue create \ - --title "$MILESTONE_VERSION ROADMAP" \ - --label 'epic,impacts docs,impacts integrations,impacts cloud' \ - --body-file $ISSUE_TEMPLATE \ - --milestone $MILESTONE_VERSION - - create-changelog-issue: - needs: get-release-version - # Create the changelog issue if the release is not only a patch release - if: github.event.action == 'created' && needs.get-release-version.outputs.is-patch == 'false' - runs-on: ubuntu-latest - env: - ISSUE_TEMPLATE: issue-template.md - steps: - - uses: actions/checkout@v3 - - name: Download the issue template - run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/changelog-issue.md > $ISSUE_TEMPLATE - - name: Replace all empty occurrences in the templates - run: | - # Replace all <> occurrences - sed -i "s/<>/$MILESTONE_VERSION/g" $ISSUE_TEMPLATE - - # Replace all <> occurrences - milestone_id=$(echo $MILESTONE_URL | cut -d '/' -f 7) - sed -i "s/<>/$milestone_id/g" $ISSUE_TEMPLATE - - name: Create the issue - run: | - gh issue create \ - --title "Create release changelogs for $MILESTONE_VERSION" \ - --label 'impacts docs,documentation' \ - --body-file $ISSUE_TEMPLATE \ - --milestone $MILESTONE_VERSION \ - --assignee curquiza - - create-update-version-issue: - needs: get-release-version - # Create the update-version issue even if the release is a patch release - if: github.event.action == 'created' - runs-on: ubuntu-latest - env: - ISSUE_TEMPLATE: issue-template.md - steps: - - uses: actions/checkout@v3 - - name: Download the issue template - run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/update-version-issue.md > $ISSUE_TEMPLATE - - name: Create the issue - run: | - gh issue create \ - --title "Update version in Cargo.toml for $MILESTONE_VERSION" \ - --label 'maintenance' \ - --body-file $ISSUE_TEMPLATE \ - --milestone $MILESTONE_VERSION - - create-update-openapi-issue: - needs: get-release-version - # Create the openAPI issue if the release is not only a patch release - if: github.event.action == 'created' && needs.get-release-version.outputs.is-patch == 'false' - runs-on: ubuntu-latest - env: - ISSUE_TEMPLATE: issue-template.md - steps: - - uses: actions/checkout@v3 - - name: Download the issue template - run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/update-openapi-issue.md > $ISSUE_TEMPLATE - - name: Create the issue - run: | - gh issue create \ - --title "Update Open API file for $MILESTONE_VERSION" \ - --label 'maintenance' \ - --body-file $ISSUE_TEMPLATE \ - --milestone $MILESTONE_VERSION - - update-ruleset: - runs-on: ubuntu-latest - if: github.event.action == 'created' - steps: - - uses: actions/checkout@v3 - - name: Install jq - run: | - sudo apt-get update - sudo apt-get install -y jq - - name: Update ruleset - env: - # gh api repos/meilisearch/meilisearch/rulesets --jq '.[] | {name: .name, id: .id}' - RULESET_ID: 4253297 - BRANCH_NAME: ${{ github.event.inputs.branch_name }} - run: | - echo "RULESET_ID: ${{ env.RULESET_ID }}" - echo "BRANCH_NAME: ${{ env.BRANCH_NAME }}" - - # Get current ruleset conditions - CONDITIONS=$(gh api repos/meilisearch/meilisearch/rulesets/${{ env.RULESET_ID }} --jq '{ conditions: .conditions }') - - # Update the conditions by appending the milestone version - UPDATED_CONDITIONS=$(echo $CONDITIONS | jq '.conditions.ref_name.include += ["refs/heads/release-'${{ env.MILESTONE_VERSION }}'"]') - - # Update the ruleset from stdin (-) - echo $UPDATED_CONDITIONS | - gh api repos/meilisearch/meilisearch/rulesets/${{ env.RULESET_ID }} \ - --method PUT \ - -H "Accept: application/vnd.github+json" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - --input - - - # ---------------- - # MILESTONE CLOSED - # ---------------- - - create-release-label: - if: github.event.action == 'closed' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Create the ${{ env.MILESTONE_VERSION }} label - run: | - label_description="PRs/issues solved in $MILESTONE_VERSION" - if [[ ! -z $MILESTONE_DUE_ON ]]; then - date=$(echo $MILESTONE_DUE_ON | cut -d 'T' -f 1) - label_description="$label_description released on $date" - fi - - gh api repos/meilisearch/meilisearch/labels \ - --method POST \ - -H "Accept: application/vnd.github+json" \ - -f name="$MILESTONE_VERSION" \ - -f description="$label_description" \ - -f color='ff5ba3' - - labelize-all-milestone-content: - if: github.event.action == 'closed' - needs: create-release-label - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Add label ${{ env.MILESTONE_VERSION }} to all PRs in the Milestone - run: | - prs=$(gh pr list --search milestone:"$MILESTONE_VERSION" --limit 1000 --state all --json number --template '{{range .}}{{tablerow (printf "%v" .number)}}{{end}}') - for pr in $prs; do - gh pr edit $pr --add-label $MILESTONE_VERSION - done - - name: Add label ${{ env.MILESTONE_VERSION }} to all issues in the Milestone - run: | - issues=$(gh issue list --search milestone:"$MILESTONE_VERSION" --limit 1000 --state all --json number --template '{{range .}}{{tablerow (printf "%v" .number)}}{{end}}') - for issue in $issues; do - gh issue edit $issue --add-label $MILESTONE_VERSION - done From 52d8007b127192c7b04b5e20d847ce87e4655096 Mon Sep 17 00:00:00 2001 From: curquiza Date: Sun, 20 Jul 2025 22:10:17 +0200 Subject: [PATCH 155/312] Add pull request template --- .github/pull_request_template.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..2e6ee0fff --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,13 @@ +## Related issue + +Fixes #... + +## Requirements + +⚠️ Ensure the following requirements before merging ⚠️ +- [] Automated tests have been added. +- [] If some tests cannot be automated, manual rigorous tests should be applied. +- [] ⚠️ If there is an change in the DB: it's mandatory to manually test the `--experimental-dumpless-upgrade` on a DB of the previous Meilisearch minor version (e.g. v1.13 for the v1.14 release). +- [] If necessary, the feature have been tested in the Cloud production environment (with [prototypes](./documentation/prototypes.md)) and the Cloud UI is ready. +- [] If necessary, the [documentation](https://github.com/meilisearch/documentation) related to the implemented feature in the PR is ready. +- [] If necessary, the [integrations](https://github.com/meilisearch/integration-guides) related to the implemented feature in the PR are ready. From dc0bd9f25ded68425e62b90ed47a6c722019e8e7 Mon Sep 17 00:00:00 2001 From: curquiza Date: Sun, 20 Jul 2025 22:10:35 +0200 Subject: [PATCH 156/312] Add release drafter --- .github/release-draft-template.yml | 23 +++++++++++++++++++++++ .github/workflows/release-drafter.yml | 16 ++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 .github/release-draft-template.yml create mode 100644 .github/workflows/release-drafter.yml diff --git a/.github/release-draft-template.yml b/.github/release-draft-template.yml new file mode 100644 index 000000000..1088be33b --- /dev/null +++ b/.github/release-draft-template.yml @@ -0,0 +1,23 @@ +name-template: 'v$RESOLVED_VERSION' +tag-template: 'v$RESOLVED_VERSION' +exclude-labels: + - 'skip changelog' +version-resolver: + major: + labels: + - 'breaking-change' + minor: + labels: + - 'enhancement' + default: patch +template: | + $CHANGES + + Thanks again to $CONTRIBUTORS! 🎉 +no-changes-template: 'Changes are coming soon 😎' +sort-direction: 'ascending' +replacers: + - search: '/(?:and )?@dependabot-preview(?:\[bot\])?,?/g' + replace: '' + - search: '/(?:and )?@dependabot(?:\[bot\])?,?/g' + replace: '' diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml new file mode 100644 index 000000000..20f2d83f4 --- /dev/null +++ b/.github/workflows/release-drafter.yml @@ -0,0 +1,16 @@ +name: Release Drafter + +on: + push: + branches: + - main + +jobs: + update_release_draft: + runs-on: ubuntu-latest + steps: + - uses: release-drafter/release-drafter@v6 + with: + config-name: release-draft-template.yml + env: + GITHUB_TOKEN: ${{ secrets.RELEASE_DRAFTER_TOKEN }} From dc1656da8eec06a6ebfef52a3e1a08be06015cc7 Mon Sep 17 00:00:00 2001 From: curquiza Date: Sun, 20 Jul 2025 22:11:14 +0200 Subject: [PATCH 157/312] Adapt automation --- .github/workflows/flaky-tests.yml | 2 +- .github/workflows/sdks-tests.yml | 6 +++++- .github/workflows/test-suite.yml | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/flaky-tests.yml b/.github/workflows/flaky-tests.yml index 66be5b823..8f803f0ee 100644 --- a/.github/workflows/flaky-tests.yml +++ b/.github/workflows/flaky-tests.yml @@ -3,7 +3,7 @@ name: Look for flaky tests on: workflow_dispatch: schedule: - - cron: "0 12 * * FRI" # Every Friday at 12:00PM + - cron: '0 4 * * *' # Every day at 4:00AM jobs: flaky: diff --git a/.github/workflows/sdks-tests.yml b/.github/workflows/sdks-tests.yml index dc4d51068..e9ceeeb95 100644 --- a/.github/workflows/sdks-tests.yml +++ b/.github/workflows/sdks-tests.yml @@ -9,7 +9,11 @@ on: required: false default: nightly schedule: - - cron: "0 6 * * MON" # Every Monday at 6:00AM + - cron: '0 6 * * *' # Every day at 6:00am + pull_request: + push: + branches: + - main env: MEILI_MASTER_KEY: 'masterKey' diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 2924a07bc..75914aea1 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -3,7 +3,7 @@ name: Test suite on: workflow_dispatch: schedule: - # Everyday at 5:00am + # Every day at 5:00am - cron: "0 5 * * *" pull_request: merge_group: From 7f318ee964729541b262484c68cc38762b42cc5e Mon Sep 17 00:00:00 2001 From: curquiza Date: Sun, 20 Jul 2025 22:11:30 +0200 Subject: [PATCH 158/312] Adapt issue template --- .github/ISSUE_TEMPLATE/new_feature_issue.md | 56 +++++++++++++++++++++ .github/templates/dependency-issue.md | 22 ++++++++ .github/workflows/dependency-issue.yml | 2 +- 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 .github/ISSUE_TEMPLATE/new_feature_issue.md create mode 100644 .github/templates/dependency-issue.md diff --git a/.github/ISSUE_TEMPLATE/new_feature_issue.md b/.github/ISSUE_TEMPLATE/new_feature_issue.md new file mode 100644 index 000000000..cf55fa43f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/new_feature_issue.md @@ -0,0 +1,56 @@ +--- +name: New feature issue +about: ⚠️ Should only be used by the internal Meili team ⚠️ +title: '' +labels: 'impacts docs, impacts integrations' +assignees: '' + +--- + +Related product team resources: [PRD]() (_internal only_) + +## Usage + + + +TBD + +## TODO + + + +### Are you modifying a database? + +- [ ] If not, add the `no db change` label to your PR, and you're good to merge. +- [ ] If yes, add the `db change` label to your PR. You'll receive a message explaining you what to do. + +### Reminders when modifying the API + +- [ ] Update the openAPI file with utoipa: + - [ ] If a new module has been introduced, create a new structure deriving [the OpenAPI proc-macro](https://docs.rs/utoipa/latest/utoipa/derive.OpenApi.html) and nest it in the main [openAPI structure](https://github.com/meilisearch/meilisearch/blob/f2185438eed60fa32d25b15480c5ee064f6fba4a/crates/meilisearch/src/routes/mod.rs#L64-L78). + - [ ] If a new route has been introduced, add the [path decorator](https://docs.rs/utoipa/latest/utoipa/attr.path.html) to it and add the route at the top of the file in its openAPI structure. + - [ ] If a structure which is deserialized or serialized in the API has been introduced or modified, it must derive the [`schema`](https://docs.rs/utoipa/latest/utoipa/macro.schema.html) or the [`IntoParams`](https://docs.rs/utoipa/latest/utoipa/derive.IntoParams.html) proc-macro. + If it's a **new** structure you must also add it to the big list of structures [in the main `OpenApi` structure](https://github.com/meilisearch/meilisearch/blob/f2185438eed60fa32d25b15480c5ee064f6fba4a/crates/meilisearch/src/routes/mod.rs#L88). + - [ ] Once everything is done, start Meilisearch with the swagger flag: `cargo run --features swagger`, open `http://localhost:7700/scalar` on your browser, and ensure everything works as expected. + - For more info, refer to [this presentation](https://pitch.com/v/generating-the-openapi-file-jrn3nh). + +### Reminders when modifying the Setting API + + + +- [ ] Ensure the new setting route is at least tested by the [`test_setting_routes` macro](https://github.com/meilisearch/meilisearch/blob/5204c0b60b384cbc79621b6b2176fca086069e8e/meilisearch/tests/settings/get_settings.rs#L276) +- [ ] Ensure Analytics are fully implemented + - [ ] `/settings/my-new-setting` configurated in the [`make_setting_routes` macro](https://github.com/meilisearch/meilisearch/blob/5204c0b60b384cbc79621b6b2176fca086069e8e/meilisearch/src/routes/indexes/settings.rs#L141-L165) + - [ ] global `/settings` route configurated in the [`update_all` function](https://github.com/meilisearch/meilisearch/blob/5204c0b60b384cbc79621b6b2176fca086069e8e/meilisearch/src/routes/indexes/settings.rs#L655-L751) +- [ ] Ensure the dump serializing is consistent with the `/settings` route serializing, e.g., enums case can be different (`camelCase` in route and `PascalCase` in the dump) + +#### Special cases when adding a setting for an experimental feature + +- [ ] ⚠️ API stability: The setting does not appear on the main settings route when the feature has never been enabled (e.g. mark it `Unset` when returned from the index in this situation. See [an example](https://github.com/meilisearch/meilisearch/blob/7a89abd2a025606a42f8b219e539117eb2eb029f/meilisearch-types/src/settings.rs#L608)) +- [ ] The setting cannot be set when the feature is disabled, either by the main settings route or the subroute (see [`validate_settings` function](https://github.com/meilisearch/meilisearch/blob/7a89abd2a025606a42f8b219e539117eb2eb029f/meilisearch/src/routes/indexes/settings.rs#L811)) +- [ ] If possible, the setting is reset when the feature is disabled (hard if it requires reindexing) + +## Impacted teams + + + diff --git a/.github/templates/dependency-issue.md b/.github/templates/dependency-issue.md new file mode 100644 index 000000000..72835c5f6 --- /dev/null +++ b/.github/templates/dependency-issue.md @@ -0,0 +1,22 @@ +This issue is about updating Meilisearch dependencies: + - [ ] Update Meilisearch dependencies with the help of `cargo +nightly udeps --all-targets` (remove unused dependencies) and `cargo upgrade` (upgrade dependencies versions) - ⚠️ Some repositories may contain subdirectories (like heed, charabia, or deserr). Take care of updating these in the main crate as well. This won't be done automatically by `cargo upgrade`. + - [ ] [deserr](https://github.com/meilisearch/deserr) + - [ ] [charabia](https://github.com/meilisearch/charabia/) + - [ ] [heed](https://github.com/meilisearch/heed/) + - [ ] [roaring-rs](https://github.com/RoaringBitmap/roaring-rs/) + - [ ] [obkv](https://github.com/meilisearch/obkv) + - [ ] [grenad](https://github.com/meilisearch/grenad/) + - [ ] [arroy](https://github.com/meilisearch/arroy/) + - [ ] [segment](https://github.com/meilisearch/segment) + - [ ] [bumparaw-collections](https://github.com/meilisearch/bumparaw-collections) + - [ ] [bbqueue](https://github.com/meilisearch/bbqueue) + - [ ] Finally, [Meilisearch](https://github.com/meilisearch/MeiliSearch) + - [ ] If new Rust versions have been released, update the minimal Rust version in use at Meilisearch: + - [ ] in this [GitHub Action file](https://github.com/meilisearch/meilisearch/blob/main/.github/workflows/test-suite.yml), by changing the `toolchain` field of the `rustfmt` job to the latest available nightly (of the day before or the current day). + - [ ] in every [GitHub Action files](https://github.com/meilisearch/meilisearch/blob/main/.github/workflows), by changing all the `dtolnay/rust-toolchain@` references to use the latest stable version. + - [ ] in this [`rust-toolchain.toml`](https://github.com/meilisearch/meilisearch/blob/main/rust-toolchain.toml), by changing the `channel` field to the latest stable version. + - [ ] in the [Dockerfile](https://github.com/meilisearch/meilisearch/blob/main/Dockerfile), by changing the base image to `rust:-alpine`. Check that the image exists on [Dockerhub](https://hub.docker.com/_/rust/tags?page=1&name=alpine). Also, build and run the image to check everything still works! + +⚠️ This issue should be prioritized to avoid any deprecation and vulnerability issues. + +The GitHub action dependencies are managed by [Dependabot](https://github.com/meilisearch/meilisearch/blob/main/.github/dependabot.yml), so no need to update them when solving this issue. diff --git a/.github/workflows/dependency-issue.yml b/.github/workflows/dependency-issue.yml index 99bd8330a..5de490d76 100644 --- a/.github/workflows/dependency-issue.yml +++ b/.github/workflows/dependency-issue.yml @@ -15,7 +15,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Download the issue template - run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/dependency-issue.md > $ISSUE_TEMPLATE + run: curl -s https://raw.githubusercontent.com/meilisearch/meilisearch/main/.github/templates/dependency-issue.md > $ISSUE_TEMPLATE - name: Create issue run: | gh issue create \ From cd0523c3f19629b4a1d41cbc2a1048af90bfea8d Mon Sep 17 00:00:00 2001 From: curquiza Date: Sun, 20 Jul 2025 22:13:07 +0200 Subject: [PATCH 159/312] Remove run of SDK test on PR because cannot work --- .github/workflows/sdks-tests.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/sdks-tests.yml b/.github/workflows/sdks-tests.yml index e9ceeeb95..7fd4999b3 100644 --- a/.github/workflows/sdks-tests.yml +++ b/.github/workflows/sdks-tests.yml @@ -10,10 +10,6 @@ on: default: nightly schedule: - cron: '0 6 * * *' # Every day at 6:00am - pull_request: - push: - branches: - - main env: MEILI_MASTER_KEY: 'masterKey' From f3b60a1dabead08d31cf19cd1a52a4d2e1364324 Mon Sep 17 00:00:00 2001 From: curquiza Date: Sun, 20 Jul 2025 22:20:08 +0200 Subject: [PATCH 160/312] Minor update on doc --- documentation/release.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/documentation/release.md b/documentation/release.md index 4d8bb0016..f308bab98 100644 --- a/documentation/release.md +++ b/documentation/release.md @@ -4,9 +4,9 @@ This guide is to describe how to make releases for the current repository. ## 📅 Weekly Meilisearch release -1. A weekly meeting is done every Monday to define the release and check the following information +1. A weekly meeting is done every Monday to define the release and to ensure minimal checks before the release.
-👇👇👇 +Check out the TODO 👇👇👇 - [ ] Define the version of the release (`vX.Y.Z`) - [ ] Manually test `--experimental-dumpless-upgrade` on a DB of the previous Meilisearch minor version
- [ ] Check recent automated tests on `main`
@@ -19,9 +19,9 @@ This guide is to describe how to make releases for the current repository. - [ ] Create the PR updating the versionand merge it.
-1. Go to the GitHub interface, in the [`Release` section](https://github.com/meilisearch/meilisearch/releases). +2. Go to the GitHub interface, in the [`Release` section](https://github.com/meilisearch/meilisearch/releases). -2. Select the already drafted release or click on the `Draft a new release` button if you want to start a blank one, and fill the form with the appropriate information. +3. Select the already drafted release or click on the `Draft a new release` button if you want to start a blank one, and fill the form with the appropriate information. ⚠️ Publish on `main` ⚙️ The CIs will be triggered to: From bdc2d1e64dbb04fd2bce2b099600e605c16b51f9 Mon Sep 17 00:00:00 2001 From: Kerollmops Date: Mon, 21 Jul 2025 14:37:22 +0200 Subject: [PATCH 161/312] Move the edition 2024 dump parameter to the right place --- crates/index-scheduler/src/lib.rs | 6 ++-- crates/index-scheduler/src/test_utils.rs | 1 - .../src/analytics/segment_analytics.rs | 2 +- crates/meilisearch/src/lib.rs | 1 - crates/meilisearch/src/option.rs | 31 ++++++++++--------- crates/meilisearch/tests/common/server.rs | 1 + crates/milli/src/update/indexer_config.rs | 2 ++ 7 files changed, 24 insertions(+), 20 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 8715bc100..46566b9ba 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -139,8 +139,6 @@ pub struct IndexSchedulerOptions { pub embedding_cache_cap: usize, /// Snapshot compaction status. pub experimental_no_snapshot_compaction: bool, - /// Whether dump import uses the old document indexer or the new one. - pub experimental_no_edition_2024_for_dumps: bool, } /// Structure which holds meilisearch's indexes and schedules the tasks @@ -302,7 +300,9 @@ impl IndexScheduler { index_mapper, env, cleanup_enabled: options.cleanup_enabled, - experimental_no_edition_2024_for_dumps: options.experimental_no_edition_2024_for_dumps, + experimental_no_edition_2024_for_dumps: options + .indexer_config + .experimental_no_edition_2024_for_dumps, webhook_url: options.webhook_url, webhook_authorization_header: options.webhook_authorization_header, embedders: Default::default(), diff --git a/crates/index-scheduler/src/test_utils.rs b/crates/index-scheduler/src/test_utils.rs index 0a705b6c7..bfed7f53a 100644 --- a/crates/index-scheduler/src/test_utils.rs +++ b/crates/index-scheduler/src/test_utils.rs @@ -115,7 +115,6 @@ impl IndexScheduler { auto_upgrade: true, // Don't cost much and will ensure the happy path works embedding_cache_cap: 10, experimental_no_snapshot_compaction: false, - experimental_no_edition_2024_for_dumps: false, }; let version = configuration(&mut options).unwrap_or({ (versioning::VERSION_MAJOR, versioning::VERSION_MINOR, versioning::VERSION_PATCH) diff --git a/crates/meilisearch/src/analytics/segment_analytics.rs b/crates/meilisearch/src/analytics/segment_analytics.rs index a96ddf068..a2a0f0c05 100644 --- a/crates/meilisearch/src/analytics/segment_analytics.rs +++ b/crates/meilisearch/src/analytics/segment_analytics.rs @@ -254,7 +254,6 @@ impl Infos { experimental_limit_batched_tasks_total_size, experimental_embedding_cache_entries, experimental_no_snapshot_compaction, - experimental_no_edition_2024_for_dumps, http_addr, master_key: _, env, @@ -295,6 +294,7 @@ impl Infos { max_indexing_threads, skip_index_budget: _, experimental_no_edition_2024_for_settings, + experimental_no_edition_2024_for_dumps, } = indexer_options; let RuntimeTogglableFeatures { diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 13d2eb789..0fb93b65a 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -240,7 +240,6 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< auto_upgrade: opt.experimental_dumpless_upgrade, embedding_cache_cap: opt.experimental_embedding_cache_entries, experimental_no_snapshot_compaction: opt.experimental_no_snapshot_compaction, - experimental_no_edition_2024_for_dumps: opt.experimental_no_edition_2024_for_dumps, }; let binary_version = (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH); diff --git a/crates/meilisearch/src/option.rs b/crates/meilisearch/src/option.rs index 77106d362..dd77a1222 100644 --- a/crates/meilisearch/src/option.rs +++ b/crates/meilisearch/src/option.rs @@ -469,15 +469,6 @@ pub struct Opt { #[serde(default)] pub experimental_no_snapshot_compaction: bool, - /// Experimental make dump imports use the old document indexer. - /// - /// When enabled, Meilisearch will use the old document indexer when importing dumps. - /// - /// For more information, see . - #[clap(long, env = MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_DUMPS)] - #[serde(default)] - pub experimental_no_edition_2024_for_dumps: bool, - #[serde(flatten)] #[clap(flatten)] pub indexer_options: IndexerOpts, @@ -583,7 +574,6 @@ impl Opt { experimental_limit_batched_tasks_total_size, experimental_embedding_cache_entries, experimental_no_snapshot_compaction, - experimental_no_edition_2024_for_dumps, } = self; export_to_env_if_not_present(MEILI_DB_PATH, db_path); export_to_env_if_not_present(MEILI_HTTP_ADDR, http_addr); @@ -684,10 +674,6 @@ impl Opt { MEILI_EXPERIMENTAL_NO_SNAPSHOT_COMPACTION, experimental_no_snapshot_compaction.to_string(), ); - export_to_env_if_not_present( - MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_DUMPS, - experimental_no_edition_2024_for_dumps.to_string(), - ); indexer_options.export_to_env(); } @@ -775,6 +761,15 @@ pub struct IndexerOpts { #[clap(long, env = MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_SETTINGS)] #[serde(default)] pub experimental_no_edition_2024_for_settings: bool, + + /// Experimental make dump imports use the old document indexer. + /// + /// When enabled, Meilisearch will use the old document indexer when importing dumps. + /// + /// For more information, see . + #[clap(long, env = MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_DUMPS)] + #[serde(default)] + pub experimental_no_edition_2024_for_dumps: bool, } impl IndexerOpts { @@ -785,6 +780,7 @@ impl IndexerOpts { max_indexing_threads, skip_index_budget: _, experimental_no_edition_2024_for_settings, + experimental_no_edition_2024_for_dumps, } = self; if let Some(max_indexing_memory) = max_indexing_memory.0 { export_to_env_if_not_present( @@ -804,6 +800,12 @@ impl IndexerOpts { experimental_no_edition_2024_for_settings.to_string(), ); } + if experimental_no_edition_2024_for_dumps { + export_to_env_if_not_present( + MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_DUMPS, + experimental_no_edition_2024_for_dumps.to_string(), + ); + } } } @@ -824,6 +826,7 @@ impl TryFrom<&IndexerOpts> for IndexerConfig { skip_index_budget: other.skip_index_budget, experimental_no_edition_2024_for_settings: other .experimental_no_edition_2024_for_settings, + experimental_no_edition_2024_for_dumps: other.experimental_no_edition_2024_for_dumps, chunk_compression_type: Default::default(), chunk_compression_level: Default::default(), documents_chunk_size: Default::default(), diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 5f82bb380..ad0678122 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -466,6 +466,7 @@ pub fn default_settings(dir: impl AsRef) -> Opt { // Having 2 threads makes the tests way faster max_indexing_threads: MaxThreads::from_str("2").unwrap(), experimental_no_edition_2024_for_settings: false, + experimental_no_edition_2024_for_dumps: false, }, experimental_enable_metrics: false, ..Parser::parse_from(None as Option<&str>) diff --git a/crates/milli/src/update/indexer_config.rs b/crates/milli/src/update/indexer_config.rs index a0f901818..845da5a51 100644 --- a/crates/milli/src/update/indexer_config.rs +++ b/crates/milli/src/update/indexer_config.rs @@ -16,6 +16,7 @@ pub struct IndexerConfig { pub max_positions_per_attributes: Option, pub skip_index_budget: bool, pub experimental_no_edition_2024_for_settings: bool, + pub experimental_no_edition_2024_for_dumps: bool, } impl IndexerConfig { @@ -65,6 +66,7 @@ impl Default for IndexerConfig { max_positions_per_attributes: None, skip_index_budget: false, experimental_no_edition_2024_for_settings: false, + experimental_no_edition_2024_for_dumps: false, } } } From afc164a2715224931e1a4fbc2bb90a3fe1930836 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 17 Jul 2025 16:09:11 +0200 Subject: [PATCH 162/312] Fix in old indexer --- .../extract/extract_vector_points.rs | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/crates/milli/src/update/index_documents/extract/extract_vector_points.rs b/crates/milli/src/update/index_documents/extract/extract_vector_points.rs index 064cfd154..a1dfa1aad 100644 --- a/crates/milli/src/update/index_documents/extract/extract_vector_points.rs +++ b/crates/milli/src/update/index_documents/extract/extract_vector_points.rs @@ -23,7 +23,7 @@ use crate::progress::EmbedderStats; use crate::prompt::Prompt; use crate::update::del_add::{DelAdd, KvReaderDelAdd, KvWriterDelAdd}; use crate::update::settings::InnerIndexSettingsDiff; -use crate::vector::db::{EmbedderInfo, EmbeddingStatus, EmbeddingStatusDelta}; +use crate::vector::db::{EmbedderInfo, EmbeddingStatusDelta}; use crate::vector::error::{EmbedErrorKind, PossibleEmbeddingMistakes, UnusedVectorsDistribution}; use crate::vector::extractor::{Extractor, ExtractorDiff, RequestFragmentExtractor}; use crate::vector::parsed_vectors::{ParsedVectorsDiff, VectorState}; @@ -441,6 +441,8 @@ pub fn extract_vector_points( { let embedder_is_manual = matches!(*runtime.embedder, Embedder::UserProvided(_)); + let (old_is_user_provided, old_must_regenerate) = + embedder_info.embedding_status.is_user_provided_must_regenerate(docid); let (old, new) = parsed_vectors.remove(embedder_name); let new_must_regenerate = new.must_regenerate(); let delta = match action { @@ -499,16 +501,19 @@ pub fn extract_vector_points( let is_adding_fragments = has_fragments && !old_has_fragments; - if is_adding_fragments { + if !has_fragments { + // removing fragments + regenerate_prompt(obkv, &runtime.document_template, new_fields_ids_map)? + } else if is_adding_fragments || + // regenerate all fragments when going from user provided to ! user provided + old_is_user_provided + { regenerate_all_fragments( runtime.fragments(), &doc_alloc, new_fields_ids_map, obkv, ) - } else if !has_fragments { - // removing fragments - regenerate_prompt(obkv, &runtime.document_template, new_fields_ids_map)? } else { let mut fragment_diff = Vec::new(); let new_fields_ids_map = new_fields_ids_map.as_fields_ids_map(); @@ -600,7 +605,8 @@ pub fn extract_vector_points( docid, &delta, new_must_regenerate, - &embedder_info.embedding_status, + old_is_user_provided, + old_must_regenerate, ); // and we finally push the unique vectors into the writer @@ -657,10 +663,9 @@ fn push_embedding_status_delta( docid: DocumentId, delta: &VectorStateDelta, new_must_regenerate: bool, - embedding_status: &EmbeddingStatus, + old_is_user_provided: bool, + old_must_regenerate: bool, ) { - let (old_is_user_provided, old_must_regenerate) = - embedding_status.is_user_provided_must_regenerate(docid); let new_is_user_provided = match delta { VectorStateDelta::NoChange => old_is_user_provided, VectorStateDelta::NowRemoved => { From 366c37a68616a0dc0216cc10053b3103e4769413 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 17 Jul 2025 17:13:40 +0200 Subject: [PATCH 163/312] Fix new indexer --- .../src/update/new/extract/vectors/mod.rs | 88 ++++++++++++++----- 1 file changed, 68 insertions(+), 20 deletions(-) diff --git a/crates/milli/src/update/new/extract/vectors/mod.rs b/crates/milli/src/update/new/extract/vectors/mod.rs index 4ca68027c..71fa9bf09 100644 --- a/crates/milli/src/update/new/extract/vectors/mod.rs +++ b/crates/milli/src/update/new/extract/vectors/mod.rs @@ -620,12 +620,35 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { where 'a: 'doc, { - match &mut self.kind { - ChunkType::Fragments { fragments: _, session } => { - let doc_alloc = session.doc_alloc(); + self.set_status(docid, old_is_user_provided, true, false, true); - if old_is_user_provided | full_reindex { + match &mut self.kind { + ChunkType::Fragments { fragments, session } => { + let doc_alloc = session.doc_alloc(); + let reindex_all_fragments = + // when the vectors were user-provided, Meilisearch cannot know if they come from a particular fragment, + // and so Meilisearch needs to clear all embeddings in that case. + // Fortunately, as dump export fragment vector with `regenerate` set to `false`, + // this case should be rare and opt-in. + old_is_user_provided || + // full-reindex case + full_reindex; + + if reindex_all_fragments { session.on_embed_mut().clear_vectors(docid); + let extractors = fragments.iter().map(|fragment| { + RequestFragmentExtractor::new(fragment, doc_alloc).ignore_errors() + }); + insert_autogenerated( + docid, + external_docid, + extractors, + document, + &(), + session, + unused_vectors_distribution, + )?; + return Ok(()); } settings_delta.try_for_each_fragment_diff( @@ -669,7 +692,6 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { Result::Ok(()) }, )?; - self.set_status(docid, old_is_user_provided, true, false, true); } ChunkType::DocumentTemplate { document_template, session } => { let doc_alloc = session.doc_alloc(); @@ -690,12 +712,18 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { match extractor.diff_settings(document, &external_docid, old_extractor.as_ref())? { ExtractorDiff::Removed => { + if old_is_user_provided || full_reindex { + session.on_embed_mut().clear_vectors(docid); + } OnEmbed::process_embedding_response( session.on_embed_mut(), crate::vector::session::EmbeddingResponse { metadata, embedding: None }, ); } ExtractorDiff::Added(input) | ExtractorDiff::Updated(input) => { + if old_is_user_provided || full_reindex { + session.on_embed_mut().clear_vectors(docid); + } session.request_embedding(metadata, input, unused_vectors_distribution)?; } ExtractorDiff::Unchanged => { /* do nothing */ } @@ -722,6 +750,13 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { where 'a: 'doc, { + self.set_status( + docid, + old_is_user_provided, + old_must_regenerate, + false, + new_must_regenerate, + ); match &mut self.kind { ChunkType::DocumentTemplate { document_template, session } => { let doc_alloc = session.doc_alloc(); @@ -731,10 +766,6 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { new_fields_ids_map, ); - if old_is_user_provided { - session.on_embed_mut().clear_vectors(docid); - } - update_autogenerated( docid, external_docid, @@ -743,6 +774,7 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { new_document, &external_docid, old_must_regenerate, + old_is_user_provided, session, unused_vectors_distribution, )? @@ -754,7 +786,21 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { }); if old_is_user_provided { + // when the document was `userProvided`, Meilisearch cannot know whose fragments a particular + // vector was referring to. + // So as a result Meilisearch will regenerate all fragments on this case. + // Fortunately, since dumps for fragments set regenerate to false, this case should be rare. session.on_embed_mut().clear_vectors(docid); + insert_autogenerated( + docid, + external_docid, + extractors, + new_document, + &(), + session, + unused_vectors_distribution, + )?; + return Ok(()); } update_autogenerated( @@ -765,25 +811,18 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { new_document, &(), old_must_regenerate, + false, session, unused_vectors_distribution, )? } }; - self.set_status( - docid, - old_is_user_provided, - old_must_regenerate, - false, - new_must_regenerate, - ); - Ok(()) } #[allow(clippy::too_many_arguments)] - pub fn insert_autogenerated + Debug>( + pub fn insert_autogenerated<'doc, D: Document<'doc> + Debug>( &mut self, docid: DocumentId, external_docid: &'a str, @@ -791,7 +830,10 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> { new_fields_ids_map: &'a RefCell, unused_vectors_distribution: &UnusedVectorsDistributionBump<'a>, new_must_regenerate: bool, - ) -> Result<()> { + ) -> Result<()> + where + 'a: 'doc, + { let (default_is_user_provided, default_must_regenerate) = (false, true); self.set_status( docid, @@ -956,6 +998,7 @@ fn update_autogenerated<'doc, 'a: 'doc, 'b, E, OD, ND>( new_document: ND, meta: &E::DocumentMetadata, old_must_regenerate: bool, + mut must_clear_on_generation: bool, session: &mut EmbedSession<'a, OnEmbeddingDocumentUpdates<'a, 'b>, E::Input>, unused_vectors_distribution: &UnusedVectorsDistributionBump<'a>, ) -> Result<()> @@ -984,6 +1027,11 @@ where }; if must_regenerate { + if must_clear_on_generation { + must_clear_on_generation = false; + session.on_embed_mut().clear_vectors(docid); + } + let metadata = Metadata { docid, external_docid, extractor_id: extractor.extractor_id() }; @@ -1002,7 +1050,7 @@ where Ok(()) } -fn insert_autogenerated<'a, 'b, E, D: Document<'a> + Debug>( +fn insert_autogenerated<'doc, 'a: 'doc, 'b, E, D: Document<'doc> + Debug>( docid: DocumentId, external_docid: &'a str, extractors: impl IntoIterator, From 00a5c86f1366212ff94c9ac44e3c67eb232bc9e8 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Thu, 17 Jul 2025 17:14:39 +0200 Subject: [PATCH 164/312] Remove accidentally added db snap --- crates/meilisearch/db.snapshot | Bin 174088 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 crates/meilisearch/db.snapshot diff --git a/crates/meilisearch/db.snapshot b/crates/meilisearch/db.snapshot deleted file mode 100644 index 29377ce4225430d3481206f468db3826b1ccb8ba..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 174088 zcmb@N<8vL1^Y$Axwr!l`#I|kQIYDEyv2EMNNgCUB)1Dot@eF z?6udD#=w31UwmDBAO=$XcE>}~e(Ek*zj*8UC3vN++(RnZu(`JO6wG2$N)}s{b(v+; zTH(9A+1%Bf4w_CtlVYR6qYOi%Foj2_xI-7S7^KBqy^t3PB#V8f4uv1hG!c>{RfQu> zS_qh})7N}jnJyerx{Y9VF9mffX>N8VxXuYaZQRKqBYX{sCUZiVtpF^<*Goo5Awy>MzC|R25WhJ{$xusSC)SvkWw@u&O71c{_z6!pP zTD`-vD~-s`fM~)JU|I60#h22F12pwL-W7Gz!@dA{&wVe?uqw)*Z)ccar#E;J+nVQ} zH~G)j&(tuxkORI#G!kf{(CGhHY2d3D?RoPn;rc6q@s1dmYq8Jr#1RXYEEtX=^8YG* z5tGlp6JXcZrAxJOyfpq<8lHQI&m+k#3!F;Nse9zC7e4r84^A-K%Qb z-uC$SOyQuO^uJnb;l7)){C+NY=r_lbjQp*@|Kg$*$xXeU=CA)ttuqU&x|Pa`A?@XU zXUA_VYjPInE=!}Ht&*NrgBqYQrfFDNrKog-3zjYNP;?Q{r{^h)^}1PZQJ>@wQ&6j{ zQJm%VZa3_sccg zy9rF0i^$RiNccEWH+6Wc~-Ka0^jdWD0H6=10=lU%A?J2n8An9Iuo-4(!&N) z(=!>kN5dUvLZ}TSA+Wi;Plm5bavSQv)MoWGJ5UjoWwf`~KOG#>a*&BLbv+~^Hn3d< znCQFJmVkYSc(;lMbatrE1h{OYSK!>DaNhZ~{JX3MQMSn2RxQcG_1g-TZl_rKVsh4sy|akgVsa{$9sv?3h#=hN zT+BAaw9H~D6-i$Q`|HBdkF~JST}Bj)R#TsfV*r`_#;GVL``AE3rf3FN$>~kyYp42t zZ&#l@#{_O^=FN)WAKN5kNQ?tQt(~j{7VlZn27+PU|*}vv^-O8~!d0S!hs?r$pWl&NDow%U*u%<9t$**oNA+qzH$UAe# zqyIb$|0%bKH4H^czpLr+$b`fQ8H3*yPY&?mNoQe#Qii z%@Vqd>L3gCy)8`jwHG-Y!XOK&b8*XrH;wvlzc6E#mQ=dMZN<9MD244s7QbtpA`}X0 zD%IeK`$I{|X)9`ezqL(Oo>`dtyNB}sNfi&kSOS*Dt`pHZ8mL!*T!!xNQs()r6 zelPBx$GNCw1m!@G(jAt^rw4Tl{OyMHk`&Sh;3O&}@OKcYl*v`zeqqW(qNjfJ0+keb z*8pS3l9!}J7zqeB!fpZ|ya@*>7|ah|qOAtA-%Br&(xjh283n0DQ0yDl_)y6xP;Oxs z6`&Y%oH(fHtYpvx8`2IAR5l8d$dcP2qYzyC2mv19n&p_(zdwr9W#_moT=elX7FAwE zW97WO5c_0CXFwb5hA1hpgk=9R>^>Q zlb|c?5Wz@$5y59hLv>1&PkgWLUYD z0cR0wHI~+@sTiOcy{4pjrKyM=#|5(~%`H~P*2kwZfc7h)P zuf%@KKPO6TE1}B_ar;E9Qi~O*R#c%ie_BEr6JPZmZrn{j3i17^x zM+op|ZtGBM@7=4Cpz6S-`+MevPC_QzR4|bD2mizQHw|)zctRvo*ba#Gn`9hk##P-S zYZMEaB4k4>l3b&(+iDjBFGlBTDR`)c*9B4QCnLq&29j6U=_Vsd>TuqyVONMk$k1pr zLG?T`^Cx9$zHh@aj7aiGD>%gOsF_w^oodgaOIl?_OWdlL$$5;TkC}EbUAtP?f@Llh zJKMB$Mya97Bh8TLqG8^ON9(C%+9@tPQegD|=uYg{EGVd4N-COEli`Peirjk@G z3drsykT=Rl#7pecF2_=iLV#r-ZoV@mKsg}m^!AX>LaC;uM5Bj;Yn=95H}@_QtOugf z(MQ+OBLSl8!2IBcbfh=E)$l}Tm?}+HtRby1&75?6ZBM*DjO$8Go@gH#U$fOlO)e%o zJp6wX4rG4><{4{C`c>tWtm!#1Oe+_L*J`+=!ySgN*OB7>IVj69_5@te@)0g z))yi0zR5>LcHn&Lij*j33w?HWx_f#?zm}FJa9zXMxTd;ahXc_KSRS=@O-${}1-^vOhrd5t{EkM#qJF_GVrp!r=*iy5MAw zI#g;yGwi}P>O``%F!+m*q~wgtAUJ9fvY^mll(N+1ZO6uwi577Uk})5RVl8Mlz+{ei zyd}tBjM;bk-Z@IJoHt87RSLURfsFaWPJwv+214!iByHSnB0_A zrFJrV6-Rm9ZM@ss0{KrF#SDSpoX2-7Anl0gj_0bAVKM$ zzZukUj)ixo*q#Lw-@B*m!W%d|3-8#5Gz^9Q z8Fu(OUkGPXXn=M-!nx^<9!{79Nijy;ic?n_64P3KRJcoF)r8oX>Xa+E1$)T-v6}&h zY7z={1y4VO1diq;ji$}Vi~TbFV{fn;KS+Cfbl`A;b>mde(J`L#JX#9n8NU~Yy<&yT z8C(sy2RG>x*CRX68tIG3zNRL*f!qQXp{f-)`F;bpZ!(${X4qDgV*i*?lZ@NNK$`Yv zN>{ms@l_uC)ZVD_b__`>Q6XN$aM(#(*##v$cOF3r`@Ph2eLKsml z>wA!s$Ko$?$<#KbL|&{d2Es!APO(L}s%#+QpE@kNcxq(bb_6%e_2~L0;5+3bQeSk9 z`YQ>_Nf-PnzZ*u$nTwn{vzzknX^*M0GZI$8UPX|mrTn|90?ri?Ew`4*E>8Z=T4RYy zlU<>cZtHN2C^}5ELgyZ-Ze?n7V?gHkk?2}nG}Q^s2D34#vQaPEU%X^h=Rd|c|KuSB z>bi325>Xym{5;lJkxwZ0)d;7C+M6?X=7-G9nQp#Qd#O=5lrlS`NXS)a+0W1~hC#~^%eQHZ$DguPUzZ8_6g+4OeF1WYGc z3TpeSve-5Lqs1Dg9bAf3DEC$6^`2vWH)=834tJc>nX$1}{7u14Ji-*V!}g^uZ4b>- zSpT|%=tD*&09k-vV~1l>#+F6V?eT=56k1UYi|}_U7=E$Ytbby;+EzIF!9P~EnL_;2 z3PRWcEmY^Ea!TVO6=2*CWNX)T!_#2py)h$NNHI-%rQ6D(Y|~nV3%pfJ5rHVKte2^? zMj-MZg3oB!0`Sth_Nw#{K1jN`A%F*A@4*`6GRH-<@wc}U2+MT?O>GQhe#4ftONN$^_<1Ch=%-=1TF~c6-f2u2 z8nI-DuyU?#k8Uv{hwv^Lh92P5y60Gba+M;lIhMahF2}A@wg;Tcw3R-3X^?6Se=bsg zoLS^l4v=@i2UDLkyb8auas`Y16#4Za@=Hrb=KBD3nQ>rZtz5083%1jj8qCpUS*6+W za3te3l;cbaKf_JHyuumIZ2-KEkGYi^x8e=>0_hQxdSjCvu|q)R>=VWcrOblZ`)lUr zfO?~KSbZzkq`f1;+)a1@fW-Sy6bvXc*%qn1SWxBSL6}U$hW_K5@EH6-DE!(UhAiXR3UsM$az^{4$x!)o@JWbT0x};5_x`;>Ty)(>|=7eB#*@z2F>)rmyh=NW5hoqy@U{8XH zWoHpezbYd_#-{90>abuLg-bK#$ev%H*9+wR5xfeP$XEaU*;upxhQQB|eAQ%|IQ}z{ zALg0y0pCxR%$LK%pXfuMyzbS@5q+|9RobiREa(0OBsw8nPCI3>kIdK4kmshZ6DTF1^M5op+KPOu&x*Rff{d<%iRScj zPiGwZ>r0L`RnyE=K>fqpWfj-Rcj{n^YD-V{1O5z1j7o}RNMqYjo?SRdp9a2I^Fin9 zQPT_kCscbFW>$}M28^OR__7s%cd70>@Qx2e^Z7%@R_? zyCdr_Wdh%K16m&12b00xCW z{$Vn~eOySVl3zLYg`BL1m|PK8E2}d~>?otS>3&{9nKy;-c)_0Mx|zCjb~)$iXoj?r zr(v4wk1~Duwn*|#2{@E_Jg+xfa`AH(ej^`ZoAF!Dq$1-t6cTCduX8fZ&Y`Yhr%CgA zjWT7FB%(QkVO(gaS?MQ-&-@udKbm!5=hH5d=(x?dt{Jn5q-f~+2Pu(0<4eN4Akm;~LF_*+5N;<156wU`@b0O7w;HYIc zwxB3;vS*aAy`{R5%VfI+ueJ&Z>phIbk-Sp-f>Vh}Ca18UiG5A;-qfD!_wKP^KSgGQ%tSUQgn=JGIOB}mM zw#Elim(&qUep|sA?OVf6cXx)sdYgx2SYv7t#{o!u8#(- z9+P^Au`@zGi3pe<*Ca%Oe}2Zf0>PEJYl2%#2xk_KB0grrWttv3Ih_t|?VUTC4?bx@3A z3%hNAF6)|8Q-Hkx5hae&z_|M7KS4=U zKEF<9LI2gXjC=!_jg>TRLBT%Tvz-Z8p?#$I-!##^zUxteL^85W_LwQ~;Y3*Qy7)w? zlP&~wVoBm%KAPNinxd@B3KQv;vLYyX1s~X4!%V90>wz?hz|uz1d{e{BfT#h?k)$v{ z1(0MseX2V^mgJ^+Ee%L-zb=;C_#JDL%P=_+`{c@ni@^wDSOP0!-y7B@ho$^u&CKR| zhV_V8$~O%S?~3;Tkx@F?k4G> z!n5Y~zqopHQ&KhqBi_j}7pwlyMvrCDBa_w0S;~Bqf~%!;X2IkVeoy6L;$q96NxyTD zFPj)_9!7eFA$PTUZ7s95<-nbSo_Z>;tl%$D1p}_1&Wk-wYC_&E{;CSIyP31;0n(vK|Ro1Lfa84yI2y5Igs9M zBb8&5SXrqZ{K2-8vIUv7n^06WJEZM4f4H)gl9*>{ougBp&_UgC+eN$mwu0|A;m%#< z=DJ&u)Wv|zka<&Y(QC!EX_1WD@wXUZGsES07F5i0d$z-rdAYJFxPfr0tRdhc8Y0h7 zrxWC)WmRUO?}E(4NeL4%CBe~kWd7p&)6SJ@*A;^Lk(|1kTy&d5z50Q5hD#J)oxUHF ztYE+LhQKjmBb&-oi*1t%cJcPx<2TiT%=FlOO|guI{*J+kE>#$hIS7yfFU*aLRefZ- zEDr15dkD`fT64C;9L2!X*e#veK*|L!onez!`n2p+LvuW?TetU@xLL&cX}`04f8Io6 z=LN`ugkHfCu-laq(mQksH%_%bTfOn>;y>#ELfC95(e)k1Bc>dA z9m-HCh<++O$nYU6q2}cIH$9r%&j554V`jC2!J2Oiw|H&7xW(6S7vx${(r)E0x+h^% z*RgyB$633N2S`1}E{+WrxJAWV;)E*pF6EW9rnCSIOMc^F)d!8BjJb61*i;IlS7Rjq zocNj=FJd1jX4|-8fP_ZSrH7Ck53yxv{#4-@r+Dw?G|snHhE#A~*&9^GKVl%X+!la#y@gH-(ycEKZJ#u*a3o#-27Tr+>5g6L=1eW7`tPHJ7d?(6Cv+9QO_=RENzwuLP z(f5-&pCv zcd?}|(8vGjV7jZVC{Ul2?6iEAZ9I5(iTPqmP`GQBIsg3i@?y|uoq2HmUWR9W(-hm8 z!K>{7iz@{v_1HBhXqPkE4xVvun^FBh*z5pH6}8_PO(>>gGzQFKaxr?uFkFA9mZyB$ zxl~uSr8%a8K)|LhTd$_Ze3KSe#0S&MI-NK_@8>H9U>&j;fC&M#cIq|pK*G&zjxhbgfKTGQ?@lYy5g9P!05P9~D zB#;elP_n;=mQ+`!zV%ceT!KzR@vUbo=AZsqQVW}%rOPnv6z2m+x&K5G3K zD4iMn=GR*8;hMfEjt-(*xIs@ZZvUhq_LQ4Le*0ZTl}RObtA-ti;PdGLU}<`lo&~5R z2RGV}QgdaKuJ|I9eVuqq*AZj^K}U7#~O2NL!& z9W7`lDI;XIlf`}6DhZcAW~tEre$YsfM>+-%T0Uc37H*#J*U-)*k^d@0^~O^;Mu*`@ z0 zR;<J?8{scTyOoRPM;N4qqY;9M#byDsx-lQNE*cl|f1twT`%Psn zC;w(uw3P*L9;a=adMl)lgwD(Pzf=&!X3Z4r@T-SNL!L~779B}?ox%L4Y>F3utYGte z6t+&6XeTh7IJm6{qQbW;)>Hcyb@0;uTxk1;V!JJ}?D7+Xo?7V8j+aFHfx+MHcgz}!KgSNv` z$L?pOh+)d5AmwAmuHdS>}FJs%i&~_Mi#2j6&!UdtL^UG&9k2hc|P;=dTYWD z=TVm(j}-|~^s<_+eJy~N$ahh=Xetcfl*MH^d4qNd>0#prnI*G7ncR^=YthNP8~Z29 zdHy^_-`(hX%qK>x6UrN$y10Pj*!~n=yUo2A&jkwaWnjxqM;>zX^knX%)nhdZz##Oz!10Ob|ceV2K$1CQ1&Fc@1mB?_ft;JPjCA6l=wr>dS zB-oWhILI93G-;HTjs$6-)2>=}%Qf>=KMgb&sW>)fTy1uikZY{ET*L1rEM1{7!muze zbIxIukOXFq0>J)+~!-V%lH1?(E(=I6fiaFG7K^q=EC%> z9Z+i5|0XiI8;v!{;d$Jd6WsNe&+ztUGDhJlD`LyLc2t!VDH3+@A8#5=h%8BI_91U* zc@RhJzH4jG)UK6S4UZD(_{!z?Z_pgP@BWOoa!55K+0|^QokA;`g)~Rp`wL``z!62OJq#WV z|7FIY_Y6CmbsuOd+Xy$86tYpyRT29Y#LR>i@!G%2T#5+KYYpqau#7~8RirxV)yOuD z+H!a>jK}~AncQ5t28yMtwW7>T4OUMvi&QzAs?#PG_(EOg4eMe=mzPZZko#C=m1mYS z5^!JEm0;4Rw-lE~TTgcNYQEUi2Z$i}xayH5;|(+TP}!X0uvl3!;j-APyr)mri-h*%7lxCg>*rm>l1e6Ns1$499F<|Q@ zA^yRVE7I*2n0>T5AZ5eI+r+gfEG?6BMxx@J{YjmFy|H^D)`-CpEHU6t zzcnR|Z^(28hb3}6wwTyayX3ms{3oHrE6h<)`UgR=ZJVf3CAdda2h&kG@?bXn37DcT zvhr^^FOgw*C`2%~mLf$n=owoIj&wocwt+hyxiR<{0S5H}`cHpbu#W)?`n=A6yP2^Hwq6)2(%~ecE(Qr*q|Z5whuL6d z<7KH*BAe+I+f8UCJK_T)5;UV;=#xQaoc_W5s@%DrUb;BM1f#qra$2J^4|Ee~y*~sj zUhwg(HXvt3mIE2z2W6v;atcH`g>=V3pj2w3YM4GeCPM1+UW8A%WbNh*`41%3$zC~m z-V(90A{RP~pt`cMAErS}bC$PWXnv=Vm;Z*RA_dvtRkrUj1+X4}-2`ctSG*&~M}ANO zuBKxZp;gLN@4Su^`&%&{gB3Vv3IZ|!Yp_feNO#BkWvUND$tmnsQRn9vh-a8aj3155 z7Gsn8@41EUW)@DfN?76?U7h=rZ16%n37T!x?QNvffK`QFtcUD?iNK}LhoROZx7lR_ad|LgRLZ=_B@{aYxhpK&6Sy0u%`bL@b zPU)8;!Z&O0?dq9;bQy!xTUC=AoZ5I6O@COyW^DqG{A|GJxsl3LHR4GjqhC8l#PKK*A?U~5ex?ABh+phoZLj+x3jICHUKhzf) zyG4osH(52JC%i}&Pa2}c**ZHMq2vaUN9R=&yaXCRF=t=2dM1)pKSYck9)2ll+hbMh z^&)!h9acZK6c7&P$M3#TRZ7TY7a4n=m&r~CUJ9l&f*68cP8~9jK~&@NU?(C;Dq0?W zk@{zA_kQe6_Q5bvbINn-xWwl*d;9#j?tA$#;@<3qNqY$`lRw=@aofml3F7mDNU(Jm z5Le$N^zzD%c`j@gJ)LNszn(APhfQZmq9LOv-1!TM*HI_OFo)oPj!7>25V{B8Cv20m z;7F})D~8TZBZD~T1x+-}%(TjI)#WaQbBw2bx^=-f!^0wJVxzgXXK3*Qk)+thsS{Bj zk4rcXK`p{U#%WN$;T(V_3)(L+6YNHF`cmZS5-W1IQ}1bf?%*O7e!3kgSz{xBVw45x zjPk1=jLNZ=Sa2Ja{4ghg*ATSf3%3SBJ1Yw#%j0YAm+Uf}e7JrTm!l`_N!KLzhBR`s zg$fO7tvg&Ob)G1D!B2=m;C4W#Xt;ad+-7M{`lq6I=2>A_GPy!yZOy;w#@8SGl$^2~ zPn*Pm!6Y4;K%NR<0V2VSS$s-|jAHg>F=YJ8G4=Qg5@R5ZHj#W+&5%DE0$L0dMdpxL z1tmoO;z*BD7|fQVw|d<)$ppZ}%erfyEr>ACr&uR2$Swz9s60uPUM31>m=OS`y64IK z2fRbu4o77y@wcoO)}2bCE~VQS{=4zq{l$Z;UQn2<2E^kgSY_GC!8-RvKopCLFdvrF zXa2>0bU<`X;oXxpHz^$Zif9s{Ym}3rm7C$O-;v& z*r=nFLp7(iQwXGBcobp8_a|gYcDrs1kQyx-4$!n)#H}4tG$h%i?jkK`_Vh142UB(b zK4fAhV_0O9c6R8Sg*hEbhgUXBOqG=y2WoSwvozwl<6_HZ_NsHatBAA&wB$gh`S|0g zTz)UzNc_rxQ0uTX&eWLYqmG68lj}0CKz?;xcNHiHPc&OW<%z zx0)DSy*vyTSc03$>Jmm=g*yTH@g*I`Uo#iqorP1^)}njJ9@22XJI&ImhMk+ga5fYpRb@p0g4|!jTBc9vlgNE7M;K{i+}r* z$+*<0N6k*9y`%{XqHfdSf(R~NzN1}TYAV)d+mJu>La5@-oK*}xG}mJI7#mI*4SzJ>XY3Z!=Ct^V1%~s9H|v; z(v>%6U!+*efrx2kFRzfWSbGU0Ah~H){J`KluTiCyUmGZF6=9Dw-$(D0?G!o+ot<$+ zkwMkf<%Qi==YZAnRj26o>bGXx>Qgo-1VU*9cCRdDwgbeJ7q|RDlmklIcFVK?ix40L#%S+_d5oDOP` zw;i8srfCq5^RQs2+jJJ~V9BEQ#QrllvUYL93=L>=^D8=q`V%|tDR?~l(e6|^Wf6fQ z*29{z`A115OpoqENknqiB@61g^b816Ra~X6{aIYGdadYPWy#!#Ss^f1Asm2=;J3q) zc?3pr%A&Zc?hu(>_$lqrY$ec;e0Ph<0xNrumpi(S=xc7aDuz5q27<8%QE3s<5*8YB zStMJ?q~E7n_E#n32#7yBi=>&$q!n#sdsSD;kuGrxB?%bxUu58=IA`es4N5rE9BzG3`I{sc-OQ1kdz7?e)*$kT5S``!H8Vp z^iJ{DYRl|Jt5dsMEL(|!!2|LG_XJ&G!TsPX9=GA8y_ft2p?}N^OU$euo~yAVy(`q` zKt-!~oXbmRYTEFt4De zdiGH8`ShXOZm#yawbq^8M9-UQJO&?4FF85c_vG*&-vTDh&4wtIfl7 zI&>Q<*WTEgLF+1K10<(Myea?R=Sr zAPW78jIy^NcFqp+b$KMmC`Bp^g3y7tpHAwRn*blzGJ8gNCwmk+u*4>p0`WiDRWzE6 zB&th;Qcrxwm7HE;Wuwz$rpFHDM2FzM$2((2!W{8$hTU3pe-`8u{i-I_xkQ$g2P9gs{+PUM^b8kJUn@?UkcLSq7ko zgK_|BQ-&Eq(53o!18u*kS96dk>tuH`J=7+96Pqn(`Tw&-S7z!$M-^&7hU3kuxDjvs zr-s&HDJr-uwO0_=3K$k3exRiMjI}XdmbBdi1NSvzeW_~CQS2#iP zvx!78LkJZ`OC&C%j@QxPQU8s!Iq1qf&sJu!+5b*2LH}M8xQ+w?bK*fq9|6gSbuOm- zD_BAZHh0ml6}r2;g(_(d*ui<|Cs9jZrNJGy>+sF8`cmBL{W6zFjs@Re2=CkCWqGlH^=5F1@(lqA=A zIvwgB!n~+PRrs<-N;m(9|rA4N)PdEB$V*edTMWUru+pB-$Y$Edy%v%51kPg~M>k7wCJg<`hWNCf%< zje({X6?ZWtze%0ICrgL0*4$LFI!pI|SeY%<8u#zNr^?BvracFJLPBku`x|^?9w3e4 z9MV^Q$=9oW!||T?gh!HQvQi-RUSn9Lb!r7H|6Q1SP(M$jN=`h%^OH~4*(3t$ zI^Eh}02G=7wKO&tLp2LJ`!VMc1u?MCpcrHbPsa_IvPZ)t00)o+1Mz<+G(}GLF5Tr# zXSP(IF0ndZ(6a?8%J9Q?!?RONlnuvcrnS0>ABy(g1rp~#%_9owd?qD%Ob+ayt@Ni1 z=jqU5rW?GxB{T^XVlQO)DL;e{s|wVY*^O^sqQ_Wpy1UsOlsK4o+15l@nqmRl5MmFg z$Ofp=b9`*i$&LeiOz%j)Um~s)TDfK$=9?uDiy3ypsdFs_r!2LdGp$>gJ|dvU;q#9d zAHB9F{f)h~xR-;c|J1SRXMm!QA(_JNP?|}u?LNhCz6`*@0x101L*6<)w*u8CPN&1~R~qbTI+3d`3nS1|w@eC#?f1XC(hS=Y0qBe#YEr@1wGO zd*RA*{pDW&_mzbCQ!tcVS*p`JR14W7%|bkeGJ=`kSaQtJSmA@=2wc(vWfzjW)V`aF z+t|pcP3glm2w97K#hVW6{35_W}Lxqe7rBoXM(oH+?1d)n)nH6^K4QsvM@WB<&f90PW^B-bN ztr1i92wwbIo)lF2Q*TQvs5c=om3jy$LR0gq0G#@BS0gOu1MmFgwO`geO%hr7PI|_T z*Q0rCYTODX=P(mei8DU4xb*~9Sr=Lg0(iobMd!r-^5GjT@rt}jPV)UU%f!V%?FR%0 z!}m|hAZFD?iNsVehEmMT4TMz1h}UAr-<>k=Rd<6!_bheR-})H2su(cpJ)rU(KSIox zQ&LJq+rmYQS~|B}Y?j!E7hv}YJFZj z6XD)r<5)FTNPmIvaWoaS#|=-@(tsbOqOkM-7*UCF@fe2^TKf9taEF8Aa!B}mNt<-Z@Q!L>fZR?4wA}&KXjR0-WUJi; zMGnvAU!!tprj+cmQU%P^G@AT;27o3PixEx<_SZ>axLcdMtcWhPC}G$`W1sh{rAFw;P+i0Kt{qaBs*ji{wgXT_K{i^SUgQ zyM&3r?99bIV4zmls3}|FN z@24lq4Cu(+$;7Pb4gNmLK;G!;GYzKUBxT1B)6JiT?xm1SPYnv4A0KT+; zeSgl#p(Vv7WP3EVjfWGLtx`^qZBk;slA>g8Ea9&E^|=>&^J92Z7YNYghli z1u0j{)Kkb~QbSXn^2P)k0FsGD;LF#1``WTeIH5O&_iBoL;TNk=C2;b5FejY-73|;- zaU9Mgdi1Sta<1cvZ8qA*SjNo@TiaWnGTU38%URS4wP!3(tcV9@>a7AdrXSvl<$_?1 zw}(i1aY2tpbRIlVnvS<_vO)c4>F@r=6n-F!D(`z85255rS?-j0joVK6WVov+3z0=uS6=h7{+=d(?TvQZ;Qczho^@C zRlA_>vNK+7Pv_*+TzqP2+@C;Q+NTl^`Md$+T~S^1!5@AoK6G&dq-S{fc#HLQOy1Cw z6_!-vldl})h!QjY1zasVumryZ;vP>H=+h|;4A8|A(MjU60IkUiFVZ5Zfff-sZP=Qd((nir#1>WL$jF=Eav4E$GO<%*oC51@kt<7 z8oiQy4J^ZMyQ1VdOH0VBoEx(y5%&&U{zEW3XP3%%MhNR(j)_6pJ4G}j7lleAWnpEU z&y7Vgg3M@F?grdLLwTD;E{h`vp!e574!^XtA__9GeCEE|l#;RN6)CZbyp=G_Q$r~& z*P`*x4Z&l7hl>q|#gjx<3`N(0fn(7itf+wtQP?D(2Qrdny>@yA$Wgbfo}Y23^$kPS zJj+JY8I$uhLh}d0Y;e%`jh-S%=^;FRCtc+@QfQ4_F?tm5w@RmMprKRzNI%<7q$`mI zT;o1K)1|1+B!45D!cz+zK4{lm%~m*0^LsN`rj=CU*`N%GBCerh1QVVcl$jAY3zpUB zuc+Se`fEqKXni6R5u{0k>$Zy8x=TEww)XpJXi_a|oil)MpyR4NJF166yU0qnaf8Gy zOGQaO_S$Lk!jv+mqeeUb0mlkSyTgk`DID=&&xXi4M|=!iuy@?)NzoRpm%WZjdO*B) zoPHEMiS<$^ktn&iG!A?|8&=6&q`to#%S>B1Rs?qdue2sxH1aIEA^cyrp zwxMLUiv$qU>C2W8xpd%Ww@}jOja@c4=MeOJ53Y|&eI3hEO|3PmC!PKVqjM8R$eQd= zx%Agp$gRq2@A#)lMCCCL&{+^=*k(*NI~|})BxoZc&h2cK0dN2}jc1Peptp3D*xd~q{zlMoUrepN6+XMHqmW|nt zbdRUoSM5hr9`_~M|Nq0X);BjYS=g0YX=rkX@a9DNau3r1F1Iajj60X(a`1mIQK`BZ zfwuZClRbj_TZVwehEUZySsYL9>R%w+J)ZJ|NoJoamw+BM@_4fo0{{N39Ok(rxi$ZV z`T5cA(@`%MW~mc|FGy!0*A0 z+ZrF?$-A63yB?at;vpGHpU7yt*j8V1^ctR$uk?eh4qj=20|ji_sa>LN-FA24j^^Zv zzPK#>`8R!b8!>Lp?!hc(Wo(8tGJ451&ZoZB3DXm6^`X=*Md5PUbioFAGnhVw{^msX zjHiqkCo!mN^c>&hr7#I{$`_vQ$XO7LKQnT8+JIHo^gZIR<0LcSniMsO5!vntQy}hc zT58E)ulrwD%9VTiubamWBmsVh*{f0m-O-mI<(qWZB{a(ni7pk{4F<|OYmp?oYB){h z)9R{r=jn%3Kvnu__V$;qh)qP|VoPE|s=KN4_l`xLZMPpeiS>x0(7~BakxCVE3cO8c1clLE3cF+Fby1rZY zR#n&Oy4BtHe&yTO`j0qdFeGsNl7Td3_0z2JZs;V2vGR%f6e_7v(f4pWYyIsjbNF4` zR$BaY+{FZq483l>MTI*zH6-I&l*FXji{Xl;51MUUUgF}G{dRiiwW@HvXQS5pHg(17 z8ri2qm402yP9Vsqe+JozT`8;dw-FzrgELq8Xd1FP(S+qNWwl#Zl+Ca@zh?{R&-7+H!p)qZWUP&`dJKaP= zF*rWxQ}wE)^kR;fme7Zau=Qp4}_?tlPsuh7bU5k4w?XZ&LR1Gmue zYJOpp?$vQ$tZzgxcehK29n%o6=fsBlZ*ZDKLO+*Lm$8erS)xW1$%`VJ(3c_wBvf%6 zE6}Y%tQ+J0mEFU1R1~ORR!721XC0JhMG*+`=1ju}B(KH8N2qZ>{tA5`8O~s?k*Ux` zeD%yDFHiQNip)0i^4YEi^B2`;4L-|L4aHgI5gloWAQTnV zFGmVF51-TPCnpD%-KX6(_T+5V*e=CJEs`U+00KZmQMujL+0>V<6BgAJ*MZuDIiq3A z>rUW3;ei(_=DR?BGR7mIFLT-oV8?!NKdp$TWb+|}guwmZ9m;5W1}psmb^eckx6kxu z>{7~0Xh=A0`)N@(k^#0IB!Gek-G`vj+`>b{%2{8!VmM7M`Lxfo$_lO4taAgSh+int z=WYT7&or1_K~X)GSKwaL(}_2C?rq!X{!th8QEXzP%IAI@J+B}-`ro7Jc&N10oa+Kv z_gRbHI6U#a#n*U?y<7_+Wf51d?NOpodd^i)|G%?85@B6HUvf7s&F|H%>o(lix!UcV_rH##stK;0iYWV`yl` zu##Zm!7m?!ALflg`!`usQwSANDXq@F&uf~IV&n0<%@5-H5OJZvF=B4f#3eP%{DN+& zg%c#@+Vsi0DU3Z2S^~W9F*Hhrf!1jdTI>f8p~2psnkfd+iK?4|1UX@}yYf;#_9~uW z-Xjln+Dsm_L%w(D$U~D@3I<>Kf4)|jrZ*AumM_r2Y%@M=OYuawip`BF=!uviQJ8FK zRyu@Hea|=4V>d>~TW&$(_a7nAK6(8Vj0Nh0TDLrKK4x1Bsl04f zJ~JJo{rM1n99kR1OaYRAv9e1$TN9Si%52#ooD3*m^bUI3N1vhk!KzyypOQtaxYct< ze0Ah^l~V9NBGF_ELDs7y<`6MtiEiIB3)e<%5Yky4X0c5}CG5~q|IV3ezZZXIHFsv8 zK_d7ROkf}G=_Pz}#v%V6GrW7jC}YUH-rmLB#Wn;pa&43?&^-h*HtO59B6bnx5{3({ z7p5$x?~lEK1rxS0^Xg?fnO}`LpUvHXe?lFeFvMk$<3Lh`aK@BkVySS+!HdO8Syt|( z3^Ls~UrwyhH_Xdi;p#(7J#zF@(3DBW4|{g%j2QKP8+^-PR!*{<_)G6s-i-`bbCXiA z*cunigB1&LlD`U{ggDJSW!-1geDQWUc>#O%8w31Yl>f(=O#U}13d~sjGwitE=MQ!u zjuujv0OYXF17_f2)^M@5;Fq``^gqYW@WBge~SwZRNbW`;&y+82d)9Ar=Ax_+yi0{CGN+FXh2+-ks16^P!Y?Dp>clYnM z@V1aDww?FtXz-dYJ}#Hc$8x7L5frz=bM=_mGuS|v9;Y1ZZ!=1ANV)iF>eJQc#YBV& zRiMkm$wguBeXJ;#|NTW(?>tU=1aGe{HWi5~&OOdLe`A0kH#rsTaWeZeCypqed#@ft zw9@|&MNKyoYT%=iz_gbk7uJItq-$z(!t7NsR6G*ON(dvr0TDd7{q0KHoCthX3=w~B zXYGv;_j_Sgy10(*O+^d3d(hQOcX>rWu3hfxX+60D)o@tA2&L|!acw*td%jlCT2#;Zz|?*a^lsuir?NeWNzP7qrHmFj4aN&jRcAPSgFT<#8u<5Lzb@Y3 z?g#BSJA(O}ACWN2XmGk1!Tnc2H z<>k?HFESWOdJ?#3P6&UZwy3^V>0Rj1`Bcd0G1=oaq8au4qQ+EZ=>Pe7_fVl(4rw9(xoq+^%_8yvRNzLRhr zak`-<$43iHU{HmaB!R~6ys+`@P}%Kw6X0gDG!0r%>Yv{7`JOJvhsg0-tZrZ3I`qKG zh#_q*FA1QRVN%(nU&@mB3`WRuF@_&6~t$EDcg z$H1l`O2=FMCGl>*IIQE%fzFhmvHH%`>dlG*qvk!YM133yBfdYySbSk*D?43Z0qvP3 zFLxEbSkLwdr|c)JCm%rhHR(?;urCVhdg@o8qtx#eGeIGHHhmzERi>oSd4F0U#WVeE z>z4~^Y#Fb!W|xyBFz3m^3O9?Qh`_nOi=r_g?7F3Onj}zu6H{uv@!px%BZlIn}&xh-#d}I8P)~r37p7*B{ngsMMT{ zel3e_mx=?jeuyiIYvvWxpzj8nMh9PyJ4g0J3JOmPk9+Tq6oW1sRA(xajYDgN(uF5e zqfxnSAakfyoH4guGvhb0W}iQJ2R1q`)YAU!#%M5T`9V0nAZVnml%>VurlVbRNSlaf zO&iSBYeT=p%*NinHJV@t<98R5O6D>d9uq3=3ocyj zGnDlXi&d_eHz%nM7P5gg4WF-zA%N?sIB76IeFaKjqgqUuLUQ zNz<+#s@>yTyfCom#nO4a8;+hg{`%?w)p`_A;H^Q^;E7v5QyYHhefAB~2v!0;y(kb? zw@*DMlNp-fKPL-AT>*3XA2$Tbw1TF>^6MR56fihe81I=J*dk|$#eU@9wPJCA;TChhFNthbRvlc{!pF|BinWqo;J9#Bu8nKgerkTFqS6Vz=iL&=G zZ8a=IS9^uzaz6q2UY?U@*LD6}`XR8JwGbmoYVSy=CN%HQ>RRZSdl_xTBX-Z=yRg;j zr{P`d{WhoX4~I~7-Rw(kWpfLG4c^=H;RbQ*bSb`PNh=tM>UfEIhJ_|P6n2HYuH5ej zp%v?wz!jqxeHXr3_sI6rvR6@1CRg^~PW=hV4f;TT7Pvc)e-9~VZ)Qhz#uzbd8VWd) z-vV%=U!2D&;d?>K0b$YDw>iz6F8fMSTL$o3@@4FV02Y*W-g@cT7_g%GeItbGb@ny0 zdHWF}P;N2nL5O7%Q>CPzt`!D)&8AvX=QaxESBm<4LF9j}9!r2oB_I2W^zprLfe_;w zJL-kGe}Bhf2!+QNB;z@)58zji58yU#M%cfY*QOeD%J*HOLWf^Cky3=6{(=`7!SkY?GW9u}n;`-YX=^$CT~&0{q_AW#C(I{yiBreE9VJ!5 z&;5@OKVFS`jHrPH{0C_+{5IVIh#%}~E#qN~MCnDHX9J0 z^rf2v%v8Riz)Wb6m}|W^^bFh2W!UiKv6j2o=a?(wL@uMtSk?1IqeNgIBV;VLF{PB@ z+Z8EKqwXO_C^11K>px_Y)(8p&*s!ST-LC{^i2BROo^>879!@VSw#4HFdJTJMA_X?W zN6fdH%FE4rlusI(tt6@#VIV;bG)zNs@$?_BW-G^2ep{gKzt#B0(aC4&eF07bed2)# zEOOrrGye3vdgxGuz0P3-;Z=C&GEowQQ4OeI0nC#^kU|?DrBi{Z>F>|VT$NHGFOMuF zF5MAKRG^+S-0j!$qfVA4JacUXwl;~^B*EZQCCF<)D#sbD;#zwvfXlkkp#3`oL-~5y z)lsMHLtVLwC8Dz(oe;z{E|=)+Jxuu_z9$I($zr}gFp|oVVvuZbx3^qx6fi)P)H_nn zScG+IqRSpgC(Ghu)$mZ&1vN{37zkQ>?}yZ>kAblesZDwKf~BSU-klJwA0A#|Yr0F9 zeJzguM`5$Eo{>jSk6O%gU>BJtl0!vAi`tT)R&m8vGj(@m|9mh|+gy*_-;*4r5nIe7 z@xtH|OFmmYjP2|d(_No7kay;c(*iR3bPrxa#_n75Jecw-FNNbaiW>`WJoW3)Z{Hs~ z%9u2;IF4@Kzb|)ixv!eNNF<>tB7-6!5K&d*p8FtleZ}yAdEQzVfcqoP=tCRpkPjFm zDoZu+@r%g#JQd=!ELyz^n!HeTQ8jy9HgUqQdz7TKE8~w(pMj>X@N|oYbZexcd0imn zujucUD>rF*4u7RUI)i?!UE|90Lgw1#8=lK2M2Awv;#DYw~S18Fv2E z$W_V8TDU(?2!iWV+{$}J=7YKEj@Gf~NJIr-HEdD57BmnU&)~Jq>a*!6oc!FRu4)9k zG2}dQKxMFWV%1MNO+ZM|;Y=p**o!eBgzJ#==ulLO6g}YZC2Pn;_dQ-J2mz@oJL1EiiG;Cg#ee--L-N;`QFXA zCt*`(qEmGPxx2Re{f4bvzaxmaZ}KNQI!)D+xT^D2KaWkXv(|ImMyo~P%RF5%Z4VvNbSt0>Ah8}6B^67;2f$tta_m54%-@XgC2Zue!A1NP%hhK-_EO2~7Y=5t0uj%>PAH}))LcPp_QId=u_GD6n zZ(NPvb-UMhi-)P?pb5WkVE(%}5a))v5r=pr#Mwsq^Ut56sL~g4ESClv0m=`Y9yLYL ze*CjBUR@+Hq+yC-{l0pP;Euny1Fu4CEL{l4W=x-^P{{kG^JZLe{prCr;<3A}klaHS z>K_%ym$S%Hyz<$8>pv`Tx|?HPlkG0C;hss$KvtoihpBREYgY|iw=~2G$~oUl2#$Zz zu9vMZnM%gF=0n|(v8Ty#mh zB8M*g@7)Yyk)LL6MA7lphZSCbFdQ2G`pQJ~k2fk4d(i09LfT80F)TC)I#9Z9C9<=p zPcn)EU?YFeJm!i9&1Jd#DNxYBO-%(IZ8}=P10b@@|M1v17u>9utsHQ!l1c0Cs8z7W zp^?f7zX*TfHtY;sG@a%|9qdgc-2I-qQ5Y|vfgEn((k9FD#ddZU&Xqr&8w@2fAr z0=2KS7EAvj&P3n&bn{p`o-g|PqNm9b<9dyyhF%Og z1|iND*GzrztZe-hQIa{aCjr@UVilLUj%f%RB-ER9PL9iQ5u+zs610%&!zu)dx;MpUjDIB7! znj7>&;J$V)=EHKQPm}z^N!kB#)NOwp`cSq&Ucb3QNSwCy%316GM&1N8SmbyuBV`u8a@FB_M;Pyo^dE=?mrW~o*BZBwyP`}_+ z6veL_38pRd8$fJ`o-Q%5lOS<6jfEFXd;kK-^M4vW21iV3(N@EwZT!g-CB&Z_xYjg> z=Dck(2v~dm=8Je~8$!<>-5R(r`)$BH67p(!9ccQ#WXI5zOs0u`ed2-%^C8CutuSP5 zo2@M>vHO+F<_9Yh6zEo2siCl_7c%1Y@FT$F10OC}6~kDu-!_vT~N`cy>Zn>%2n6HM?~o%-D3BFp76a;!Yk zKzT9h#|MD+*0~j4m)~Ca^(%bzkJ*o@d)I>20Ex02P*`c$emDi#9dm$PAln1-v<|`x zx_9TlR2$zQ5b{7Ue7UfFuoR2oHT(P_=;N2$^KuAhuFu|RQvq+F^O+Vg=2%edECy3e z&+K|kOjC~d{f>q9))<%1@y}E=;TP>< zDb>m+kjv#9Ro*Ku^^pYW92^VCWs}V8QgDVnlCN&FcwPaN<#O3rCq7y6HfJxe4?N%Z zRmB7PD_uXxfrz?pMp!PZr`2+!j8H6|$+T)@n8&S#m>lJUce1i^x7U?cyxSGqaWm&L z@86V+%ZP%_O&`J`UTV1_uTavWem#uZ@B4W%R*4_-s#T?`Sau(Uz^!5&fw#&hf|!@n zI8s5)?OQVR-RJpFPLu0uc|k{pl{6wq8uz}E2W@8y;!vB{fhY?Mu1dGTNiyB}cd0qc zdUR`A+TlI!aW&rX@~y2cWYL0dXEW=NJyV^JlErIfL{0%LItIx7?p9b@g7J|1k8BBi zyO?LKW@7yl0cb@BG9))zTgVyl7uLwWn=@|U*6oX>fZ45dClGGeO(<2a?^(zFcc#zU zYLLF{Ad-sJPY=L(qQQgDFUJvfknH?t*#_@ZVW|A+<|9~=)E9`fN!dA7i_!*j!O?X91nV` zS#PH58JU0qp%aVe=dlQI@LqymM-g_0T=BYledSDc!N9GPC0hS4BvmTt0|lFlAHbf3 z>Q9eM6S$%_({&?5@-B%Ba$2Ca0`kP z^g#D*yKMh$LDFn^H7U^e`;TQ8|FWwxJX4Pe)*U~Q>w9&mX8~;U9Fszd`t;@53D{?F zc{&n&jLUz0!xk$v`d2t7xR2%X?wT_vs6z)zTNz7AUA-xM$@+@=qX?BwLik~3`V~F{ zGGCGKna+701;iPA(--k7;}xyto2l@Ee@DtPR%M}GRH z2n{VAYV|rg)QkBDmLU0qL`XlvR*g#i2gHzbn9AwY$6N4rTtMgGa{ee+@V`N)V zAGw~mor!t#IVfuIWdVmX`R@`Z)S|Y*a~?5B)I&C=X~|1Dr%2q5Ar<|++-B{q8Ti}1 zw_8|(+JRT}7c?2Ka>ssQCZV(VUlrmsx+p#>VfPxsG|&@*mA| zA0p)uZE)Z~{&z#dIp2GDXRp7yKo74Yf5qd~*x3^kvF;WMflE3h5x;Do;B36q>CRe8 zlf-H6i!FaoHYHm$qC1fEMAgN3t)oSvlIE|3`!-E@C0_n#&>dVfL9O5@JL989- z`yr4cc7LKNk0TBzGvM{M8NfSKc(9wzpLN3fxmQvOIcfqx+9-Q`UXt}1oZ;tUSgO9X zpUPpqJ+Ms2ec7tP)H)_vDv+QiA`bH&9Kd(7t$asmk@-D8MjC75I3VONx4D&3$jB ztBkm7Cz{#Dn#|1Z9cxDmF42|Zwzyqo%J+UUeJ@IJWF@UzoK2q%6B8)lL97~KLhMn! zHx~=zqQyX|`g)XB3d{fl`Sq4^7jw$$JVi{XVF-y2`u;)u9^%*yLry ziMgRG?6vTM1fosCm^c2WgD0Ii-T7Wtlo|^%(Zeoc zjd3(hrp z5x1y6fjh8?x*v+URQFYLpMiRWd7WqsLvPRR?Ma!NRORFEAZLFu0g>cCkG~Y!Uv8>Y zxExky@gPhzI$_+MR15o>rGXE!_y>kyPPM))9V&!09c@k4z+F(qW}GG{U-)@McBaU| zR@gYl6>{`4wxw(GU0da#vMR+a13w`DG48ma*TyBfeT6>o*PuBu^e-UyI7}KVhC6`9N>sQHPyoD2TmYZ@WTE z)IQF255nt)Eo$;2eZKpK&!#f~@%F3pMz}$~ftd^(5q}Xh=;%P~c*p0xG4s_lWXy`> zHv9XAA+NT;^4K`W)4X}^`1|Ts)iOt}io!Gqu zAYnZ@2=J)RRbB6Jahv~`}-j$hfA_F3DwL~5A{euo|-nLtLmIhd-Fg9T|h zT=A;!(__$TtY#5rvB@0m+Re>j=^Db(J8*F^d~^9HwrO$XJS@Pk=is98Pxn%lov0%_ zkyTp`rQqX63;=LoliU-0;2E-_=pE1x6mRSJ=I0w{ zD60JG&PHzQu!5E^+$|by>O>4Qk%csW;GnoIW08X zUZ+(pH=e)ug7G|#yfR4G1Iu0bNQ3J7UY?)Zw!QZFlkhE1ydTZ$Gvp~62DdN+wTD&E`rq7p(O zh7S>Jwz>>R=a1o!qjI94qlog%3x8gpPE|qV^{`~0?pXyh@8ssmor;33o};4)(96>T zp9NsRZh!-@%J)>4s4JI8>SMnHOFX5kqNBp@;)M=hX!gcquM7CwKTOu}tq8Sy3W`DN z3Unj$NPX=OSU)pGT1QrsXOU<`KnlF3L8bX>;(0LDr-&Q>r>vXAncLEtCz z6<`jRbsTFf+Gn0d0LWtoFyZA7+7^9Lc|m${c@d$&Ir8#YS#hb{p89D-aetr@xsZ79 zQ1eoH3aVeP6kjv`M`fQs!#R8IC~yO+&$m}Qs<>?yTZU!cA)Ja&*Wc+jFx5xa-`)Au zH=NF%HrbswcFf)RRQ)ol5?%L{ud7#esM4ueo%gg^_PNErJ!))cX|T0x(+72eemF!m zHl?H!7M!+UP4BP{owKMhbmg_P`vlY`v@`nTyCRPNT28*)p%`uxF(-Z z+rGBOtvam%1GggRBJ~vMibE7_MG$O-C60jRm;F%jRapN<#d7{4Y~hX=MGZ%+<^{M+ zMcHoaXZ1XfQreV`3|i@QdfwTv3f&GJ{slH7CunNI?#%P0NnL`JcDq}Ju z35yN$4O$Cl2Z z&etT^b0g;c{e9QE*M3@dWBa0){YXqe=`)W3<>9mS`e{I2K>lHy`AD!>g_F6GO@5QzT)@WtMOj_yf|A< zpp?JVugS@AHRs^Pd3rfJY%Tw;z5Z}EuP(c@?hu$gez($oc$SBs?RY4($lG?V?y*fH!i?{Kh*YPL0*0a;rj^l`G&PvY8F6$SDW4ZYWF9)L_72WG6 zlS)6ERo3Hi&~$5OTXo0rnny*OqshJ5zM4q>x&QC$2#@RIY9p^9VXM{8xhCiBW!HN9 z?)!wf%kvG@&(01``y#-gdd~nGhkZ>T)mOiXB`*<2qvywlQ()G7dz=3MDPrWoua||4 zvss6W9?3nGByh1$3qf{@+u?gfOKk70>VpX4YeWEB4r8IuqHuSd{D}?6qyxa@^iFZ! zVK(WMvdbMZ7;PPoE(tMTtrbB!(_atnI(h*%LONP($Q8dy#6Imy9IZAiT9{K?Rh$nMSq?Zv={om#I?4mvwvR`X#5x+Bl?*rn8O=u0&vt)8wC*E?&V z)~7lf%`a_M4!oBQ3%CLDg;ZxnyJl2??HXMJiplhh!h>YbH>wGhLa9cm(PO|vA8$!ldk>~NS1;+lsIi!JuxD)%|K{Yq?yBj5VE=|o0)M+Wt zY5I?m#`rw|CJH_?zZRpbY(MM)vf}tFNdA(CrjJ}W<&=S~ai>zBg6cYegs zFVk^3|0bAZ)ZGv(Z8rZrk0*=KpC?#9TM^!n#$Sw8p#%&?45e5)>9D08LR~oGZZzSW z{u>SKZYZhUF3PNpUX=P|H>-TpF5!huJ}$Y`6uBUPX;5ZiV4D$LdA`F%=I=>tfbqFE zUKvSEKFcx|4K98vR%b~#t?rw!*+=k({(#=~+u&l^jo%J1;oVV@l0eB~G@@@dRK4uU z2g)~v;cmnSoXl$hP(-{9_%4daKK+a(!87{ctwx5o;7Z^}7v!LQIWL?3_b&Z+&tV_x zeOS4(+F-ePrh+GdrMC4Wl_67$(_qneMQ8Rz7pJ$63&vUUCAdFj+DuoTe^?x&a@o8( zC@fE;awvoW9zL>|AsBqqrK{Qs`Tm*V*{P2i*Oy)0q1SGs+M^}CEiFz(nd(m(jX~{@ zn3?UKS&2@Kh8eqD$aB){fS41Du*h1u+1ak0IM3E_7ndGfF*5aJ|T?|hpr%Rj=!F$p>l#8 zjp(V>F-X@EE!}8$xjyLRukiVc!>EhSc~?P}9gaard3M}NgL&CXL(y#Ac5%c& z1sQWO)`0>NI;l0ip-gaQaDY`ENzW5jm%E-Cr@5T`Ka`fW@Cl}7piI?XwH4`Nv-odx zSK*wo1=$@jX2YMTa5(DUld0oSM~#R2YLn_>XlC&ImH3N^JU?0*)y7Jmp~)QYS2F;^ zPDOk^R9T>tVp<26S@eg1lq9_55buCV#o3-zk=!a#rF=uw@LtYUnJ-X2+;7sa`i(t& zQeFb?zH`&d}Z7O09XJg zsEp02R8yAF16Ru7NIr~d&}Y6!pjB*FT1~EY9zFH39$n?fz#qz1-KogfwS3d2tBVX5BnrJ!jkP4S zeTXlGq-mo+LuHQ|(<|ajKDz9d6MkiE^FL>w1bfqhUWx3Py}%O({AC!K=d=!)j*+%p z*jnvGmK{~pGIo=)AYBioM*7;z`;n#{u`|CkX|df?dv^>pyyEkZr|ho^$qBS>4s+M@ z=p19+XG@Tzgu=}l_i5ODKK}`SbuKb)HV!hwZ%3pE!`bGh8R2TCSZB4FUX~dJPwEc( z*qcfZ(&<#tL1M%vHL?6&64SiS%SJ}hGtgd^5q zheUb>Bl}Z^6Xz7T1WcD72`FyQz~!9-<2o!=YxvxLu|{IapaPQNz&o^Q#?J@m@hWJ< z9#OrdmW5JBXUA{sR^R?U#XFmPys3gVknLA{04aj1uwD6P2){ObamKUd{N|f6UcS-N zz_rX3HchD>^yD!`Fj4O&dXF_*x!YL~Vmv_pwqy-`e_l7Ft@vdb6I|^1_bGy=pq+b~ zAep(uKQ}q#YqYF32jT+Q!>#o?6ekw?`Jq|#r&VIzB7aF^% zu;WaH<4%~j+wi6ME3~s=kmhiJUSXxpuCz7W{|KDo@i&nnH!I330T zmCn9Km~@S#N6QLQVP@W3m?(|w>m9#{deR0KHB^k40`?~iQn_r}?TA-wLp3eM4Rm{J z*)oDK8C_}3|KKHg`tKMnl*g7vL}?Pq9(*~4+WqW9u$LspNtX)f z+)Z3*maxA%VjOKBiDOzEUR6vlOdju#-FATbTSpeQ}^(Nivo0Iv5I zlih3gk~p+=pC)Ewzhy6Sez%t9ZU**lffxW?bW=_r?xt2Dj8cq7JEIJfEFYTv7J36l zT=`168-8EkGlVZYWH%PL&6l*qbm{1Ql?@IE`A#q)O2_N>aVB9ft0g{2YD#30$$6jJ zO-&*l(|JB?b=;^z_HU_a={M%T-A*5t0g}Pn&zGpq|+spD#$3N=6uNAED?eVOpe!Rrl9G zD|f54YYR}T2|F^|(ki(dr6+Ys;tZp>ATDS6d}Iq^U(``%2hHdwIbAzwvlXLX&_@-E z^PZzE%L`W;9A~cP%ZH3ud4`*!HuQV)RvDU?gFl~;$e$Tktq18?TdQMyu)wmaT$QdU zOipPW{3K^;*ob_d2=3ee(%h{O>mdP)!1MLGAg6oe`(j%VPT$m5-}3UeL=KI0ZJ*Mk z^_)xi3~@DI8&1Y_us%*2xPsO$6Z+VZU32jQ`xK8>iA``9``3)%EJqIcCp<;J*@m)| z!#-E711~E>8eJw&FRPi?=)p_W+ici*bd_3-7!Cx?hq%eG_KQlo#{?t))n{+c@`7w& z+IS@L*y*A)<4M9t>FVGFC=*&|g)AyZF!zV|3Ny<>f0A(boon!*=fV}dwd$SQ?e z=;6jdCjG&a`EN@rIue*=o=Oo|Nx9-C&V9``2G1oRIAy<@A61fW_~X4#By${lEAX1E<}*Pso1-8E*P>2j|z# z+bT*}b>uj{EpfpdsYqzC?+{}2LVqeIbe?=m-)iaRuhdPa6~}BTD6-%%f^*|Q4Y3%G ziQzO#f`k9TfE-@?$A|>xXPeDZRBG7B8dNNt zm)}V4@)GVGV~QzK<}JTD5lz+CF{AN!Zo#C7*1i5`yR$rDP_XdUA%!9wg*t$uQy6{S zxy#h2nCqslKb^8d0WnrPEZ?!1m9{Hl=Ww4N0kujk@IHFz9+YcXpf z4hXQlIXR#!Tqo}cxQY1p_^(11C$2=!Pfj3yF1}OXYHRQ?F;eb~q9tJGDG@q2OBX3S zMb|->Ef`={u zF^jw3P8Jm0sJ~9m74#S2Oo58>6~xm5sb3myunskRc>?HjRlTO=;X%t2%kj%NOJ2(+ zV6~x##=kDxG8EJ%hcGJ=<%^ zYszb#YmB{mpJE|17V9ZPAo|`!?0hV7>_8k!Tw9z$Y;wOxzji;|B&j<95a+v%@Z#{o z@R9&jwFvz+7uXL)Nbs zS8HDEL|+(4C0L9)a&f)nB>%SCXA)K#x2^g@EgrT!b8$SzbUVxwqk=~PI;e0mc#d> z9mcjDTI?_W7q{K}DETDN@r~8cChR~nXWxtA?tsu?Q&;EPjmR=D4S`aYvI%8iUfPsytC)%a)7}R$2te`X^IiRYQDrbM}LnJZAmj@ zXwMCasV;+E<$Ti3)G z!r%q#MkX5~SD#YIV@7lHTeJk0X^vu=v_KX^24nZo&Ys)uidHojzPl9FG}^-c802Q< z*rln4npJ@| z)TXh{PGa>n3dNR?A_Iha`RP-EUERKpi;g)KdrRiOy|jB9s?Yb%Mhs8B0(vor9PA&3 zk5}!0MDG!=xZ>B|g>L_#~RCA`xo;)8Us7iG|PK@vq zhYqJt;7ep1!CKi%r~7IMburl5wqH!@ z_hDFmBNj^Q6D{z(bZR#2IS(3&u|Ji8Fl}~TQp7j}TP-gf8?v%5%U@0$Jh*Yj(}j73 z4dC(|?vH3R%i|EayGFNhV!2Hic;YX+j_`a`SnUxtyI`;RxRXitnpRG>vVdUy1`iNba@178NvrP;bA#fr?fI=93;zHT==y8I88)P{-^E|bp<3-7`{rev!`5Z=WZIjnB{;(m21Yz2R@Mc7Ni3(tYesDYwO{MO>?sTT! zGq~USV!0T68N#M5paVA1_V@|ZFAuA85!VeFmd&aoXo~}uncJpy#`R{Da5TA|CoosT z^vyod50llRaU`V||Ec}&6~?ogyiAxG#ZK1zYLiIDW|?~V;KxW`Z`}$r+$b+{b#9@d zk|hOVg8mu11J-L;xq-fL*q1mD!y)>a4b;~>_!)jkYGwU}+F*&J<4&GwVUv`>E~1}q zZ3}8hn&nq0AKDP^%$uVX!?!?COu>x4@|3sC+P3W3&DwS-ap1q6XW6I2l1jA%;%S=v zt+@}L)XLtCV2lesob9i_zy>7ue{^t2VekED6&}`iJ;F66R!vk=@~33n%yN;()A_vF zaN)H&O1U{XV1P^6z+}RLes-nbOe>+JTqxGO91^rzsG2N&C)4R0_R=}tkI)0en-KJl zDbkem^vGa0s>718-lh8Md_6hy>n9n&V06YEZREndH$$3vzzU;!~>`Fh4O?C{tI$3;@%D{~xmnHf|(0L6aSnAI)X*=>@ssD+8e*J9y4X2&7f z=#{*syz;ZD2q^U9fCjKI!g(uMdG^un{7M>vDO3GfdX zJ`|+J@jb@ixE2I8QYdViYvBa{g5opsnzgH%Jbg`wz?tJmc6)&&Ofeyj3)YH)?C459 z7n09z_`46qN3sI*fkyL3EzeIZuuee4+{CK%oweZGxmM@jYAQ5a0_QL}6hjNllpXnD zALXOSOd^VyJ?V5K0Q-wn+O+&;6I-;+F%$^15=0|I91-PkHKd;~wHf{;La(%n-jfI9 z$k-F$n1)1Hmy^rt5IAZJn3jFR%iv~GqG7clsa+hOmDl0>T;~%@Z!2p5nuI!H9INtb zkmOLOL1O6UO3pS?)HY{PKn&gZrg+#9=2Px=GNLn0sh7*nG$`Tvs)G&i0y&rTVKxATE_E=3g z0ytgbH-9G!Xe|B&d^=6xy0L!ByeT|%DsfWKok~7Lu>H^>x|$KV9@rb$4|_X<_nE)V zTy072WVvd>AbF0V2E)6@f3kg~eQh*5pZlE8kyn+Zp?~5w{kHCj;VEWmd&#R_q#@R` zjKACic-&xj>gmzI^2dpl^Me{Q&Ihj#0?bIe0goz=NRKX$JdfIM;r4SIN};7FRFu%C z4TsQW6adP(!fFyrk)7r_cDvZdQ+Fv!D+(UM`ThE2SEI>6P{ny)=fD&8jTzD7`Q6&G z$lA001fgh~@7h^l%d`7^=xST*`l)Zjaana_+u2cT%dy_P`@EIWVf9(H^n7Tpm&e+Q z(7KPuet~MS8}RLK(S(Tcq2T(Z{6GgqlY6Y^jVHaZ*tejHm%cl><#S+H)yws1=3T=n zRb$8Wo$K;q<9U5R)oDqig@bMrg`MV7l0%h4mwacgKlkJ6j_8m<5>xW@5S;>DrZQua z?kU0lC56c=jJm~#t2}&B(F&HFjMiJno5`N`zgT;#pt!w0!#jNf*zp7Wpcm{pHm;r-sXqp&T zuGwi4;g{4&<_VK*q}{t2`mukerjEgEyWqXl?G2#yQNCZ@eqEPPB2#i5RcI}q9VCEE zbwq)f2^TTSZukd;@5CqcCirw13UJmA%1WNJc4gBkueo=xpKzc_)frvrrXyy?VF&ms zb^U@|-7sy>nM^6p!cemeV?e$VMPVxqW#`t<8BeaO2>vNKszsBFddg(A#iwIG1F+-y zj1!5dk2%{+hdCrSv!&N4WlX87Y%KGVy6V(L#xq0ltAFw|lU=qBug$ljOzZJRN4Tw| z>^MMxII_K!S#+VchP>``K6p#o!`e)DfQCa1j~MlvN810)AzhxS1qkKZ7_7iOBA~6} z@w@)coFo}FiFI0@{h~9(vanl$#-2P1na-3nDw5sN-QoG6$Gj@ukAXA@{X&Iju0qA? z-W+Ujkkw=C`pLAVtfdgUKwJC%Hx&Zcc@9?COq(fvzCs#>-EF{iYK`%&?}c5#lwzVM zP?Q<_Cf%A$;KPy*99c1Zdz)3ybJ(=-`H4eu`r9N+QkvU{N#1Y)%iVo#NBE-;;IENz z(t(rKgpLDj7`+Ca8Di8RoVTD3C@7W3k-K8>l--o0yuTf5{^vx*a%Lq~~J zT+h4A@B4~5V`enCvh0AAN{hU%0)o|qqn#@O8yUpa<0JCRTF2lkpHfMHDTPkP0mf0Y zAD=$s{)kSxFLhc}AWlJ)h9-Nr8KUa4cP$I%Wwj^ixG4n-y#CO7_N^eGB2|8Yk$E)z!2I~xj^v8YkuTGq z>QBOW<$>p#mPEDgcyy}MAI)zI&0oFc!DDL9k+~BUIdRb^n4k@}NR(F~r`UIjAhr8x z1lG-o3V*W120HDVY|e$9W?fSkXZ_J=n=V(%1!Do{A~oNtzE{yv&we z+9yaI?dtGBPQ{NY`uLt<^jR4%F{3=Gtdji8%;ESV0E;7(hT`-Mm_=mr52}NG@b^B3u(o$SQ+ix2IAtg3xME~wrmhNbOpX)X!aBCSR=4^dU4rVpoN`4yMd}L7`HBt?*xnkFAFJ zj!PF*rQEpie=;MXyGQQ68#z!tD**Z+>~3FVin){wNvR9Q2Lpd!;2LmNbJRBoX)rRe zGFhH%R5K8?qoxF7KrOZ_K1Te8si^RhZl$?RxQ$BemhHrvc zhGC0@&_{9>=ZIqUKzeYh(uCO@<_k6~6UfXCHKB8gE0B2D=+4voJ<24~X_lfMt|!iZz;uQ#zh$Q%ZkTaz zgrVw3&gq;4ZSsX=e6@t`m{6i_Z2LZ0+LEuJT&t1Rjz;>Gu1pjLzfBLhqpysmd~^$Es?&rip7OSH*}rG&OJc56Qg!x)PSX< z$iwSPW~h0ovNk><^ZJv(w%-VSrBd!gC09U|zeE7c_oeU5q2*|8oFY(LaT!kpm?0I7;TsD!S%^GCm;ELr>{z}3H`p3_-&0-*8E}hQWBEn#Hzpd#lgsNzfa=JO9 zDU>zuODjxCoi|pNC3R&~v({|@1kFkI#DSqb#QW*R~lzU z3?o66FEmMtqbr*F)NtRXgGQynHr^G?PdDvaL8w19XR2O_k?G}o#W?gWh=$ILdq>`` z=<{@m&U%X22L|a(-?7aM!7RH>&N*YaET-yF$Py5LC)ywPNA)lJhaFpY3=|;s2MBPL zybp8*EwK%S9C$Y!NcM35Q9DgY+S+zm?iok;>$_C2z|dE&sii0*<=3o;evzOg&rsG?9_%d`-bv`h!jO zI9P;vg)SqdtH;J>f89#h@$482q{(Z2W}FX}NPea<1w4n|WTAXWrqv*nrayP|cvPHuXZmQc}!MU}AUhG*GJ(LFOI&ROOhoJ!TV8@pfI~f4CgF z74yBP`Y^e5e5jR$S@NpMf4yny?>Cdn!-P^ev-M|&I)`@1_zg0e_xa)VQ;cj^zMI~O zNMh>^klW&Rf76&vQ88Vr&&|l48~6qwJJ?b!U_|C?MVKRK@TB!U1T62N)W??NY}JnF z6~ovg2d(t9_hU)8B>wuHsa2j9g`S?DzMZw9$BR~}%SL3-gaDiUd$-hL^@o%-246GU z!s@&#?-x1jGL*r{EN+pV=2jh9rQf$Nb{HD&Q(~}0CGhqwi##aLjS1cNAC!mKh>F|f z)v+zu_x}8>3xZiEKxX+ZaHDrxHOi!X6n2`lOw+bKC6{t zFB+9Lj4%&qMz}bfTWVZH)78OZC8vQEzBPS0>0X=TxP;7my8{gGZFs(0NtFNlBO6Qs z6#HAC2&bfee>Lgxm3ZDaJI|Zi@at5t6A=!O%@E*?sFc3fc_d=R_&)v_;o^N1Gv9A# zx{7oFN%i$%GdK~khQj zWT{uM1CDH7o&yhX(TclcFe{}Iu4V=@d;EYNo9FM2Aw!EB+K+&6v@pcb9cDXEuhYJz zy`>_ffu;M}&`)m7SwU5SThy9J`>P+{r*89TTfEl|Wt~Tjpw09!;d4IhNXtrz)RF8< z=fm~(^Ev0X>Im~l{`sf29L;R>(L*~#hT5~l0lj*aRjKTBP^d2P2e zeHO%jod3{O*?+JD!FKJI0#@FOEsHJlj{y$ZSJm(3?@{kH?=EjRZ$Uy;`uVXZ9nY96 zP%A;j=9!Hx_bJZ_&q2?`ci-=*8}syv&Q)IK8!a23+4r&cE3e}mn;j$B{sO9qR2Zqs zD3kPu>_$AEWIfc*lSz}#Q*2XWWblvJEC8}HIz^U8>)ng6cK8CUJc5sk$5Lg>b0%fe zYIeu({_p-$&lhK@&&~IX4D;(3@5NfgSZee=wl`{MB)l&=2Cv%vwf?p3uL|K`fm&O|D|(uKhM)8&Mh>&B|b z=)}l`mD|RON8QbkqSe8Dc7668f2^}*_A0;M_jhRlW-d4I=9TUh$raAkc+3Gm$(cDW z<_@>p+Uiwu%&HsFNsI5`Jn>h)XZ~yc>Xlk>L7UNLOm5@m!bazA_?f=BpZ$yTi^Kp%YMld^vk*L*sY*O(W-k&z`Aj zyw=bc2;NakaWN5f!kj!7tN|ff|A4B;IE@zFnKDqBU8<=F5f*dLIUX~9s5=Y7+!3qo z%CUlkMVVHbHZEwMwf(8-OWwC$7IG8+)lm{-Z=K# zF799Y&#x)0yK|f&jiDA;DZ=BrrD_ISF>wBk|HS&!`vb_S}sieI=&yj5R);y)EOP_Hyh`YCdbPT z?{!%On^biQCQO*3oHHh-Zj>&^M0FQs?2@MOzE0R_)E59m&DJ8U+AkKpns^S90;3o& zsrdpGO|A?@&;sx^MEU(4dF5W`!+uEiGf?nzPZG$EMm@UYnQ!dI+5)jlB61Rs=cP%3#*Ub4JC>#VyR%Q>`DZLb|Df{&cyzG*2dqf!lr# z0-l@@`{_8~@>}uqW1Xx?o0(aqZSyBYPH5lkcD*}3fgOU-$#@SP;sPtC*<$Uj2rb?d zS5?LVLgbIyIEmvTi?9td^hqVAK;F`RjF9Bn3;h)l5TCV~0&GuEt$CfCNC@I|NA@L~9Zx`}{js8F^K#K`^`||h_uk+7owh(V2RRwG9Y0HsdA$8*H-Uj@)viaI8TS$IvzcxJ-4;5=`j_09O zAm1eEW#j~Zg}i+gQKkhE?#si@Pw{nc>yWIm30zSzmoB$Y_4b8I*sYaja zyPB!;iqY$t6@am=isc0M*4%dBMu@vL0jNScDATbt8kJz@Nqj5bw51k7_1(#WH#2ki zA61j2YJCy;Gk_+LV4jRiCyj6AdrLKk=m#1JA2;VO#_|2)7&bKsPWyimilnPXMKA_X z>FKB6R2jH30yFy&TXd#}zcE?o>;9g7gPYw^K|v6GNE-Pv`#^$(i4kq%c$j=9EbRz9%v< z?m}YSP++>pFyW%(R!4&$I;1^han~nW@mCcF(0$8Df9vz7EvYUOm{N+?!1@zOT|@Bg z+<3%Im_3CA2eWULj30iY!XIVPKSVI%DKzp{Q+ zw?m3XekX+ zbHUF;aXJ6zyYw(Vs964BmEndnMUZ-B6N2t0QZqimlT2TYOJq=}+JF3Ik^Ecj7*ACH z+aD>Z^K6mx3XGl{>K_IKmG^m~v|=uhHWAI6lGc%1|4?q8tK2y*wj>>CUe>Iu!`{Ml zh$mH%+6X~zyh^o}AxhPzS(4l@4iJ20j=Dsws2_ns7{d|Yb{-px9haDh(L}C*7%271mGxL*H^MkRv6jIU;uBk^JBTUDIv5Z{8GiBrdHF6 zoNFB)CcbgdjVP+lrMPb0AkaTG_;&$BaYGow-yCs0RFl}R$*UvKDrI-#1}*GX^`sUB z*&&k-*#i!*fE=Hoqu^6kn%x?$Qbet_5*Sb`b)Ruo*Kl|l#TGtNl$&eaUQ#Td{{ieR zy}6;zax>ACfDsX0@5>HRxJCbSa`3~R3uHtG${jRP4n2=*KlLZuG4Y_ujPAS252oom z$`(xq9iLR|6sXII4uk_I;T_QZ1%>WWqYuiGxkG-^YTqr<&gI9=4X-5T)ED=;25bhD zX_kK$lV4Jh@!4v|GA@EeK3g-TvXOoFlv^>C@1s z)+ht{+4vp2Q+Nk>7NpMtUXZtlnkHN;>qH1+G=xjs^-nTp+;U8EZgEv%PB^VL!5=~! zGS?TO6zigsb&ASJu17fqikipWFg#}H;#_2Q2^}s;11gDc%0Eg7%t@^$jo)*L>!v>(!KrD3LxA zgUT_uTQs*2vE#1NIw8L%%o8D*;PjKKF_vn)ETsq30}eA<{?=L)#uF?!VBDZ!lu})S zjEi>2^a*%}#$0F;>2g}Cd30#mHyj$VHiz!eO8J`VcH{NOTHc!z9@KpypiVehDbn@R zAQfc;4PoubQ(}aDs^Kz)h*)3ug0^n7+;Qe2wET;@1;WCXY+A0s6Mp4M*@zcTMr3rw z7Q@4vrpFf-OhNu=MfNZ7+_Dh6R~IsgPlgUcinQM~`FmJTulsp<(uNsf*};lm1W(jh zm2eRK7_XO&BBMyA??<6t`N+1oo~+S3sL~6tVvmoACC&o;UIH=J@^3oN87F#HqZFI* z?FWG{$mF4qMzWW5O+Jcb{3E)g_~U=$otH$m-vg!WiK)_GieJ9uUuWsyu9G{*)#W-J^`eQ!^Cnz|?PRa(HPDF?Wy6 z1-6Qe98C#isY{TexJAJ=+BJvpDGNlJ;OzO2Dxp@(!2HDA6+JcJj?o4p#_9ZFp|qeH zRB^P3=1gdB&`Qf%ze8I(;n>|nen7>X2^BImU zPXFwKnzYG6pW`9V?RAlr0R|>(AM?M&bDR+!S-b#3Xb8{*eaL|QT=hBx?v0Q?v8R^p z2r&6*G_QC`QqZfr`>kxXmNE#p?W-3W0aDkkBNav#`3+H}7zJi!v*j-;O z<)?WJEWYQ=3LPT#6Fv}0uXYLXL}L?`ci+?5&;PwYfp%B1>!djtMU=aT2NFYd3W4tK zAR|074n<812$0NJ(l+SNCPM&MExnS1L)dJz7Ga`i1Q6WGqkC*?tuI{&93$#>6lNMb z<$<1Q22%dzS&ej8v`o9>Q9ZJuPSx$hlOZne=^rS?I-cn~uKv#y3cE{W3SK4Qyhx&e zvE-{lk5sfF>{fv&d|ZxYQib;VL7z-RwmeO7uu?;|o^dzeUqPTMOmF*&urr)#eTBq@ z$((ehm~oIQnMd8gDWs%s`UEq^ayB$lUPrUCyXk)ec8R@A|E~jf6-6-oYe?Q9;a6zp z5EMPQa$N!^;N}C!jwCUCXtq0vaY@2A88!r6_cd>pm?byy=OvqCg*dO*%vwAju<{dym>vfxT>tB^tRdQ9JvCMI! ze?qG4iKZ1#{#V6Ue;h_yR$GQk@fPuz{j{1)nu;6Q9uqtC~+L zTtq+0?(1CDy(B2$WXr%EgWWB!vPIR& z5^!ppu#41jh#HxTi>1D-tc~J)@dly0(N?g(6znBtrZ>F2?9K$2`PaR)?An?^Zax>x zp9@%z+Jo?%i`%tjCvm0xgX0`1LG9S3*|i(rhs^Z@5b625=YvL@lCxPG^BKwc`MKUI zlk-QziIlDxh~&Jj;^beo%A#4@07ykw2XqYDIP-42NuLWa`N*Q)D})#|4Arf8c+8)hm}*^TV>{|p|o1>)us4C^?b z-X|8_ggS3(vmegdeGYJ4C(hfg4-O|bvQOLw+}_n*BW# zk>?oOBXg@A*Xf=7Wi-=M&B7IPCxqdvW59MNcA;r&H|Rm6Uh9FXBQBSix9|&)$#jTF zY}p!a6}ZJVK)%UEsZp;I+vBu2L+aMmx%Ke=1@0n6eIuTIt1v*V@i2y zz=fU}U1KglqUc52?Xif|M!Vu+3il*aiXTPmJmL%yeABsim;^!*@U7FBuWsk7Gj~vg z+H}uZ8~!Ygw4wHB%EXuF1V=-8etWg{mcygCbm3UjOZmJ{OSE+dG9rl;(U#|ASGDC* z?=Di`vVdaf4xyz-R)|w&_F&oL`TLiz&K9&G$VjEhxzVuquDznRYeB-> zTN1NxwpMVVNucx-23=K!!=MGqkHRRsW=EUArsjo~Mr4-4R5%@JmL*FGeTkP->6RpS z@+fq7y#g{C+rypXGJIp4hJ`C7au=>Sd!uqu3PQq`663-m7r;7@t*H;1yB5f@Kwv^5 z9U0m?siEY)WC?r1#>@N=_ooejlUZ83i|B3~U ztQfvcZNgFL3txA?2vUGN}Cj-61qmbnN&tid@7u(~hqY*bNP zv}rx6C$_h7UCtYlPp#!NFI zWBhsdbXiw?bPhIb$?Y`6Pgv3cxH6#PDf!hGz)b#y#hMqquoVM!Pj5p}+F6Y%9_161 z+bkeNYKbI7G+Yj{_hjo2c4iSY#lB&M)ewX2fVkv_!^HqIf)R8`uSVjIX1HFHF5fu@ zWJIzkxs)+t;srr%xs1Cbe{8Z1;r_!qWB(T%+TzHUm>_a7#LNmz3u%KrUw4FiqKyU< z`=Hg^5c{S5@l)=2PHtG<-zQ&;^1gH^Z`sp+aO(9ncF!>GkxJm0=Jat^*T!0<2J3o< z*#vgFS(vhnTOB>xx3*rPY)gvrAA${kzrg~xn1J%_sa32sSWu)olk+IJe-_S&VABU- z&lSc8?%IEBtyu2ZHaL*#6CkF>Q1I1yRP^AvL3*8(RD$@e81+r8zq21u6t_9MEtDs41W0p^AobXhfqy_lWLZCEW73OQE<`Mx4l=yTTQvdM_wo(XrsSjirr|7v3=1BK#lWwIPE{ zhdLT_nq7t^sjM9M+z2~Vqaf;P(y~)<&S1RHZ3-I7OG>7egjP)?6KRFgaop!)0Qwja z9oN7b4^(}-DiYNe=t)IWLd<8=Jqyi*YS$KlcuP!#Y3D0#>>9DbNM>|FDM8n3Pw9<+ z`++DYn1IX{D??K`rRXKQ^FV~0l)X5MvIMQrJ2xTuEpnUCA&KUv`EvhEuyHE~3r^4W zC?*+EwGz+9x?b4=8z{&V5AIX`GBcqNuee6|lMjq7MX)hG-|KwjL>Nm!5JA=?C7(%) z4e(|3G|3n}FI?!2bL+J)mZ3k(1N7S5ST+H~xoiBZqE16Km-u)1hAs=WyLN&I&7}G? zO(iZR+0iypz7?bYu@UZ?6Lou0#c(H5`7}X8@yL?>L3OMCq6wc0!igTV5N88ZQE)2^oe)so6N$v#k zMl~Y=F6N{3qZMS6zn>tnE9Tg=$Ng%ed}u z`I1P{7JUyAcU&LeMk0?ZXm6i<2^ed1V<+YZe7A(Bsw5)_vw6pug0B)=h~>*5>)CKq zbtSnKI;&YifJHJ%M7AVhk07=iNt-j3C;KGyi1()oYt@`rak1Q?M zf(C6#__?@vU5}P7H7}N)Yaf?Z3z(yGNPG-#|4ok^HWq|RJO3V-evs57_3+)>{!e@d z51X*#{E$DojRcVeiHG0g-S+wRUGIFa9f~0Pr=P}0yczYs{511%oDDvu9*2RqghvOn zn{u1EV0IrLzFaVUM{FOwxqPQxCs5c<*ml!y)4=Rh?iBcm&q8n?_3ZZS@tkVw<+CuU zl&fl(x2S4@J|Vs_xv6}L^N97R*JAz48JlKNzy8|gKJLEYKGw3?(%Z7pGSM=#WO7TG z8Ev_%`^NccnIV3&dXDy#>68>Mv}!BmKIxp=SkPM_Sv+4npQ~O(STSqx$tbj$KW==O zaTtLeb#;7eZ_srqQcL$YCvF*t4Sj{t=;J4Q0(x-sM`ql z2-TCNM{eb?3@@4Aw+^Zf%Ip-9|GjIq$SstSp(FEQdHjC~NUH*KNwY}W4eh;a{OoTG z-nKwx8Y)UfBnm$4E?&nsWk++^|JlP@8kKY1)%$CF&B1rK-g@_6{VF7EzutunwF@=V z_0Rv<-|a>J$glik`rYfr?&L%4!}=rn4f9QGDG z+**7f5<5M+QQvm(cCG=Mbn|O|$y{%C)8EuRe3yR!pTB?NRPNR;Kdg^x4=%_jxYWFLO~cKSG{V1)MW59scanoCIku?hX4?6DXMIOmT44Xi@U` zH)1Z@WcqS+;c?`NcEpMjIyA^2zKRllsn9;$o$qbHvN$6M1fIRy+P8G`29Cz|>G_r8S-!y0Gj+B>AU$ zRPW*hr!i}NU>+HnD6|KUY&=;}pyT@v%i)dkByg(PJ)lb1P?{%5eEVQ$-a1l#yyx8B z2RGngD%&-{OYzrqYmo`+2hshEB1+(Zt<@4vdQPsDU?7>TCt=W()72di>0}(Ps31^B zobaqCq^ADnlGeog{MTZmYeA-V8lo$WBI$EG5vz&V$s~zG#q2imlgu6xo&Z+(olY{P-is|T0~x;8+2d=XLtrnH_)EZ zxT!-#w~sP45`S+7K!NM=#l8>wgGD1pCeRof?{*o0Ilw_n1Y6cDZa^CfZQRB6N2_^K z{MWH**lmPrStu#fL%6#RUT+L*QgqWYGC?S-IbP70g4r%~2w48S%ri`wOr`R1f#6c{ z)5kmuju@5+@(Fg*3%FOtlnyi;h=IcV3`nH=k_jLd0VH*oyT2i1l!WesH=nqixAV_q zOfk(Ju$4XMT~=wpRlLBW@4~J;4Uk#FKObG@NA7wG$~j$nBowqD9s0-0_w!5)PDIpW zYURP?>CXmO-p1+C#y=eEr-F1|NLjjRW*=GLnNBox>hdTMi2$!JlQg4hiDzC%_4FYO zdJ1iT!|6&04Ra8mgTz7%G(L_&k6R<9a?QQsU~#9Q;p z27rtN_a!;~aW(QrYfde;LqxC^Y?ogup^(}hIPb5EWH@M}`WBEDC6lcZWuO9x5R7(P z^6cNY!mwL3Ni%>HUZuM$vU>rXpS5)$v1qRvoYKbde7ZEEv=GJGs-6jClwvOl%)Uio zK^XhD8=`wP=FiuWHf>8mM%*n|Jti?fla2X{z*uQf|IpRgaQX#NtK777^ zE(FFWg$!)pt(ig}XG{pB$Nu^sln(;e9#P?8#s3hK-AK5$3cq@{e>QO~=6V^Dkmfc! z$6`Co{Y$mt@=%_=)dx9=<}6=PF!cw0m2dtK&#|YO0;F1`MZ>V4TeS^nIIXu>sF4SD z79%Uimk2YLNz`31qVi3CGb@#B`$9(H_JsEKaXaB=h6L^S#zTc7b^hqm)I@x6Jm_SpQ|2+O{6mlr_N8gvPcA1>K^QjZ65(X`gYQ=J)! zG{fKm_9zSWp0vkpmXN#Y5W9TiDT6vr8sHZKmv_48kYCqU-zz8wR1tAUWNj6L<4XD34!@f`wdo5)o5hiazc7~|=gE0|X>iOD(*%+kg1W zsv*r87QbV#N?`#YsN56!HdY7jh-0kV!)@qfC8ha;axjrnT7)Rjw@%4!tSOwD33#zb zeDnb9r=1RDYL080{lC~X^P{ceO}zqGBkN0NzL)YZtIs>&MYaXL_`qhuV7%Ra(I=QV z8wf|6$de^37RW|$?pDnx=`jRD)Ipzj96z~nj(`K*!XqShIwO?75V#X8{@k&{+v|iF z^UF~gB*35}TqsRg%YOxolL(|Py$N zP2cGP%T)qPhD=}4K9XbuKUsn3E|lB*;2`qpg+U8;`!^AKQC{PpXle5h`v&b><=feN%zrZm8=V9RKBsC$lPbqGi1WK4;&?&L|a;Dr}~L zDSUjM+itOsrk(vfjoP^0oL;D2;9nqKtnL+D`@sW3hJ>y&wvB$k4x0{RL7>lI+i2S~ zy{3j1W{ROrBSBSe=HJZ8%-GC_%;wBo?vio2apn5WHEZ=uVNN~P2?A6?eQsM$Av=H> z02q}B2XQy>@@)02`dskr^Gy7V{M_yv?F(HP=-QFo2j{Nr1$aS;Mz?Qtm7Z-VOque{ z2N!`0h4O_;eHA)Hvo-!I$Cg?&>gZMs%)2F}CaaE!j|h%vBqfeiI;Pjj9cgee+>Ll$ zpIqNu?_R%M6JO)*wR(*r=@jYdUV? zPMD-7mDy;%xzDC?HFe!cY^USRZ8tvnH9R*l&TsX(i}2Q3OFmWbRq_TN81Y|VD$$V2 z`!dl<{Et$}|Gs+pp&>%p*K)L+B9CL1r4sx=AP1M|QNpr6rl7E&4G={;h9Z2E1hRQC z>Wna}h|}Y)O4Kg@u-q*5;QE%htYQ&=Lw4!V=r6oP(1)F+Vrpx-Lf1XWeo15m3f5=B zs>3cdKl`GA(cWt^X^Bmpy{p>aYSij2G`fsD-<|Dk;W4|8!JP**3+!*^-ejRJQ+-(VdKQRIMqN-4cL+rP6V#cPTDZXH>csjn?7ezU{4 zh)KplGJJItVlc8+2(l-|Mf0K_qBw+$Q?~5*^IRGDLEL4k(i6CW!(;r4PHA~pYw>vp zS-~n?xu2e!U|BjH$KT&iG-_`ZFT@vN@Q*P9RC9uPxpFJi{tdy^K6$TsT|$!E;DTiB zWrGcccBdr;Q+by=4wU5u&m>En=jf7ZCHrJtg*7Tv&W&rge!6a_vg?l>ME<7aQrGbE z(HJ_95U!1C!V*#C=MeM92Y1&5R0m#k)HWA&4KopB=?lYb%JwHybG1-UDW6 z6IrpRGVK@JUoMR`Pt`wpSIfg;(kyWbaXqvKuL)?3D+=#NTVY3V@{#FJPMjE>%q$_uM0UG^h%EGuf3Z;9Ej5!>siKI z^HOkdv)_-19%A>FQUwi`@Ipwu#61;ytKJ+a7ujvr1Uvj8g)c+qOL$8Y>=`3$!kXFM_)9qr^ z6|n@dR<0H~Rb3C)=IVK-mDRR|rj63a#!2Iwn|j_y{IQ2$(3uRzB{z{U#xTD`2?;v9 zrt~EAf7EdP-lyL@pOjkc!W+CJ_XBXF^F1JiuSyRHraN*IjRP}OEnZ z&1UHqm^V*npH+TNue)C3XPZQ_b|)lRGSHCOdN9@36wzBcr%UcWdn0%8FGsZ$Q}(wg z`&HK>Sh6dksmbw9?8b42>ohgSK}p(mc7Dw%$0*E9IYJ?dg2s?BH-D1lW<^!L1+5fD zC&98GA=Fl%mYsmJIt_|eH~XQ{>dic%$S&{=IQSBEDXn$J%%T;#uVjPI!ee-xeSZ<8Y3l-RfW_KS`N9 zQ3u$k%}U7%NC&ekqkgKzW0}1ir9`j_SDa1f^toQs{e7+NtCUy9y<91LSZvPuC2Ib$ z0>769YI&KWgh{}1klILB7b6_@ar%hCu;g&Ei8j60GZ|m)41rhLD1mP)lO1K4kq;>x zAH}k|)=*Jm#H6pRv)bWOpA6Z=nAnp&C1&7D=ioXZfI6%MTf!ll=BdeYocH3*7P zZ2-NENP=vr_Kj4^RIt2J{rv+032Q9Ipe-@@95xuCnUo?cw&7@Ssvn9)YYGGD2{^p> zTXIv~orK>AJXuh<)5d6u>0zN@mQr$2ejtTIy4B@NoI7?CVl_pLKDlFI_8N%ZlbvfP zAmR2z@(|aixZnQPO^b*)dOoR?^KfoIk z7p36yTBnQtCeX6U-;n0Y92G2r$1vJ0kE~rXjop7aC?oE=AuW5`SvgIsvkSB)>dy)U zY#Pep|COPgjxJstxXUlkIE)@u`6eVIVCk`<^29767l%8ze0oN04qa2A<R`y(FxB1nc$8{Ay8f|&T&G_dqnt~_{m&NHn7 zv)FYoBj!eP89BK7=>$r>xZ5INBknOfERswIFTB4dKC@h=297JoUsMgRf(J=)>x^h1 zFJ@7X33Dd2FnooJ6nQ(a`X*ew4!V$P6%Uu+Op;KWDJFwiXGQ5IWrul7=wS{ey{Bq{zF*rFrv=XTnSDx#Pt***@< zN{x&Fn&57t6(PB>FbELK@|KyWG4G*qDC||*Mfa9mo4mEC{J-eLA@<={ zN^p<}0%Awh*n6rF!b}RDMb0Q!G0Os(je>XiTS}Z`83R_N5te%!=)FB8qA0VHt-r$e z1$!CAp1PZz_SZ@co)d#yK22?$MlS;!gdIk=hz0cp8c{JZ5d~NO;cyUnIPc}eXG&fF zFV5aEIF~1C`~7Ffww>(QcCusJHt+b3o$T1QogLe@ZQHg_p67f%^}cnw`oq-BRM$+^ zRQL3?*7{wfDba=E@jn&?*NENBULG&KyVdYHcvEnM*j>&iD>LT7F7S_MY9}`N?lW3YpDFPUmjXdL9;PO6hKL>XJTn*sqe72pVH5uaQ&uxY zul!qf{Zi*+KwVk0R{dwf+%y%o5jpFarxMTm$NrqAXm0TGS4#`bHPusY<=*N1pd+3A zf6;i!WBDq--@x|?Xks~$CB`&M19JF-TQ?ms#l~N)9PVi!H;#M~J~QjG%9kPz`^Y{{ zjg5YSrSysoXKQ@{5PJ>{Qfjr@t@o`)aoz#TRO)>CWd}*k%1siyF>1Yz*Z&#t1f7{x zNm1iyCXGYjd<{|`TB&t!JjvO>N^>011C`!>oqh>A_*fspi1bxRJg#q0FM;4Ai5)!9 zUyW@%nbqoaLk)`mhIN%1Y-Dn?c`{^ep1k-~#IYl%4qJG(G1ZV@aF`{9e1PMv{9F^; zP78j6g;4w}Y2woDK#Y-x;;%h|9MZ@xl%discp+%Zz($c#-WeqBDnm3v@#3jg9epH^ znCpid9%Ny1Y%WD#xn8HA4F+tgjVpky?-u#0c8Ng2;U2S)phBft!VX_>&y}g*Wib-M zjia|2XexjwA%d#?$1)TGn(H72+^`k212rU&l9EUe!N4R^GDMRE^AGhOJV@#U!y&@s z4)->tG0d?qotTMH8<2Er5V;?l%C#x~?eDq~QmjEPK}smQ#N9jTRxNnoizf3dsW{me z)od)I_+a*)^q+DpyB$`u38PMkMazA?&=&R}sYh0kGoq&)8l%(JGcv4f$=a({t1cP0 zs$ZTIP0B(Kw%0V;l3Db4Bln04L-)I-~Xr$zByC%&%bhjw$T0SDAI0+Y<&vmX>!M*9;F=WC`UH^L&oaR1|C7q=(4fztvq3oeX`H%1%%APDL5fBK_30ijj9~;PQt4lqnX1su9AYz==zu*i3+0W zeE3&|O22BCAbb#5cQ_Jx!P%Yx5^a!pPYo2X5_yPw?B5VTp&}Wkhic~!Fg!|k_m^J# zjgS(9+Ss@8K(e=pFLEO$P9@}HY^mHH&3yBl$0EdeWna)_pIm2+wha=I zMNIC$w|@!Mr9u{ol6~MvBtz;kA2Gng7xE9ye_wy(H)N%@Bf6=*ZPOjVY)k{a{-vCl zkqQPzV;MFD8Qji|-W{H6hy0bm7qk-|p|2~l#6>r=-qoqn;mP5VEmSGg-2oHotp}-j z|Ma+Ghmr%58iK~M&)|EN{Ff!@JZL|FC%>~s+Q*rB;L(UuZ65b18G-a`IEMW97+Y1I zYNS zwM>G?Ik1zz6^BbcvBN!sj+yl|$CISFe(-L6upO5mQKP=So-P8#HtL~b;vab*ExO=r zLn=byOk`JK>>j00gG}wLc!v}Bs9*Od(pB=J4iE-vbEygk3oizRmT)qPyC280DxS z+0>XLJ{i!kLsA>r7LlXE9X4$;ECj6?D?S)C$f7^6dIXwO7G{V0>x{4^!jdpVylde! zQsa32m}vgZBCzK#<|CIR^+mI+s9ho@NjaLvl($<bPZ~15Ax%1T6`XIqYC8tFTn6?Z(m%+JG|d7~+=ff}&3vdZ>kmz>`*CrtZLW2;ZOKJ|++c-E%=7ET= ziLvdC&0g)0wmN?)=(f+UnVM%7lFM~xx9}l)L{6k;+j=yFRqcJxGD-12naO+N9 zH$;|vmUyK}k!ay^Y(TQVGCS1C)1iPR<&ueby{x#1CrfMw3Rs#`*rGOoL`Pf3E*O8T zzbss%@n4NBYqUV@t{>jpV-daax)2f9kcZ{A4&G|*3bePreIFuUe(k)pGC%>b_%Bpq zvOYoOZbOr!?Jz%|%sLg+pHS}m!dqi#)pP~X1mn6-dhK;MElSSG^lO%WJ*LUY<;9%j z38zoT(udnFMP#%)H+5^LyeAEzlJS?TP&XMm;WE4=1xqc%`;QHR?kdqA!upj)0uKkO zyDM1VxJkc`#46RMXWvA?5D%b=cBOtv$aiRi^cQ~HT$lBGQC-^F#TFDAx)ba^GC*e0 zzvVyW%6hSCY|w&c`KD;!*dwqES%AK-TaMN0J3qi40MM}-*1)d%`k_Egq6UPvB~zz6L1W#w-=a zJ1$Pb@LU{k4~n~Tc~ye7y3-2q+JaB84C3+l#K=f`iCfN<@gB9~>6$%7km9M-sNPHs zIU7Es?L~ndRH?1xQRq-WDdlF&!KivINeG8*|4vaO#_5y&z;xsDHw#;)5YjyKvPG?S zI<{oo)gEk8#G$`XE5l#^`j>jh7HpRo?v|rX&Vs{iPR2ile%`|Of|s;OBkkH08Qwkc zocKNip~ z5cPx6Y(>~QljC0@>C?ZG0_5WQKdp)`)JMYr^V~)F`r$V|H*>oe2*z&wlN~` zRNiy_`^954eyeGlcG3K^xgbR;R-JQXr#^-i;laij_65z6>0zq%HJcP~1D)sWqD_*c zx08^;cMO!7BRKeRyS=-3JA70=?>33L&}d?NUteUWOtT(>!aNXwWDYma+WO1@sGy0%d_EfyUMPbx0AE9MILpQ^7(SRDh(RKLVa0* zgsx2fgf70zBIDDl2mLLMt>LZnt@T!DL-YFQy!%EO zH}Wy!JK$aT-T7VeJ-*X0r-sgRLhNaJ$$W`t33h2;ZL@8g+p4HS5D@cN@S65o^&0=0 zs>$(#(##;lwZodmLd5w#5jf2`zdmI+mp?^3bN@uxoGtLk_$qiRbdekf2+;Xw*fn8Z&G|6D4I6yh%=aQ4 z1m7NSi>L2<;`>@10?~UZ#lESZCQp;A$7V4yi1Grw@AoQ}Pg!N{W;v-^X;D)yvY%Mq zG*4a^lP9FLkOe%o?MBVwAqA@|LKnlMAB8Ndgzwh_`YbV%9n7~613_Vsq1f)tg^3o4*u)!H|vKx3Co`?xUwRJhs8n8Xi!{loOA*OML~k6*uVS# z<9_Y*`{ZCddJrPmH_ok^>&+3(=0LYUg08y-qe z7&#I>j4^M=}pcR{62lgGKW#cJ$J@xi{ z8|-$KHFb8Gy1o}3e*E)&0yy|X@3SG)Se3Z!b(LBFcMP?`b0~)y`+_pz<5Z;l6o1_zjz)8Z)q1Q4BctGzAnQ}&otpMspK;N# zT^B1^e(}`Gz__GRnK`5(Fby3YmmikyR*B&4!+TP&>5;{|A!&!?-Cn!tVH@F$- zyW=@qrPC?di`K5{XuD&!QZUg19;sD&%Z8=9DFX4OW~JKNFJ^{`xc5|SEl@uZN#_MR z#@`#5<@yGY{MM z1weZ>c9z4DY8Vdus{(a`on&in77frCa2*E5(eB)R8FYz1IHIEd5vL6*ZDfT$R}%T4$BkL_Z7Xpd$F&M}G6~CLF=m*S4fdvka2`_UmPdKAUdAYrk?N zlt>jS2;1q^SPyi&54P)bJBwed8Uv;-Ob<VXGo#nR7GT=p(D(Bml70p?M(bC{u3sVawBFP0YH zEY)~S&*+%wZ4(iCYPn0=&!VpY`S~2(OxRteM&{h_T77zsjLuPLr{P`{%O7XO4#|%H z>r9s6;|R@701V?h4lG`+bI!$-rsn+7;ahz|(C>LqtyXUGp(VzK;@_o;!?M!&3Oli$ zAcfwymA3z6qmIh50!dgq>(V-W+v;##cJFkj{<)l@B1J?Pcxav{L1C#&pfL4-Nf$>LRH@bHaA&*btG3v`cgZ;GV= zC7&{@Gpg@8-kAAom7(48exA}qvp9dy`}~(g9_d_buYAh|)P~~y7mKbHIKvMW&4)9k zpovDfF9q~^Zwb8i>j%l1;!@`8AX8y!V7f@HrDZ0 zJ>wieCXe`K3gWuRuIDk9BC!!9Y=qkj?e786j4w3z45K2-{ky4JujLQ*&;G9Gn0FD! zNA6Z5H$01B_5%vF>%RC}(-JJWsoYqZcUCDGTa#ue`LsISj^Jz#&J@n#w(8N zjOK*y`_!3`CLFFds_cL09r(wGYnY|dP3s2E1$?XAMiurLfZIO64iG$m zxXKjSh#mUt60aFCs}?726|{FM1TAUD(gX>FIVR`D9_gN}v;yZfW-`Qa@c!lHH=aSH zWE9Gc0v4Y+QGYo(DdpS_#YMuh=XTIU(-q>v=)+ZGQ1L3a8RzaWKo!;M0d%^jo%LQr zh^RNhg#G-m+=i>{w-atuD%*Xm2g7qwnb2S5^~{EtJi+h`ouDr*yctuZ5ObK#nBdaZ z;JfjnN5NGTbaV!7N-8&6l_>HVK<*y5ux5gKD{fOA6+V0g2-4^A=^#8FW*F>#3|g$XP1PyvzHnuxMJOSd%qt=QR#5rjXYXPL4j`a5||$G7re z*0?33M`{Q2s$Gu-z6HH&UE%XEc-E0K`>SW%UFPgq1aulo|M8Je(blqtWi`gdC9irh z$)QAxOP#o41@}RTCwI^*Z(}8DKe)$k%T@e@xQe^qpTjNDvK(mLU-k15sN0VrXq()7 z7#)UNGZy~Tp?DJwl~$S@=o{lTDEE2BRZ3BPcT zVJk!nS_G(jW$+HIi85P`dVhJoXZstTF7?ouU03GKRa&NDYLA`-Xx?Gy5Rm|n6vy_^ zHh)J3tqFLp4mWhUzt3?Z&Z@RR{X;tni6TPbKrITiGM5UFU%V}R!^$7)xshU>dyL0xMajWxT| zw-vFJoM%n@bLZA8NX`i%cD9kvL~TZ@*-40#0YNab$`)DnH}SR5qIWV238E+a7`uq_ z0hqNO%JmoA^KVFHeYVOvCQ4&st<6ae-mm=-t~iLe)3<6|1!6L1KbqGC0)$K*JJ^@t z1%C+cbj;T>xOZx=_)&~`1A?SgA6RzuT=y&^FFjiEnTQ=r3wvW zd5QK~T%$O-I3={Yqht5R44c`%vP%E$jgZ4Zd|UMTX^|nCWtH#|^RektUbAj1T*FtQ zh_idg?n0U_}&@zvybi1DKU z&9S(BSc9X7UkZ>Q+e4^D_4TaR>TM)x1CemdYPJKGOAM9Lkw@RMcWqNL6Yu2&TVvAC z=#Iljvc0bKF8x}tvzKb;qAXBY^F!SFeSo?fXcWR&c7;xJh;U9lA!$b01S5L65^7+3s- zb4sO2qNbDG(%L_Or$qLuR+hwX7_7pHN&i7r`g^%Z7sI1k_rP!Niahz@0E(3)S;`)x zZ(bmQLG@-RJ&IXnH#D_d$Sz2vipJ11-I)?^w?-LyMEi%O&l%MsIJ>td{0vtx0lTID z&`M${;mLY7Je}gH-%hVe%_Gx}@qq$@No z1jPKC5jf(f!CqW59%MkD1miYae)Jd6DpCCaE@cYYM6A-%zb>u-*Siw~$pyr04mYX0 zN390M;)`hn7jP8+9VD_v#43<2uHX_KiEjbW#UoXuvrTrRbftDh9&h~B&bk71R;yTc zWvfr|V{>CLg%>*|R61=PnZ-^A*z}8W z3GHUk*&ucm7b+Y8^@FLpMPAXCe=Wb%bhMi8Jap*1ZZ&QaW%vPgms~ytTHTt zQOYnuON}lzOPN^OYXA6T_kw^Z7n3GyNJXc&X@mlV&2I^Ub>jT)QsO=0JTl+&gh(uF zF@M=PMZ4s)v>yf#4pXCu_HBTMK%OsTjKwaI3v2!UVctl*1+H9(S5f+(+a2K7o4 zl}iCKX5LtmJ_d={fxk(Y@_x<51Ol?}hCku>8|Yq|VaC5KvU@+(c~wj2Mn;hhmuCoO zkfKWQ9>rhbpt(r5E;C1WA6@5{bN({aLD?HMG=G*tQJ65sn~;i5!#tC$Fn!pC&*KlY zS1(X7kU#q*WuQxW9eN+40)o7nRxSG>A_%!g`uf_dv0 zYf+``KfgIwUQr9tlZ#Yaq6ehF*MxrTTSJnD#L3iR8PO=2loYfWk0^~w;fY>Xem8!N zzm=SCS9CW=|KG>x{{P&0`prCBddMsGA2i_a6S4Tbbtv1+@K>iIu>&{~bb90yNrK4Z ziCGaG#Nnl2C!H(8N88uqoA|k1Rd>zTiih{7|6{`WmWOqx=V#8v^eRI=p^5%9e~S0- z_JmFbPg;O!!D{pcAz-`OyXom_Z5$w9$=A}<=(4<)*LbjYqcy%}muAP3W3n}+Gs+v+ z8arx!HDbaXCgTfX>ZAMV@mR3Pp{}ptZQa=Q)^j$hYRA^J*7-I)Hcg{nx7xFJMLXHA5ubUUS)TEq z$(|*j<1<|CT=_VzT-tqk1i00+T(eR#DKe#}l7S_tVWyca^Oqks-xl9o_n#9bEb5o8 zmnj#`=Giqr?>NP4Ai%Di#bv1d)UavzZTV@T+ql(umEk_>(Z|hQFqv6TP$Q>Rz!H}x zP=Zfm<}I*k`e~|{_0{X;`c`y0J>fb*bA&n{li_^tbHCx_^OkCvYWxFk32``na2=u$ zGvnspy`_IeW{2`uT%SCjQ#_)3Gvds9 zJ|>@BCrEsI+Ge}b!`M!2vrpv08s$+a(JzPE-7lijqhH~Wi61kJwjJ-^GnfN(<#Hll+if7z>&z5T$ zhQD~8?pC^`<)c3voSaXyihw^JaGfnqPPA<7yiMj8wMF`fKF&7{534}HS0xUj3U zFDI9 zae%OD6hiVX3ErF*--F$Ow1LeqRMMf50g3gI>;BaN*Zv#?hTjmVl(*v>%fM>{U*6}r zn+)LeG#`_f?MrPdWbiofc>+J1hpA%Hy}tlh37qf8@ApOc#P}xvlKdFXYu?j``*b}? z$}0(`hwnu4Wz8mR16ok{*nL>&M3@I-1dHu|3e(*J>Mc%|d#la6aX#$-M+Y+(!16M_ zY3&VxB+Q_LS2MpR=&*a5*_qX}aWS;FcX6w6wx7OeTxs;LhUCY4;eIl_cwD)!<$xrJ zP&IxY)WhRMzBh`2&%{j)J%qf%bMrXplkfp!Q)c=9==}{@_r_OS23jNRZg}S5sZ+?% zOmVitZYQCB77&@61W9+;SbS8(+{}Cg?_L684Q)px%9!~%EDODw#8uW)F(lE7m^O$1 zZj@}4s8xl7qtHY+iVJb9B<6`L-t%AZN#VDdXhZQjsPee$=9$ns-WMP!=jfD|9|Kha z)R*BiF~gfgRHR9zLnu%56lnT|lewsuU`W$Js--8SOxv8jaH?8w1_T#l8Y2}U-LDw! zD0*p43gyQhogCKZ)2}MR`2rLtPm;R!{;ab}r+7^DY*vkWn=CuSYgRpE^NPh;dRmTe z&@@7-ujg_NbR_$;10SW_yCX@|m*ES!B>3bC#0ARi2)nx|b#OEZ@vbrKCY2I zK2m>oMZ=}Uh=Gx&`*SUD5)3(goiX(~O@u%U@aret|Eih}WU6F%@T7zPC=R-_%Eldv z47~H4sK*fTUrOpoy?(F-2QotJuM-=l!$1GI;T963!kw3Cxz|KYDmdMTfl&Uj{D@(~ z{S`x~YCjiSNR^3Iu*v7 zhstGuI|o~i<-BU#6|k@l*%CaL&N!V5Mgr?(CpmRm90#B(YU93hnrFw?Og=NoK7R_# zTxU|$lIO|B76s{U0zv4A0v{nRgTpP%ObZUl(*a+UH4uwQvBn*7WR(UQAkl#g_vts! zRXLG2tiiaEnoG(aW5!P*ju^&__@xdY6Nx z;4$XMW3M_X3%?@wGi1i=4Xx5U)HJNxl8=ih4(pJSXNhvykjbpUc%A+T85#`7w)`_v z>EBxjmdreA?D-YOkGw?x%rKPIaAa|+20r{Iw${5}NyB+1Gd_c-?pEeXI2hGXa4^WS zIJ6fBrIVL)t7MF&RL(RHaHdX;=m_!Sn$OPaQXGZ71p4!g<1HF|F6y*A9MPZ#%ZxwH z9Wj-GRX`j1=LySOOO{5BgKy1>CNhH9|EG^VT7n>LBXsKIAtcBvt&YUOS4uJ!8; zH8Os~+Ql^}El5a)hM3!nycLBQMC^D-hzfGYsdJF_RkPu9YOb+a(9f1NI=rrHfS{Wp zk3wF~!o?Z{r^?@>j4`n7*E~p<$WbrQEF;^w{2}xA4x`D_U@qB3#+4QkWCjc$pxn%b zU_#@f(JBiG&Q6xP*WS!^{H6Z4i!%4gY~}xDgH=zAh8ExHD6q>HP(1!R0+m2-P*;KK zQiL(4dJ5!imLl6JT3AR%7m;?2Fiq#4&4|>F4c{A-)FD0TT5{Qg$;ZAVq4+OPR5Ha3ffAvwWw9q-TsN=HQcV2tT!qF^nSV+9;jB z$UtMq@{(9a+BI*6U@L;;ts@O{>N=OvtTBkmSgY4y5KAbJk8k!#ysriFZBVBaZnBjt zQZuAKH4V;K45Q2(Obe$ARw}6AV3_q91C5)*UQnMOW$IjGw*$j^*$)1A@ztX)#^B{a ze#YQLGq3<#BXfiYA)jG&3NKy(aS1ys6d!t!c}vFC9?gl-t0nnL9obv*gIx?xzW*4~ zTZ(^jC*7WhudIfy;@C3_MgF8J!qxmLFmqWjV^QkRT5u)F<(F-~$NvTamV-F9R?@_A z&ilo<=^i5y{sr!GMpx+Xv-MuYts$0+tTo7!kP)hKx$%c&_MJc|DPmm&23VEWCD1sJ zvGq~%OV?zgGKW7$kS7b@*@Bxb*Qk(l{bkECc09LH2%wu^EJ4T1(Y7c))@mO8QDZC$ z{9Nyf`b5=VgCIg`tC9)g0XC%K9CX7{(m%pA4m0uM^sml7X2j8=Js>)%O;!EyWs5)F`rx=;`^LOkmUVT>!G^}{9FsJbnHYg8=RKRMNJ~!cF%^yl z$5WjOi5j`}dX26!S5SY;VbiES#~&S|D%baeY}T7%6<0n(L_1n6Tq~EUSRIegwa#Ls{a{Y;}A|}fIh-=!yl(Ze*ju8gj8mA3cX+H=1VO&G& zq$z^WC?a;d?GiJ=1)){uybRU|H%!4D0yCrOw%Rb$ep@>l$>(_S+L6=jbH;bq$gPR1 ze^t;_LKCQ^a5hVRJR)gO0;d1)7LwSiSdofZV>K$WA&I$V52M2X2N`2YCMxwI!%@l_Z?=KB7qi4_G9<*5bNk z1v{L0l&082^}G_|r)LqVLv5P=g)S8liFt^L>ZreFU->akQolNt%ZoN7zzEr?{@VH_ zim#zLBfAO>Ub;a_EydpRGd~>{tzYa&`<>0nsotOPYKi~M2Q*7@Na59Pjh5h|CheXP z;qm277NjcjK?*HGK-I)jQXnzfloyXM<;092w)5!Ln2xG~NEI77PF@r7Gp1N0IvL?9 zUY_|=uRG*ca3SJ2S)UDMRZn4f{N$|#6j=rVI+>K3I28DnhO(mLPF|*qU$&RT7-WcM zOVyZrvg}h+x>tL^U~#4Cp;w`-oYpHF7gv+IN}`S-T`47(-WhzJ2K>y^I3-@K;pis1 zP%Xc4$ej^+&4o64Vsx)xcs#5%Utr&b)z8>bO=v=bHe7}Ig*bQa$hmD!@=-Cm*5*Ge zf#gx_>|eA*EbNUXD6(NeW|(1Oj$j4OCM-3@{JHJid=ab)(xR%Gchx(?!5)dR?qFjs z9cy2xK~(lH~F%sCOTQZ zjTW6U%OCg-A`%?mv=tNnAiA;k>xW(Q*VSX>rzhTy5?<-8Z2edQx2C}1tlhyq;jw}# zilDW#1O7$jH+_n_sZ~s&+uu!g@!G-Y`t|ZtA)#eM(1rO#Nqf{%iN{pvWFeO>P(k}= z4v+5-8gDhBP<%YJ9rB%s-_MNh^y|JM+*4a_L-kPG$8*hDrv=NsRK*ZlSd_YqKFv~W zeG%c&=*(&JM&vpwqC%;F=+e|uSvYn7N|DKniLt;a_#Cbrt82_f1rGKE8k=-I&JF55 z^9$)H{EeWWb{WbrxC^JRR52xRcU+DpkgLtUV2E)%eAl;dv7?1M!cb;BkWqL+0=w8# zmskd9tH3n;7%Y@}`CBNJY%5c7f_+|GW#X=4>t{F*_9W*aEehe(G=73UR$XPLO;mN#V9Dp-a4ZALXyJy7SN2dWC)5%UeuaD%$3tPY zfQj$#*GKW?^7H4)9F6Ctl4YKy?=hd8Ps(?-C#k2=i=Yd!Ckys8{Hd$auePVfHF1Dl zrLLx@*X73ACxBvEZD|ki3GiIX7Mf3EoIxEmaUq_L~$sJT0l@;>z0dlhJ#t8=9LRDJtj$kJR)arx={ zocO$!2j^z`*W~-u`+(=T=a%OHjpwsl$Zh9h>GGZ~u-ir8r-C>KNa!+t<$VRc>`1W@ z`rdw@bsu)$1RVUy5#f2}8OH0!25Myio;aSwpKzaGp9-F$y{o<92<1gDLqFReD<5kg zo4|d(k1OEMfAh1X)sQSzA8{N}42*v(0*5=6d5!2(`+o$FjN7PN=a2pu@uBlu^L@nO zZDyCf=i!(^eMSPd$d_m)47;dr_V>+;&LPG91kFh|PnQWd%crna_Wg|u(0U59%2k1f z>xa|T=+$mxfkFORftb#s?p%T5lt)+N=QD=E=o9$?rMK3Vnl5o)qNjQnlBcAjLH-%N z+3Y!Tqm%{-+Uie!VEfbML^#HLTZ2G~K=!N8>-=GJvc857@aQ#7+?+s@p5c08qtj~} zZh#3ZgR5bKIYFn3{d4ibx8yzLTk_AH^Sxme*WrHCPAOW(urY_Vv#I9t=hqKw;ldSPiUu8b$Sp)9Fr zwroXdS?PQsXOu8rJ)v8?JIV*eo8eXJ?({%-lx4KJz?HmOL_Pk0lKDCEo5lA>w+^IO zF)pLOSYD_1cO!fK32R2~j&8TvqsfB+ z*lk*`7JHMssGws^0^L%2iv;6@xKRP}D^1S2T)LmU2!UGj-T5&67@?cB?A znCFmybVmj!^OxgZnTXX#??9Qz^-ETm_1-qq&F#0TwftUrt^N@|-n-*fXRG?r7M7>S zRbK1y@YKX6-}8xv^To}^r|~>cd^oZ;thT9jd+-041@lwWaQa=l5b%N957kDTBHp^` z(W_W{DU@Y=iAIC|U9Y|2uRSukGsKi7N4oAGaV*_?-HWh-!Y2TIM50Z)D*-kUkbA^2 zhP9%zzA=W`wafJagJru^&YNStTW%zua_8S22-Pxq-gYgQX zK66XE-biN0HVuYob*^?wL+&;T#m;vFsBx~dnVyvDW;d`cF*kwpLS>^1Op|Ml++{dd zi+Df>ZLKm*tztFj#cvjqt{%slaI?`G%B8!Kw1jgiY)`lCEwRZ)-IKCht9Ss^AUc^= zyCd>~S`YtH4#S%cduTX1Lvd|Sm2YS;cwY)*`nXjp(wamc2d^pR!frr~;9quj7PjC^ z+_j>V{jhP1gr%)vU1zLx(z&465Fj|hp5!gftukIqpOPMQTy31AY*wa9qGpr<&t4a* zGIMKl*PsSv$q!kbPc*V>(SWloKA#vP^RtPuU2mgE$|aq@sZrr0YJ`t$lM}) zNptAZ*rrwiC>ue`;rKttT24w*62MK~TW}`)ap7*6mTFaQjR^ASwAZobP+%`l@+j&+ z3}X_&v!5G6*&v8J3otieYgVSzrIjep+27IJKg&{*#jBBKj&#ZhI14T~-9oP2L^A7B zv$m`K&(A`m${IN^Lc~(O#wCE6t_Fn_9w;J6e1wDr9#IOdXO?aziGp06-O`%uQlL`> zk?b06Cavh#!?4BXj7wYiPY`ik!^8G6`Ig!3pOfO{5`03H%ArN64*oTgRLFYmYpa|WSZK| zTwfalBkd=}I)BXILIy*Te$-hR%2!rH^k6$qq|1>y!;T^F1+mAzqaW-rvoJjGswt0Y z2gKj|lG+%@^l!2oWxv_v{LKHTJ?qwH7V|4f6GO|xf>>w&-hDJ98DEie5w66XMPIlZ z8ruvGM4W{O{DMs^M>03`w=M470cMjv+qpypnft0 zS@Cg5N{DjZ{H!-G;=3asF((L$4OpN*4A(3Q9J$piVVgR#M3gr-HmQz)k2;?%53so^ zP0ci~V?O-UkfpX63|DA?!AYZ6)(NuSHBKRCopIC;duWJrgSTKLGwuRrGb6sC`q=#a zwKinbcPPgB2x&UW5@(gFg)<~osZfajv`DfYgNs`&uR0e&xPM5}|B`?&6iOQ%tZFQ@ z<5-DI6*28ciIKai%QA?gSQ9kO`v1y7nO02Xhgq`+)(1CS6t(K-*|D5px|U^9hJY^; zQ1ut+r!*KXMA&x{2J}sQ-im{?q{?`XlHEXn{H)|H-{Is4%vk&P^Eb)jFRa@ zz{zmxz=tX~_A@O;+;mvQBbDYv*$%9VlxQ+(63^nvuZs*-8I_nJylF-54YAZ*T3u}@ z7Hc_rP&=J!y--G5*>nQZ;ut%mG%QQ-CAHXS|K0H;v)Xjz4-6vT-e|=$B#vu9Yb5!` z9K)?D!ElpvzV12?M@n$%hH$k&7a3u9tCl6h7RBeM?c->8Feo)RR}gIf_8t|iI{&lgu;K~ zKjp^;bw#RFe&fF&bYA`Y9}Uw8(?Kc}L70`5JM+=6?#raJ=*LeMHyBI~VEk7&I?_11 z6ARrn7;DD{4y!>~i$G7^(EPxQLaxJpG)>}dbQk?ggr8YtNu6bzQ`#Qok$6YQUwYI> zQ_Q71(bwmfn?5%9ev1@^y2YXJjH29NTR=AHC5Uc?gf*JuVn>6kGx3~1@%u@OoB8qC zJNm?*Y(9!*=sqeV@9nH)$2~fKOP5hDVg-^VxTp+tm3p{w)O<3~@DxNHPY$|KPskjc z(XlwsZ7!VkiXmeRZRmca3(1R1(#Cjz_av#bGIVeWDeLm0aVWX z6_Z=ct|KH&3+^dUL~XLoSU$%0!o@@O>p)SR=mGrcQBc z+;g5oDJu0U)5AKZY@Vo?is(_wEzG?HFi(Xy5;IN=8#eMoh-wu0^7T%ba2lJmr-}t! zH^NKoI(>#!hm|T)@v9yf5dMiCTB<=+KR#amWhN7SRZgbf&r^au3yNHfExKf>^`uHG z9co<4_0~zLv}?H4t3Z*WOC+R8Kc-*0qTxs(@&`3v7AKC<1|h%Z=?Zd@e6*F1$klA^ z3HSNx8uHiaTdJQ&E_?7&7=9>uCPiLgqpi2Rg~SH0%R2U~O_!*|3-&t4X5c@Ltqm0K zq%7`O6O2T>G{GHz9>z=RFho~8*s%!)nR?@6Z`rUZeLzG}Ewj>MGl3aAkug?3z5+>=96 zS?*v1A9REuWQn4p%#2=r3kWc}4Z+mi1>0DQR+_4&i#>%j@??j+z?an%fueb*U|u;cdbe5CTrQ=US3^H?kK4NL?bSwWLA zlzBjw-`b0g!Tp7M2YB-5939lszbI-`ztp8iGVsNn!i7X*YfbwTHmXJp+tiv|CR?Su z(*0sekXL-x~`vx1%-O)5v+!2m*N!Sta9f==7vWHNecBku3NX$ zY^*r?w({<&-B>eoAP63QbjHA1??2i>i~$}1ul5NIHVF)5kyRj41ei`na(A@)77Bj_ zm3;%g!h~h(nb>ZA*?|~AHdSz|zTho^2&OS)6UqY0R5zv3ip>}UdVrFGRuxn8Ep=_={EeV(4h9W-vnc zecf{M?1^TSg`0S45^+>kVzUE3PPx?1xoA@^IT1l;uef8*ewcr&82kx|SN^h)`O+|L zup)nlaC_@p-_*(!C}Bxm{FLO>nL3oIlF$3jejV=PGwVh1QvA#?Jv&itXo6dT%W~0F zp=OuL$oruW&K*7~5wG(>vPf`42m9#YV1_P+ z#t7IZ41@nzgrbX(A`g+02<`Q5{`SU3%&r`b1C>z-t4P$rrInKF-~9^BM2+OLRq=rb zfvFp-CQgnp)MgIpBPOaS7U7Ik(@iqPo`jI)NE4CYb-1us@8oxZB zGAhbs0&jIgE?+L!ctt;-U-1p}Dp!?+e24HXNu%>-mR`=!grTj(n zh52R3h4Ry-+c$qy$cg8<^113cG+?+57tn>;lH$WIz_QJn!$inR$XIW-Va?{t#LuzC zmd!%IO28P616?rqqWG=htLm%7aPDew%2)AS=AGAB#8;_1b2ZDXr2<&=A@6#arvFh6 z3|2L>HEcD%xSZJ@&Cb#m4j0tUo)S~iHyK8WFOK^85c!1#Y z;u_o=4Q|2R0|a+>cXw@|k>J`u(=dGZGi%LSbx}W|>N)%D<6iG^a~xiYcE#S+bbT@a zMUyA|L)1a|mh(*H-$|NjC2xOo@fGhQ;j zVGb>>R}-m!nGI<6{jAfbUhYJA*6v%cNg-P#|FzVK(DPsTHjAEj84j+o?4NrhF5G=F zSQukU^xI2ty*{~$c2t}gFVqH&sTo!eJNTJIPgc>ndx5OfT$JF=x2D+a^3|!ng;TrC zKfD)t2V@bhU)W(#6PiljN_THS`Q(Duatc$eSwryFm|dCOrRyuR@lpKSt(Gc9$UdE+ zLb`#WJP4QvYDJeWT=K0Ihg6djZ*>$t_EVynH5qQD`PVa};v~Cn2`$wY?>SzEjq;{b zT1(3Ucu~Cx5?}=+mn*~VYJTyxvV2sym7as9wCIRX8-D)7{>D7gi-}5{ z?w5Wj$(H=#uxqkw*PL+1pLmwyeG+WNy|{Iw%Cgj8Y4zpPtpWNTR}Q-&^t0dw$`+*; zkkq&qiiHmS2ofg9txWOs;v&U2RnYNq_W1hZRFSf|Q}tpVQ$=`>(BS*QZ*7 zNe?kf2(doylQ`bwm%t;td5Fg9(_Jzf2A-;V8up#>Ulq5fIpO;DY792z zvf$XI`HAd;YMt391uP$tU#re>)KX0RrG(-SkH}C6uU8nbQ;MuZm9p5aCp_%}jH_{H@^Rt{x-w2kS?C3#E2K2hVJ#CBmxlesq(>*uY##5|xM^iIU*)&8ve_rRF*eR?#ssMngCTf|VL+=9Pu6Da<6P zE*w*%eDss;SYtM4!n5l-Tp#XT4pFkP-(FOjABF*C6eVZ_0m@S-VzPW%ykU*%SSx5PO5NLRB8L2%iF z86DK|ccpj&cAv-ctplTEP+|NM8e!pkTPeg7s@|E3r|1?i%*5c9E!bnt`GjMaeBc4@ z^)>cMFQG~~dgbzNK9>-gQn}`~sO3a`r_SGa{?)-)1HMSQHbw2{`cQ z3p}Ray%Sl5l>fv;Db2ci`c4PZ)h_Zlj>ZbmpL|aE5xpri3bqy!XJJbZ|D%l$zL@8d zDtf(U_Qu{AQ%BM+9L)$)r11cmx??kr|9VIY(G9P7TPbUv*ImE=WjD?D@5^7;rZ0T) zd=5m&u5Ip5(VDJdP^lxi|GD?h9I(dYky?HOH0xlSA zkjNs=oW!R-T%-U$gA;o}Y&ib#XC}rN7)h-?`h$*@ptxiVrQxO)IbKVWA$H)~6oUDn z&9yKCPnZpQ7exW5%XBAeU+WD=rGoZmF@|sQ6$cFq!VrpjlU%XHkYardA`b_Ck-`$*rgyx*ePn{`f{b|dkQPk-%pXCQTCS0`u0T>hGx(gU`@h7-&O<15jR2lTXJ*Ad0 zIg{f94~emFGMMl`aQ%aPy35DDXn1pkIKgBs8b2;#2W0WC+w#NuEwRLL?6xfW+_Hs()x!?LdvVtkr(N_Dh%fI?TDTkOiASl*u#Qks(A=RVn9 zkA2sAVJL*^4rrvy5hxdQ{oZAKrNg~C4MgiU5GZ??5FA_iCl?a)RclIeR?~pw0S$SS zxMQbW9mfY*vC{(tkx{QPuJx4B8z>R-?%u@|Ue>UWTX8ZuqHVC73wrYQiocW}S;3P2Q9zHvHJS>KD3+%9 z!iz+)sQN}wDxdl3Q4c3c&XMC#BHXPOf`C09?SNlFFlTzqRK{mxU2r8i`q4p@-*5M= z%_5Dtpgj@#&4|7IdhgvPiP<44;?n)Iga)$bv56J}S$n9^b3i*zQUY*S$M1SFJ>b2O zvT%8H8&88Iuf%GunWO4P;(w4)iBZx%ndV<&7$agZpelr$wCXLFJDjuMj4ks!-*pd39s|K#$6;s|M^VX(h$B zTFG6UnSGnMj3boaQUdfR1w0I4#Y*;3C)>4YL+%(q1!y;PqPSS|F~Xv1mC>ke?R z##rd01n^LjV39GxhNhLq!?zVJQzT2iRS+9wl$abXC;>vLLgEnb1yRx^7btkTk3Z6q zRdEA=J+%E{^6dG+;;iDK-=pbG>+#?`d;Pc}y@7NcZ{53v*3a=GxtV_@Pp`6OyE#q& z%mgBQ7jRTOl{!_N>3UQXoBDbBwew~AAEa`&R;K|}<$nNalEGuPASIh0wMA<3- zLPD9>LTg#IJK)9niu5YrivH@0^Q7?pH@`8+3Un@DI$&V?=annB)6(|A+qwV0t|Jjp zN^U3F^r_cdOK;0=W9?ePn!%%@K_lb>P<2%euGXy@t!6(fNv-RsGMQGJ)}3aYBEQ3Q z>yORo{cJ*VMWP!!IRf{~T7?*JwtGT(qI)lY&khZ9Iw61Iy+FG#x~au4=cN!(K%BCL*ogyZaOSCp7x6w!njMA|ej9t!`6xbk0}q_w84tJKZD7 z%-LDWyo6c#BY=v3jyLno$Bz0>fh2`jVer9LnVa*pD7TmGWnTN#){x=k*6a=EzVmQ? zj5slXDv+lJC*f_@$|OI)>T&BF+9+?W)d+0_jP%3@V0YsBiRi>{6L6ase{|-1@}FIf zCZ-0tv8Ko}6{nhJUU{H5_c4pTp%Q^q57?o!te%c6-fO(7efy=n@dzV zn*davOPKx(u&wqwAC~oRzYp%mqsG#`^hu*iOkG4d=WBkD_CrTV~tz?WPrAB`A zXDce=WDZJeIs3`j?u4;dG{JJW2iU7P&wN-p$3El`3^M}>))l|QBslG_765K&C)5l{`iXn*vdpGah?5vxgTA38N30>1CUahgvyyZh6-c}0 zHK?P50dkhG=vpn3P_vmqDVJEVwK_xgU5{Q$7iKt_VE;i0f5G*0V9XHtvQoF^h|2f% zb;f2`V*^euuV2(rQaj@2DNMJk?0Xq~nV~Z)P=~1iX^VPvamKtt#>BP|H2i;ABnQ1I zC+}wc;%xzGi5=BDto0odN#u)(loZzOQ z+0UG%VkIP~#B*pjdL5-?RMZl5x642Nga)6PAMD~XHxSJ$E){@OI)R>>qSSVh85WxB zGAl#1(r`Y5-8qPK`c(G%OkOWt({6{7R3Q!S+_Tinu}XA7U|JH&pyoh@DS9P_M8Xh$ zjozGuXM}6PMdrou^w$3-YGO$4comrD=HPRs|M@VKN=Rg^G7DSZtSDQClEKY?LQABw zEu|>xNHr~GziF!d>_MaWh#Bnz-x+vvxJj~#;4duck##3uDXHvg?ho5b(e&RR#V6J0 zka%{GSumMtYI+yZuwm!YaVt>QdoFErU~B4@i(M+OIQB@nkN|(*Hf=Cnmj6cReo`m_)(VzUR{gl-tFQ z1!=`52egV_;%%(%@rR!!@FkHK&{CM6liTua$l}^cM&@$UEDpEV+JH1Ug1NtwB&kUq z5{EOI{u(46Hx|*vYyo%8w7P|eXud?w?9i@~2S`!=7qFB_3H{8`CH*%=lIItWd+z7K zTsGaBO|c9JQjJo_pnA=8u7#Mw}^ zjv1=hYPSZBdDFQ60Z>CVB}=^6tR#NuvK(zc zK5V~*ZXtnv0ecA9Hy)NuWQ_jM)^sh{!g?YNM;S*>0;JoiB7BX{iC`hIT*puF0qt>G zUYo>3?&bKdbOm&zGri0%`;f6XW( z!0>Jqv__EtuWtp!U;Yios@ut&Ht%8M5UXcMN@1Fu_eUTYwW!TV%I*mKRz1Z+_K8dk zsVJlx1Gy&IY+H3=Pxo#;t!Ki(|nWlgohgxn>&fo_5tR6P>JPijd9-0=?OGn zHt-bIs{hHdi7Jz(#ym^NO_cGlA%(Ezd{to@*6-GvS-!sd*Ix@VYR^YXQjzdu--COE zJ2@ceYD;1=Zc9Hko5irf>cAmtBKi)=L|)6wu=4J3Xv)4Z z7_*O>cJsLlZ_50aaMn-*#_Rz4TPo$q9U+V;75>soShV&4-=3KNtze*y;5viQ?K({G{WXQhi3iCC)HT7} z7b++azsU^ZLwgl<5$GKI&RQzf?AnvU>Sl$l`hxl7gFQ(%H+D7YV`#QKCYhT|H32#><(+O0bBR!Oo zY>{ktFPL8lZ&TCDUhivXJ-+MwP7QlizD4pIE{Uz!b=q|~&hFB%Mw`iGgFVGn66Q;V z6&=z3y18b8sa?ObK*IRJ=yyzxB-5tCLHLW&1Y4C>JAWf5ESIVtWG~6UCQE9uE1-+* zeR;&0L!gM|9Go-0;O~OpgiJYM&%LnZ*mOs@xK3#P(YV1x>aJ|XGDNiAOsZujCVxx& z_lcyb-_)wSs!b*X(*lGcDvy%;ko>V0OMr1_$K%mPvOTDIb5--hf!b@? zGt-stBY0gdBj_%*RVPoMxpu4hx8bSL zGd)!CW%{1`6loEwB%!4FUXP>{a4mLge3y5_;YWW(aaC^Zv%T|Ss=xdWb_5H88~jGP zCq1@oTc@Gh&m+&ZcMW%8i#U*gKbH;yqdA<`JK6I&X2%m#(%j$~vb^U})tOgQfE{?| zRSfzE+5|0tg6`oS&4kUoHm|F7<0{DSjPDLptQpD}rfMn(?}!e-C#&<|Vov$|NMqiK z`PEp)IJc4Qc}VP_+`tL{+d}Vr^@+_r9CUX){ekg5pW*Cfc^v7UpHb~3O{`Fy+s6m@ z6ibC7`}GIJP+7%O?d{7=&ExP;C!Q`t-YP{N)3=O|}=k z2eq-ScH(S+hS!l>i-TDX@iu?E=NU_99|uA4K=&<@e2U#hvd{0#4?h3E^{tvFaoyq{ zYmhM&@>-xJZVRlq?bHU%?Fz_9gGDcAKr=q6pI3#)`NRUP@+xLv;B{Q+b7>joC7DQz z`*Hu0=H|lexZ3YpdQQgfP}1?@@$kcw{N2lpjSkGF2LpEb*PJ1Txj>}`|GuH6er%tVf>p&s<-!( z)c(|UK$LI<_uviCf#xgOw^y;Hv?trkv%+`GZl3$$twHIl>0A;a!6vRHVRy^Rn|1)= z>=cyC+u?Gte`xD0bQZ&NN|fhG{RvGgig7dkO5%-n*|Rd%zIm%TCcp?cH!OPQtkD-SkuC`|ibv|9k8A z^nFccc7^5y(c3?|8y0uV3YLLfXy6M&!s@!B+tbQ_QeiikN@hq=trffd=|c->YC#Cp zPtSfhVhA(w3pAyrTIrb0=K~+4i-eV?{LVa_4Bt#qy(VB|) zutYOFm71H^@9hK=fs~a51@Hn-L{54EEKY6yS!hi4-XZowf^<0pxTqh$0W|YYaj6K5 zh0NV7-ztZg_`4^U4EZAc5*nwHZ<2n@kiiE5qlBZNQsC^qeWoxt;Tk# zmy@^MuVj~ch<;UKCj}m7Z&gL~{V0*OKbU8q(-~YbuSnE-I$%2z4RE6P4-eJLKE8Gkq*-DV`Y@-hP zk$mt#wDSdjXXwNf-&Y>Cm3lN$Kz>a4*OX1-?#_iZ{!~PxKV1j$Key~xrSx}FwzME} z7WM`HKD*)6mxL~11t+OLB{wKLv}BUhIlKDq3cEvFh*AVcy+m&WpB>kUDzUpdky=nc zXDW}Md3L&I-D$AqbL@4>`zA5F4Lo{c=8ik+@Y%I03W#vXS`K{dVI6#lg@+)AD5D{4 zjSuo_C#_=K_F)hJ9;#vri|pYI+qYRB;0~$yzh4hJwu=yx`iR1&D8>_rrESL!APBw7 z4V8hk3R_&QX?hr7Ih06VX-Z!mJ2XXatz9ksUstD1)FmQYm;Q0wS0}2I^y3 zEJMAry#dp72;{0DbB!oc0e`KPMU$g1l=RUDB!+ricMY+_G>3yb)vXf)=CVAD|0oPY zo3^gK=H`1ARSIckED59fn5c01+6~Pxe+|TZs)pYt*q`*m9%>y)_DJg+o;`Rd640uV zNW$G=DB7i)NErc0Y7`x+P>-YLto+ft)JgIe#^vSMt;b9g+$-Y2J+DJ;uprMk*p3_` zQWsXt>qE5HtAS(B??oc%B8t>_?>0MmB`A$=7dZ}?`tf{c+T8C1#)|AlL*q)e-U?wE zxZV~@2{2$1Pjxf2IJIb_JC-Za!4?{8gDUKKhrAREN6Uy7#9kkhcHm*iVhajN7GpV$(BkPxvdYFZgGXTz+73 zFQwrd=!>3qP<7<}25mL$yf~&(bD6+rwK+@7Ld0I!{&uQ3-r?~`NqLYIJ~3wouTQF! zoQgN|>8G+VW2!R@8~?60b|-)Y!SPp<^$qx!$fZ$q%WN_y+7iii*`n$)lcHLsmq1Z+&4hYe40E<%I_FQl0d;6cS2a)FG-`h8j%^7) zu9Za|%w*3b_x}sJ9xY<$Zk#hBuTEu_hsQD_<+ zS2sBzRw{3R4UtUp5&DMQT;7gGltR!gcc72l;#+m3qRv5Oo33-L?C2axBj!B%5S2yw z*&K^>AlknssEm|p<9pB1YDpwU(BN}$Sqr_qF}QL}iuSdeE7yGCugcHF(H}cRFqjL(8EqT~GKA7@ zw@}UQH8XAa8$8(b8#OtO;Hp~wTh5nbcvE)CVf75tmLc7H!=UTbS_XU{z;E*2 zs|>^{lVp3f1X75`3D2jIcF`^UOBRmS7MDW2DBmyrV}fdO|3E7i=8z*EA@JpEP>mf!seyWJuxFRN{mGzweVR)@Qf;6 z9-kCNRp>P|J_up25hS7e;B^;^L02n~FGqlu!InDrSu&0nk+_?%tK?FP9ti(eH|M*k zc0A>obx3@gxK+*v|2XAb^9}6M(JM1xWfhc8)s$c>OhpA{M_t8!Z6rT|Z`8n2C%iZk zrJ5^$x~1fpa-(>d)%Ao%Frcy9bxP>kERk~6LccfSlCV;Lm5tfJ6AOirP7_+{(Co$? zL5=4=v>}zG;81N7Q=9!7(&!|humxmw=C}qP3h@utzY<#o%<$6TZYUBn`VvYBe>U~; ztXm|{xEN3r6vZ;v|0PZv_Ju&2fn4hL8Fk9sBu?P@7Ww=R4K%viyH485Y}wREn35TWFwYx07ZFC$nGtSYN0O*vU!}xJ7DO|Ow)q1 zcy;KpkGU^rL2gc4Q;C#$@tXJYlW+v;8oba4=kDw~BJ;uWk&cNI;a>HYfNM?#F@T{5 z)HxGQfqVoxvwev0swyo0wKB6NPw{VZN!d)zaxDuJf`l$3hQ#0fqKgFCwTZso`e`6D z9iOBz!DNCDD+* zN=Tav^e80v11seP&dfT|*Rltx?lxt`M{7##Uzku<@q}p2vGO1JPFPQNH1NfgX#xGK zX5YpucE@$Sc|?uvGt-xLEPk7I;l$nG&7s=OYB~um6g0Q{Gc3_eaA~@WzWZ0O^^9$C zT3z8%wC389a= zqJ6w`W0});BS+2ZO|m3%U)i(QPR4fn7Giu$RMnyB)?$?q23by!knJf=Nwbh+q$e01 zpk(Qy&B>aF38zKVJ6!vr%364i`h|RxS&F`gYx8mbu8hWq`i*XMyy}>;x1^t`=3N)JtCh{VWy!jn_pM{kvN(toXh% zwxS?%J2ILm>Mweg7LM9AQ|GgkJh8oGI#Az)wV3I5WPK8iW$7|DAMm>JOKH_Wct-TO ztD47#eP1*)!Ql47A9MQon&>pSE0#7!E}(zU8O(Ag(-d)sT?>`(DAQ;+zI2NOfmvJ@ z0pthrFsxfO)e45rN|UCEX`omFmU`@&fK)7f0l}=$xC}Yj+apS=WvnBHbo?PT+23on7w;&_eBD9MO=uPg zcB4Q7&sw+@aFXoGMJ^xNxIsO?N^;QIRZEw{cGX2RwDU0#ht%STF*=e^z7Rx{tFCM^ zxxIExRCm4|6c&J6_7Pj>o{wg&3B~hYwOXefuqAYc5C{e1Px{oqU zRZ}n{CF!FWi`RS9xdkt0Uj7PFd!mG)oIyH}brv4ao@#|87RmX;5 zx*B(t+1#pLNj08GMHOz30ZF;w($8%&lP94om1KQ<$`Rz$VfH;uAvJ1D1q)hH;ucpZ z)i-&K8;>A6{ewp(sGE!UeFzjR4??gs!dih*MW|3q=6ci{zutecUxl_(~ zoR&SgPyg*tM-eenWUpm;v$=tXlA^++v@$QFgulMW`eoWM{QjuGZ!1r! zH>0Cx2f8{}*w&nB&;nk?j(_LBHGABHFCE++h;Bw1J56?XO0SGJaCkn31@$9xxfjRDt# zqrt#y*1-gH@uS?&Vmw`-;a~6MLjUNybUr-OorSI2;9q%e`J_U}d88c{Cr+zBlcmWo zYiEvCH|Ne~)It*k%5tPtm!3_pOZ(kI{cux7e(vXmM+-%$(!mM&_vQ}w*0@b$h{e!`(Up6R9(+M3Ay-v&3ygC`uo-L23k`YQ_Ima_n^L790T_JVH9fg zJS;Ahz&1i?Km;&+vH^m^BZw}Io*EYQyc_Q_XWqhdXEuf3bKV1u&&bYS%MOyCDTR&m z0j@n_fa5N1aiMqTeaMhCIVI!Hxi|Zb`+on>9XXbV$lT91+I$6fvTr50j^xlNz)@fZ z#N{%zU3^2_8PExReni?-YtFY>-Jr}Qb=+NiQ{K#M7e3BuuNUM2E59zbv~K1(`?Xy( zKIv?3-rzaTj>fg@w1_{7xJS6V89Z@M?$_Nt=tr5$@$G%nb+m~O8Zf!3r|SOgUC5&u6i0Xu@EZ^sS1mZG zuf{TAaF*_2*sN7Qr8%vPs{5;mBM$BA$>jfU1M`sK&Z}S&p*2GYTRo#LB3v_)ZwqHG zfLu^7+rLYpxOw3g5!MhF7+7+c^J337goD}~vi=Q*E`Rz-`|r(!Mk6eV#-;C4+G6hl zMvv^h#+<5Y1(SS95;Ie9?{kHeS|aTniJ8Je+`q8d?_l14i?C=#6l>Yf-j|<>()sE@ ziRMG~WQTjWjDe#go6 z)ndID6_~{-L){2NdW~dCrAkE@;@{U@K_WwZn=B6ae;+(5(yXbMtP6cPbZdz0{Jpq7 z;dkY(BkFYFpZq8wb+{$d6Cdm_sL%o4l<(1nY?>yKMOwj3R20-$dropufHa*Lqr*Cj zxe2=ZH<%CqzW(_k1b`DN`uyA6aaQ}5i29RZ;e@n-WulFrE5mU_4UH#mEe=^e<{m_K zvCWj^-OqfVAasZ3_*cW+9@+BcBGi#jq1?tr34cz4&nl_FSrc{B6fOvNGQ0Z ze4boPlnaP{h&X)8@Hr6VqKeWpUUDPRCUVQz`rj7i-cY=522zd`1zDjKfS9#|`DeU_LxPeq$Lt*%jIB`>s3 zzwb!I0>ku+tz*-8{>4gRxE4(5yBJH@;*LuEkya3RF81NFwpn<>Nr|U)$P73BcU$4` z+J+#Lpaw)6V7tJo0LeClhBlfBnsbyQDi7bdH|6-X3&2Szu+K*JLNCP&3^$Ovv1%kt?RTXkP%IA3aO!SV(gwgakvmeMqtW zN@$naFz9q@wupd5@H0|i>Lpq@8Q1x1m#st*LiV19&bWUut^`l4N!dM_8!^g^lq`ug zVg|9xkcQLJfFT6~40zNQfrSwDcQ?D->z8b~{T%kjigCvuV6CPxl_mH4=04?bOb?Mu1*MGo-BvYX)+56l)R~KJQZU2FE*?}k_(>9|! zg*KQzy}T{2dzK6HGGoRhn_g7!J@P?kTjp>-*FULQ$G32}y=is-iN{UC@T&&iyEhKu zBqAYXI;F`m%|7D%)@6~)G{$TQ`eAorA%H^q&u6(^kT~?YjXKy%E!U*_2t{-ni|$f0#d9&WHA!@4K{})&&Fh-XoQOP^K3L8z z)W41)2JBp_?LTt3T`}#tsAZO}bY0b@N!K@w9QlFdrmfo}-J|%xfJlx`|3BmHk!DK# z&YWk6Kx_;WJn6rsl(X&1U6CKf@vkSw$bmH8Zj{(=OKS+_d$hB*?pkx>V!OCA!JDhB z-~_ds)Ea4~wF^aefcRF~eXJTI1c;1#!6ArIYXKL_=&a$rIZJz9MFXT@h0n4t7K1?) z8u5|-Q@{Phj(YWUZ#w>K;9r2#g4UJTAfS&j9R z5>yBxb=uZx>syPPe1xJ~cCM<<(0&Xyamyo-!T|}?lm>Z&#u>Zh7bRT)j=*DDbn7Wt zL(j%r(tHi8lq?)X!KuJLeW_mKKlVbaHJ=9A2Awr*V@~Zx)P`-^WwL_%b})5RV!u8U zNGmxNfzSeFjPNm<{A)nXn$g#C2tWl6aKX=P^PK?>@N^mb;4Rim*9P@{0^l3E6bVlu zZ|SdoK}W?`I~Td^J&V;DCdvQOVVa48Z=M39j;zrcWwMUN;+!bVEv8t~4)2XEt|~=8S;d%bwA5$#5Bo z9b5563{uaAaoG6oU+XHy6hxbppJK1FY}q5@;W9_Ot2>dy@5UR37$q{blSIVbP8cHF zVkMm3bf}@SW<+*g3e08d;6wofui(mfDqn1>W`vJ&#y7(C`wX9CO>H#|ALi|EEBXwHg!(98` zk)-!^+0pm2vpcdQ75+Pv`DBY%3=*bA*?;RLUc7e$@A#Jv1hzzXf96Yv{q2Oo7|B~t zuksco=vyIeqZltbiD2+bT`HSGTc9s8gSn$@J~ARK;6GR&r+kP68BgG z$IasU*C-olwy`kMBT3h@&xXE;;%Pfp46PrrE8sGH!-r3ImxTb;QorU}wwxe<6J-+a z=e8PaZ<#a82R2ZhzZ1QnJij-pQLp1N8MVf1EoV3X+5bTMLn7=13cEbPg>1|QQz@tG z6?3;=82;oQy(d;95%g71_(LZyiRRSnd3I~BgkQ=c$I2f3lcIF#WOJqtbt`O0OG%D> z_2UL6`xDTB>MVvtZw?|}`+`@)t)cZ~aGJY7PKzQ%KQyvFdxd2Eq#Kgee-A%Gph^)U ztZc?2cSw?`h`i{=ZYbO^G&E3Jf6C1m&)%-}pKf&j8iyiO#wqvJ(m1dzXC9uD=tpmv zJH!1;Nfs{ZdWHbdq|V~(;hJ;ohS=mCwAsJyxHj4jG)a1Ni{o#L7&===(&xi$)e-*!J_@O&u4bSkotdnuf4`d=hhm#Dj#ZEXCz5lNEJ;&DZ}Eh6 zh#?YWutg-$ruJ!;$lSjYKlPSgF*%><5%A)pRy|RW?=D}g&@M26hIMm{!em4x(Q#q^M!M5zOd;h327Bsj zhe)f`iK*SWo(8Z`Sc=!pIOeQK+U#R~h9A->?TN+n_u@Ar7(?3ngqm>gX_|x(vc{Rv zKnWk>7VDy>tfmxvtXAGz6g{Z{LX!!?`YWpHMTY}Z7&s!I>TeEj9evFd#4be&738eV z^$|O&ROlEK=&Nu~3Vo0j$ak`KZx{;KSF)7m#I<(LJI^qq&(u*ebCrr*jI^F6tZ^vx z{p+6elKRP{_h{9j(s;M6usb|lp4ogG(- z+^unW+=DA7Xl|P(3DBf3$p(p|s;;Vx%VwVewPYgBcHt zyfaSwU@x9^c5pBv{Z~cCE{aK86~FnDwMmoW1r!-_CKJsr+O*<~W9wMxz;r8_L9z5{ zuEZ`({UOpD@J_sqnMju$h?x7_UeV@^QWGEUK7wN>o~th{Xa%qEInSF)>p&DVW| zwB(rDtJ9}UK=HHgybrL9_&in#9)t0cfP`&IK0Kz`$P7%E`NFBR_a`V#wM%t7{z6;A zTP-6T7N%;(l+3v~+v?@MAmbAr5Ho=y=E!ZXIwBbfj}p^fFDf>3^s|RMXt02o)374R znCDLt?;4LZ9%hW5kd}_D=yt5e1cPgdYSb9#kJ9EnuHJw2l^?7+%2SUXZ>pc@m_)*l zytvCu53bs(TSL{qtX3qvD~wF9l1u7UFxI^L5u)Gev|qsq!{h_gLAP*cpIC06#>T+! z8Iom*P10n9r@#M>RqJ3e>3rf%mqI)K}$yBiDlS z@$b&VP9+{2XG|D~>zP{}K#6kTln7}?1+QyDBn43UXBNQEmdeE@u=X~q#b$pZe{K}W z91t0e9+FH}7GERsb!+X=llaw%7kEAJ_IO!&?Y-d%_O(NlieB)U-E};%YTfA4 za+WnmuR}Xz`U^K7Cn;A7l#|Ed^%1OD>P|d3=sjJJsEk89`heV^jEiMg0@5YWVUin(v;7H@8 zBc0m9eB_*QLvip_nNO!FmmiNxC@gwl1MG}@)Q)K7MPT<`jVgvM@uTd~yR-4lZDf*e zzw!PNgCdYF3N~}Ne>c{iprXAfr%z68muLSG+e&fLhj1f<-twt_ZS$$hqSQfnCB2BVSGJ2!$97ak$Xu zv(Dj{A_e`I9iKHz!j&CQRT0k1)f5guzRMeKwX!e8mql}#5DL8=Te3NaaebGTqEBi< z`qtXD^f4-G-r@I>P?50Oo8{~Lue3tG%-9g%4qaLruQ=2mU0Qj$ny)QlT-i++Sv83l z!w;Y$`5}%UCl8o0o|Y@R`p;y{IJ-;e`QKsb`9U&h9H~LR(MyFS`NPOPF3#AKdomNq zbu%yoiqgVkWUQ$+GHPVl(Id_EwsW+;sJ-c2?IC|ReOfdX`u&&k6v+lW)0Qr_b~2=IKBCH#NDAMmOGGp6pG; z6u*~Hl0iA38on0yLno7`@{2d~TBpgj?Au?2a(0s691pXT6669Oj46ElOT=zV!6sJ- zS7ukSURPsA#-4Ns-~8Bjc=`EmGf6-DEJA-iH3C;7^2utjb_LHdurS|6@53O2>f2C< zleiBD_@x^sy3ZxKG^=#6w9QF+{}=GTJ>k%IA#q_70W~ZgB%tFin>4ExOukE?cd0rhA8XC-qKx1ck~&^7-(fNpEgPMn}^$CS3+xJ^wj|=J7#6dGdaNi(;!gixZ-{P4Z2UCfck&+v~}vQ4i?$08T*f z*2}Pt3bvnA<22C^Eb_B>K5?~sHUz_McTWP!RNdbbU+6ws+NVpUDJuac z9|HQ44z2%jZ9Hqx%LO#$cZ#Q@K2Rwb%TUnew6CJn53`D_EkMzL#IPgnmFL;Y^}KIU z(S7f@#%&y{)A-D{>n!h3>1)>3sh#3m*VF0#S&Wy5piu(nqt^#^D1FSB%jqRH|Kp;o zbMdT-bEo@+J>El=?XB~AaR7p*OQ?R#NuKW^(r|4x+DHxx7bc1gM-m<(oEuQP%bz|L z-pK#@8pjXJm`cvgx{P@GOqI93!%x6tDEnNB0GV9pbv46QPz5H%DLL4IZzx5ysuomY8B%PuzNYVSFXnMGXTG>gnTe=ZnnTOja7aCSA_Ty-?yk0qf!c{ z#+CP4jE1?aH0_+U4zLzUo&}K9!=6Yk>G21LCI6w4FTj4hu5FERe~R)yI9fy{!o~1Y zs81vUB*At6Y&Ns?x#^ZI)D2h(?heBy2y4Q1B;v13buHAH!idg^X3o}wZth_3Y6Oc} zrYoyJ&*TT9!X*YRV8oTbqybw~CG(7Mzi%cW`y^|#ieqs_A67GK?!mih9MR-YkZ|0~ z)hbxN!|a+!4WKwDhMvC7!M=~{1lzwF{uhXL`~^z$&3EcJW2rGPZyB+vr_)hhsW}qOUoa92-mEYYw#BerbIjEUx9@s63lq^=cxW5g=1@&kT@ES;9md~ zPaH-WRcoFB8P^hn{E#0Cx7@`7JF4+9U020R$XWTj8iqnPi4Gv71#0aC36k zw+zU-TxNot{y&U-65>^tw*wwRsvxjv@%akErcfbIM`dWo6!q!O0(C<^lT>ky# zD#m_h;ZELXKmP)Xi9$x`@hs9EJK8|8tz2$Z;B#wHnwg?R!OIumc(W)%VX6eCl3M<3 zM}XSTwbJ{b&|D3H+8+OK=6C@5p6$q-a0Nj_)ZIk6g%Ag zn_oTPv1Ip7Fo$JJa7v@euZc7|5hE5Yp?a4=!ivO6c?(VFLEYM| zyg@ZTfptdHrA=|Mf+)2@TMul>Z(zG##}hB2lng#T|9T;+B<+YDD8e z=n0(Fe1HT4x~-vgxy7--fkc=0K|@s7P%n%_k%qsY|3V>t{c2 z)iytz`Ilj9{J2GoV*@>#iSR>G{P!9nrdI{;~ zg$)o~l32rCF|wBg%4oo1K$u8dNeN4XY7fT%F@i5WS?tO7bzfKHz>>0;%xTRJuuHTw zfxm39<~S1j1XhYNRh~`gp%HdgsJl+EH$cXv#KD9^N2%cX%h9GS2{2k4MG?`G<&-iJ z8B8?t=X4?e0wiPWN*Swb4|{Q$-2ln2u{)va_zR&x35INcPK~O_k~EEo=!xWe!2QY>fOL!fZ`D&|pVghh-i`lY&|5lOH4Jw{ zPQ-?ryIm@{EuL2#BAYjKC_N4Ej}NZOr>5L0I|S)Fc}Wx!`Ym@ygP(2ROeXlS7nfkf z7z<5-J>f*#d)sPjc5I3-o|brFeK_>p$Wh{{j53nnbA+46KBur5a0maLzG@!C3UujW z4AF@b)E^)J99K}wf=v4C>9=d-TV*skt$kft1qr*yr?A%)ZeK|qO6JK>F8zgy=76bC+F3}N z@n*-3f!zmycO%Hlq(2s(J;!g4`iagg#qY|&JN8f}3&@MYUD^dN!IVhupdc(g8&4iV z9~t4TPW|(s<2rol7N4QXOtl2k%aEGdSfARf0wgS#Q<3plqgY;<@`4DtZ9H1|-G)GD($^PSme;@83%jD`)6Dm|<UvG6zmUPMh_Tjbp6xotBMBJX zaXVl}a5xr=FUd*4&6{?J!C|FTg(&z*o3m(@CPQ#^g4wAdN_*d~%xX2ejj;QnN=TXn z9JqloN;0;B85SV}*G*HS4rNZz_^!ZE&%nMClnQ6;Aqh0#n=s1CJc!T|GG5L@GgQ)YU#EV&>y^3*~jPY z%E7={pE;&_mwM`>i_1y)+_h^Y16%LrBPmNtT!Q6 zt_+SOy%8k%^bS%z!*q1+8kwGBw7(Hf*b-Yo3h3_ehc+tt`wX56CqV{p__+b9sk+Su z?o-N-Nme!A*_#KbwSmNeK-sXh=U*6A=0`LWe)XKu4l?IDf-`R@TfE1!*iWAkvcRKo zJu63CwtlQ3;(8^l@E!*bYslO%hj=1Q#B8QnL=q`*&ztE~r9D^LHj|yHP{#gk+P0$6 zjsfFxy^Qi;)@J2eXq|gip(B4d*65)%tj%b2H-Cg9m*SVjr(FQOG*9dueZ6Xx87O+K zR%Pc-`+_zNz&98qlSa|v97+|dkSJ&g?~=$(*M*5|{Z?q9cBDnatdX{|=3TqihYy|c zA_gj@&nW+K9A}pj6`_(3u(8BAMG`ukH21Khz8tV5x#xw)oB4`(WSU0zn16;m0;AI3 zs0S+Q>;{>%QbmNvl$SONbbF?4B$FW&eG>#OjsYRc_QEJz=Zf`OokeiwA*kCuboi8Af6L*-T_#CWd=c% zTB3SoWy4P8JE7)^MdIyh>e^Yeu9#?ftpDrGzhzj_>0Gsi`KIniQ z6NV~txSvU~;70b+;CICA_Bbk4@WNxL;vBS)>6W%WPtl{1EDIpo@LKmS^onkvYJdl% zQKanW)Y~4&-jiO2S%@uGkDn$7ZXl$7*hk3grRf^TTXgTD#A?G5f{>*2l##@pJEH{3 zA!K5k0rc@lvD}?~*hcKPT_HXMQ=T|``6;?at$6!!sq2b0R+n2V%6;VSsI3}k*V_JW zdiY;OZVvS2AFF3H-B$Bka)bNMHrZw*`#;!=kor8B9yxKOVx0&reFdomT*E;ON&3;h z*iHzV;Mlw0HF5Kvl~EzH*#y)<>+f;FGh&Ag5%~ccQ+j=CGZyyIpu7#kM=;&|FYTb3 z#KZEBH!Bf6q~cu1>=&>8nD5{m7;MU@CEyhTn7hf_{zTZx(7iz)ZX$HM1=BtOXri_& zV@Dk`Vgl)q#kv(r)B-rOmaMHB0z`iMz7@xjCO=@wJ3inqE&bcT$=xg$kiL0C*EbpM zW=NFW;!<^gZ=D%f*;2B+s|PqGIDwI_M=fMJ(B8a)!;~a$tMoZ;XMIIpTF{WLFuh*4 zI(h-hZJVesRy5flUv@oc;{et^azD$;m9`kM8r|(1B>7uD92g zy!t}=wuS|Cg&y_2%k)N3*U@SvJxP6we>1v67T_E)4d;ZqWr9DxdAc{?q-Di=Cvr!0 zXy;ErTzS+vDMkRVFVzItA!!uBolZ2kf;yoGhek;9j2*qH=iWn|!&xu8-$b!k>WB&! zp8h@nV~Blj87`18K-K}a!soc9_%$v%w)7>lHc)%>3SQ9nbXxfUFLd`U;q2=@Dw{z= z3n+Ijo}C1>cLQ7AboANj5bX6LzSa3)Zmm4|6$;pQp`8+}gy{jD4WvNrW^e;>VhL?g zs8ar@aBL<6wu)UYU=c9gd?8qy9cYrbn!dWIn=~`h@+OrT+{~J#+z%V)W>s}Hd_5hFo->xNza^M0zmeFg6gSme4G;UHSX7%5 zzoWXxwEyTdCDLab3z{A5K2sm_UK?J^+$URSoZG>zqaVXxn?9pH+ufJB4mb9=ZtGX} zKlVLNd@kFLxlZTPKGTe&j$r@M5-;N}SFXw~p`X8=5xGs^H-6OhG;TC!w9=}(TqftM zeJ|u6v>tdLKpz^~O>$XiomM|bU&metUhD2E@1hmiuh|+#0vXLeYy8UmihB$BD{oHS zE--yG<4*a_^UUik<*m|w*LBfo3qY@MYK1CQ3mS zJ83=Io@QqiB{?PBsY=obC3%VX4)TVih_au|FH@Ud zPHM-w=~XjZ>~8LKoa{U=^1qiuez&q8H(GVqxu4-qvyJ*=^qjQ@Z>pE&LaR}dYm6A+) zYxGLiJ;{5JM9U#Z!c^IOIK=bgVIi9`^;+$5P*Ry&9&mNMLo+1voOIPSSVOwlv0fsS zohF?Zlg$UksFyEaJCDMvUUX*B_ z!DRI}m5R!4j7st(0^%in2tu=4!77nAs4OU!;EfY|c*&Pl`cc4QT;wKn5JK~iJ%li2 zBX_fH8^*p5QwJWw3iwYpAaR;MVQIiR#$> zsS4q_6P_aBk!v+A3G09Xh)JlH#09v2tg_I3>{iXg0T+*4LC|!pEYF6}?$XzR>aAl! z)DNfE)s2V3wtuqVwe|fi{Kcbp=MMfCPY^~EL zl~2+eDd!gbSm`g`>@z{;WMfhIn`GlzmnAdlD)?drwrxT@#mg2N*$oWZ^O8XxEiVR!yHarv{gnP=*jC&KElKi4R(I?WyDDq{6zss}i z{Z>o=9x{S`4s}ZQn2b@V>q`VQ6b=uP0q{=r?-8QRpJ)0CMGF1Pq%dTmK3Cw`)|q-OGB}55@UA-a zDlUv9FHBC%?48lsk4>8jwGGG4sO{(1vXcgqf<@IOcJKrB4Nw=i0rb5-Lz#apiumLg z(tR}bS*1ghRwph#h(cCN`ok$?0Mx;+ok^emW>xBIQ_pZEP>>4~Xfz=H>sNpxWU>=0 z@MK6J?W#Oa!{(5vz14I1XcFkNmmiZ5$bvO`8=J*Y3L~iHePq&JTrx|@Z)c)H=E+Zm z<_?w+kPrR4HHT34`rekvoCb9?+303ZtaR0`JUON?gNSw7EF@DXArQoxF`9J0feL{e zu(1J*g~rm*%0K+Yfb+pwBVMcs`;Ac3%5gp^3opig9aqmo-sb@&bx2$GIo3>`zVMOU z(s)*z%`waOQ1%%Aj?Iyf&`QJ2CRn4i61F@T|k z`mrt!ku*KKWu2tZ8u?dxpDXv7s7h3hSBX&cZ7xWfh} zgd!c);H#H%(b7|hEZhC>4Mb)LN{EqJt3r($!glo^GSQv^rweV+ZDg`_A^NGQ@rM&! zWWo;()Ux<`AP5gB=PVRaq90IaGRxoxoI7Zh-9B?%v=5g2K}66vTW%5sl71qQEfhc4 zeNc(6qi&$3ANV51lr!kj6x|{fB6Tv4x(HfRt-sh94v8%pcRBG!)t|Frh|JDbSi9YKK1<5?T})xs$(MH7?b$8SRt>%DM(%}*?Z-=GOzr!A1` z!Mz%!&j8_+2)UyQ#M>o>P(2%wLun2}To8uPOfhVD0ovQC;sA}My>oz5cg_C5`dUqY zD328XPUl&3jirBpHHGnPG@+}ReG~Oa?N5}5I;|vwr zW%FLJgp`gg$Tg}fXAL4d;?cWDxo_zVyiYKx1~~Ji-|i;Gp!=ZTl+LSXX$_loSh??4 z7YVy(!x`ZoL-56OjN=7!1*|M3?RSkWRpn1+k4VYdU$%%_a!&*hOyEVheVQ)?XhPJu z9VWougUra>pBjOaa7FPENI2(>ki3WU40=;ZSo=CM|31lx?(NqVg1>L=3S}~g*Be7_ z{$D>Fv-PARO7JYUkV%X86)gg^6LoH2D82}2-*^cl#9b?*SZnwLkb_DG?+h;Y3I}@N zP(dJ_?_(}TK8D>UUz)!-^3k={h8fctlbe}ZHxDt?9^xn@<}*D?j62Q+|t^ z)t6NbZqwGamFlWxt!3tA(hI(ec+^JCn!Kr*N%pDHsYH*mij1ve{E^lXuMyM(#Di#u zvIJJK7{AB7$C=00C-leB$Bu7*-%i`JB2SUtQlFA9H7`*wDK7=?lC4_I_2A{wua?iH zd+-T*T0LF9W^Y5z=FQ$uv-7OQkBT&`KpR#WtaO$Ve=|KYsahG<4(v;;f=VT zx;y3j-hdT!oY=!$(p(vO(`kE1|EUkOpNuc_TmNBIQk-mV1`nGB%_AKNxnJIoVUOE= zYXm*ncBY@Bui5vthh2}`ck9pwavo`JmOJrpRB2#HZa){&YJWO7!Cdx}<7I5~bcou? ze1G(A^n`9QC$rP&I_mCn6Z~MToqZ$^+t1nCnU3kj>~qxJ=kxon>dx(8bVM(LAK8!N zI{IeF=-_2kex!5c2lKM+%~T=|BZw1@wqJt zf*;N0dY$c`@9%`i<7%_FCuj4%{azhgm7+o0aJp@s+g^1m%8MynUGql}r$||l#z$6A#zsfb*+x{b>5;Hv#X0#omVl9rU$_|=YJbgXoZbeU zMrR(%vZR0DuKwO?;zA99xrVxzg?Bedlr@jHhCuASIMCeUTMnwngAd*gU!}!6jOy3s zcKykNv7bY&Dt{1ll1-@SxMW!nJ$-WCES2Ha^;Ccjgba8#jK7wp_>b7V!(IzWj$G|h z-<+UZf8uP&*AWKN?InAJhE$%QUo}%18eLy?1mMP8fE0{Hkwl}-Rg%-vW$ZjgxzD{U zr&YWXnR!e{Y+w3-R|(D#V_~{?J+5LH$=1AXBY&su2|uzD#JYwr?u2CcbRyAUu9 zq{WzlVZe&WhX=eBB*@Ft=R98&$m7od?D~9m#_kJ%faukIgqII`Go9rErjZiKh zNl&%lNC^LJYw@l2Z5uv9o0)y5{DU6mx#0Xpjt*bWUt@5jt0rJ7qjBK-bVaFlY|8J8 z=z(o7v|`MD%A|KZ*H2M>siSJpDf;k^*qm5G(WHiDR1r5X6cOe&c!q^|eGl+_5=)5| zvKmp0xvK-H!gfeh#^NVj7Q7fOaNt4%8!)o!N~hu+I8cX}B8g>BYizE5)%&&OUO@gxqilD#uiln{gJxWEOZgTQmFWF(Qh zlEHm+qALvpeCyF!7i9%35%oTIm&gOe@R>q#yEy+b(i}s)(Fe&*)I^ui_dCKa#JRhP z4CoCy?-pn&3iC?F=bnww0$M<9q)7qeMoNk7HA_pHk(4HWV8ndnJ_S3A8y-PTHXy5` zKj3KB4v3(r`pa>q-5)o}OhGHU72hn7>fK`CCc3EgTtah6^k#=)_L5`8!BYVEzOWS^ z;Nb`-vKJJ$e}o>x-5d#OEWo1qo?6A1&2JOU67ALbqAZ`K(=t5jD5^tdph;>&NmIpU zKs`PY?MNzY?nL?rVu z-q~2Gbv^UCv%CoiBdiWcf*r}#yc1WHATFxqR@iJQ1nz$ZL<>(WjCsof-kEXNI>i_3 z)j0l}h+rnTWtTiu4&nU-X)f~C%NU&`*ozqAEXa2Ff%)RXl0RQ2V!#6z_7Nmi#3B?J zS5&w4RG$Z#AkM+s>pxJUxYm%DwtS@wlmVED{iy<-ra=%8%v0jWSmdB5ls#wr~_&vPsM%6(L?@xN{5hR98!0J zO^Q$%`8~nP{Ws<_p{+gvO%!Sca{~30KjZdRoVdeS%%V3<7umdNlJra51rjAj8MfI! z32*ZKypy_n3Baa(aoIFkV2mtg&UTe5P?l5HngQ-|s3}TOh!oj(35W1#b%}> z0m9Tnlcv1fE`6lR<(OUb8g>mbKJK2WPsCu4YYCJsqTGhcL?=TG4-qZV?Po#cS$$@P z$|VUT>%0G02y#V0(oNn;A%j2x9i+`RU&!n;K2^2^PwI0!NBPKY;Pei-IC+!?^4f_` zR{WPh;mYv%`3XD&UCLuGhA*=3BfvfLSGwcLngA0h`+G{7x6i`a=iI2~II~SY?eP%P z0!2OcrygDY&33EoRT<=czGT`&CtHdPsvZ@F>k{GhtxdtrwWE*q0WLum46%bMrdfWl z8)lMT0(5V~II-Ujp7ot9OKCo%T!c6PW1uAQfC<4eG0j#WgqXRO9g3KGSU9@8NOM(3 z|B7P^u`od{j%ewCKYt~hN17FjU42epz{&v1^ht2>mi@AQjo}8>F))qu;fp-~bpQ~J?DgtSOuJu2tOikS zmK6m=hjh`5fFvKYIb4%Si#))Hg+wG0>Qr0ZABS_boZivnmC8@Hqo>oOMPch-#N8>6 zMY08)NCSWgxw!@I3H=JSuYYs!YE*hl&I9|EHa`}a6d?YZ-GR_VS|m^P2H{_Li{Jg; zJ_`-4x;f5S`=d~wI3FqpyfYYKpr)3DUo9`cb*VCxL2LDk)2UhbG}eYo&2Yk zNb$SWq^zZExCCfT4Mue;A1TMY|BgAVd#SqZkmT^Ee$e}z%?JgOs)rb*HUO$$rJ!WU z76p>rx=)kXwEAv2*zxwatx2{31u+g2M>c3{4u8wj8t>Ne<0#`ckZTn;3Br*n@JDT^z$E&7PN^_Ttfj3h{;Mx|7o! zyI>#y#5~WDZNj`))3Sz`QbtFM-JBqoP3K1$i(b?yc z9b!_H{VCa~9g&Jl*io7U35GY#4pGX8@gf8+{~fvroD0LYe}r4|57$57dGpkjzZ;{# z8X%6#LWah=Qg*}#RE~cHCh%bmLbmQ0RBs?-Gy~dKCv1$7gu;(8k`oRzTlB!n?d-z_ z%wl$0Ra_q8bOgJ#;GdYs76OH`cPFs$M%PZx1?Kxk^d$@4EU{$1w+ISh$`mDm--Qv1 z;$$QepYzpTKCwDmzOUZPE^caj8@?ys2QNyVI#=Lp=&N7qZZv*qf3=auORqJx#u-!m zG1W$RV|nA6OXn8ukDK+n|EL=c&(rG@R`PW^y4;PfGwXA#Ha3R8+R4r2Oa?vE&s0~U z2ib$YDY=v1v9T#>(mxneO{U&D-_G~D^Q`~y{w#@d2ry@!S->g`Hx4$=)Pi&M)V8$ByhCavzqO67qT6v)*p} zZu&0xF7__;uF>v}*1fGglkQ^nbJ4Dyt(&c?Yrn0P&67>>t&YyU?LE`2V(xRXP5vDG z9Qthd?3L`3Z1Ei26W^2K#{Hpe-4px|^be~KqYs-8(htgy#GF>UR$gw~mkqDWooc)4 zxN7$5lWOsrj%I@`1CyF!R!h<9ohiGixGDB2^xxWm4v##K#l=a*#^KMg*ZJ33_bvD3 zHtTH0YeqP1Nh1WkdYy&dCH{H6mAobVx&KVngg$~dD?f9-^R<>Y=Jb|vmL^xWv{l2+ zbXPM+H>v!(zD93rM`5Y*v^zbQH1ag9bb8W0DZh;ZQs-lf?TcR(MHMQR3{N-n^3rN) z+Zvt@XLBX@^IOtuC9g^=^Lb-8zlzO{1Yedng$I>~*Sw@0iB{4ITCb+J-(AWuIr3tF>&<{Xv>N)hhP%VwT*&>v7W7)ktI*0o-owt6AJ5OoXX8WK$;{*qv!6R3 zCqGBrv7e-O#w){}=6=SIa74|}Rs?Z`iXq2e^4|!IniKw1H{;9Z=Actad*=z<6zwGJ zOhLQezTOCbbT6JCn=fY%B0r|v{+zqoXWbgUDSwua+4K0NaAV1ums#9oscD$m%j7qf z@c+Iv;Qycb`15|Kb#len+)+zBn60QdCECKc2Ef`KfN)MQh*ZtB0pL=RQFLf#XDPuP zA9<>eyg^Tt`2z=!p&NvG+wdw%EL$XpD-H#vSjIvh%q78Y2VH>HFgb?+RZwcE1RKLb z=PWaR(*GiB1Fxh&8$*-zdJfWa6B?!klNVs=67~D73(yCG;C0rf8VljldXAA0wfR+C z#R{gc9CHeYJxjddD3uoyBC#YMKxWEa1zb!+GZUm;?6uZQmJLM0?*5i2U3JtFPH+dH zptOl=!4;%?!H8e8Vkh_;hl^A6@tCNC9aVfO=|Zm%qP)N;(9-&hksKIlP()jvT_ccoXAV7v??K<$+%IqC@x#iOClnHK9|%@TK95upzA8n*s7H6AN^6x)m6iC>1;35q0W zRHQK!3X<1JmYkDJXCaFlEE2>rGg%UEWWyIAv=9=<;S2QnrK(Ve|8os`w1VXrW|1QT zCH=xl*r>$BB2xZw2gXsq7QtA#Q(3Yz_t$F67N(khR~mj|Au@C;d7BxlRG&s1L^;ap7^O5dp?|2{K^oGf02O+MQjpGSD7;Vz~t4s4eSU2N+#goyXcm zo93_D$0OR8u=5{8(SkW}$6>810)+c&Sp)$0 z%7HmBHo&y;4@y6?^&c$Ggg@wrc(bVlghMuEh!*DO_WHBg4-ITl$1B>vIO7`T{zl(TF<=~%FVkQg$Kbph0POZ&VOaZ_ z%t#{!y4YdZy9nQvC4g{%f=N`ihp=ogRu4rT&dUx zu?qlU9+s?q0d@zl_UUot*!!7?kOvI0)4w+`E{*lSyHJ4(anN@%odCu`H`>_@2qwGFwE# zYS4Wo-yW(IXecCIG52HlEUiXSq&kIP~+3s)zCs@lXB!kXs2XEMi}D4S{ad8YoK{3 z;ct1@iz0~_RA?pw?v5Nh5$G^C%?1~SkV9A`7G*Lt|4tFnb$np@NJAioTxa0vu_S20 zgqDa%cp*>eF)_AXE<7hzL=44@y|yBgiO7)j=%1n7?~$LPqm-}5+2i5w3_1SWl;487PZ!Z+%OmTjO6*QlusJ*Wy_D#KmX z1!d&b5L=~1afE+^Oa4bxC5OyB(xTaTsDw~m;eCb01Y$CxkE$hLSP?LMwTBCb6v+rY z8{ML!D14)4^w+_W9O6ee8h8;Zmeg;XkFZokm&h1~8B!XxVB%8<6|xiKbz0(gqIsi(x485hBi*pl~qVx)N$lwon$ zpP>w81m3Gkh=?>AB27;n(aj_(!>!6iE#kkI;)~u`ae?r6))8#x=Z-9Y6t11c1*@A% ziR98E)4@@O7=HIs>3lvPA!)C6hVbVVPN1JlpsKpA=D1)Vjp`@+e@J3xoxjrj?%ZvO~b=1I@!jwS>q zCXM`QU!u!}tAR2crv`sp3ywTZd<-^n1GeleJY0ATj)a)UItLx4b5~xWUV)}IU(Wnp zQWQs?vQvj=T>q9-eF2Fm_nNpDQ59-I6^@vVgu6;uM#w$`8lJ&~*v~Y`L2qCf2@29& zln18kYKmTdQ5oW@APDOc?u`rhssR#bb`}zOs){y6FCuYJ^smdJD=AjHN01w%kWPA7 zK@5ZIZd*ePkMdmWwujvwMf+4mriZDB*eN3@<0~L+X(=IU82KAC3K)eG@m~qnT}6oc zY5q8V4L+@`Kn) zkG&7P54)~;F0@^zIovok$OXGT{F>_4Z`SG8)S+(^4b?Dpf@tDNj6c{$bc zI?d>q*?C*du18mkmGf%Fy8xUxs0@6#$*k!RN%EM>YOV;~hv6eZG zk2p&RJk9QGES_qIYQiNyP2vcwN$__R`1CQGuyy#jJAWNbKAHYOXfny}f|etQ2^3@9 z$vbqLPczTt<&)K@T*V>1fIXCFDOaQr&G?6%ZbaRXO9*oY2lV(;N=Ag);tEQ4t~u?O?3McIJ2VPv9tbkhduJW=e4aW;4Vs1XKVYuN*Cq z?Odz3zX7s%wHU7CgrEp$Wl);aE3w{jSau9McX)BTS*~|1xemb}k6^oBrmi82A&tHaoQvEB!AC+M>0F_m|vX$QWRE>h=)8^OH{AY{+vC)My4o${de z9p4H&`==SSDs1$TIg-N zBBno?r=P5sww&m>i=R0ofEmYi4fVgGUp%Fafig~LjF$-oX@oK^t2K0WO+C?HHQ1Ez z;zEE9=pig~MiV~2VQ!qw`Ui}_+~V!jCNbNA3+*h1K#382#|>UU!nN^ zFeJrZp;U4P*^TAh#Q>eGZPeYUaSt8p2?IkzD#dz-*nAQ%;E8lTG3-@*+$mKhP{Kw^3`T?tS0eCsV!@-xJbkJ7aZPvOx( z#r&`%CKhma$Dmp$53*|F5|^0UMFU*gOGs+p(&Ni#yGf)N1Qb!U)$<31*bc=^#GAmz zZL&88viB1=#IY5o_@8XZif*XuD4gqvhP+*or!a?ggkJU9`vgUxcXL%~9WZ1o?m z9I;YCp9VLOMd46cp0ZgBzz_kPQf$$F-JpCnCYO~&-c>+W>#$(+brb~%L48E{}r3>bmF~wCm!VDkCv%@7n<$Sd#2(vQ(o=&Fz}iw z2=;rBU&&HZfIQj95Q4JCpRElsl8jp!lxbWQrzJ67E80Ce2oM4U6$G8u+%9W?S>gWc ztBl8vLouy^07dS4=CjEeiUXanR6D&eaX^#SO z#!$UphYE_KHp?KR$`Q9uc2o8uOt{?OtOJLgPIlkdgAStp04UOHU^YUzM1gr~h}8Xz zvmFjgq}FLTT%~OoFBPSDSbVq+(9$!NdNkl+ewJ-Tg3v+COGGJsn^sD9(yG)n0Dzk? zA1$I6pt8Ru7%Nt8vqBW*g+7-JdVIzI$GTNku)ko7fyoh<$G(_F5}~EbUq~NiDwa4Q zhZgK*`Kbl<1G#Z z&q7+;ef5(J1hzg`eGfUOr9e6Tmj98rvcO%}O1X0REidrqy^LDYg{c7i0)_%8;IJNZ zxm9(%nmlr@(kuKxK!)WavHZ({#i?Vj;4S&6tDv!)ba_EL84gq_D9Q+@Y<6=9-?FTF zKhElE3JA*U{ImaWx64w#-{xbrHV{Xzrh7@E7i<>*ZIJt?L&dQLTasn&Ci?DS`C1Fk z#?&x1&(s#-@9WsE5KDNx4mWs|u2P#~+V*=$^5e~bm3PWvn!En1=a0WW{1!k}k`ZgLaa}hC{ zNIb#7W6L17yZA_R+opp~i%+e}IGqpxh!NK^EZ9pIs`Id*ZB85?%V(zK=NRER0~vvlG~&1 zN_+;2l;07cILviK9ogXSLE^N14=qhW7s9j7kzHVYdWkda1ev5?Y~iUS*%vK<)34oHhuF~W z(7_hvmAZD@{OF1_9P5zqJQ zfHVHDvXWmT6zkb4p~nxz;jmFBAlC^*#Q;Kq>tIGkKrel2+`~w+Ge)H2-#=tSeF{h=yI15z5h!6vEg6PcfIOAbdq}g9A8o>pjNQS0NEY* zke>kIV9pp2XUC9MF4gQ3I5gXCeMt;}B#;OWurqW>LI`If)H?|Z5N%7uFC4e){*k8+6>k0)0tvB%o`)iA(>#(v|znLflH-p@+UD$nMwgd163PFEYvam%&U zyd5o#t|ys|Y%R}r$D@-=lN%W)Oz2@4hRuFY zbt!ggd>S5JMHXS2x9=Y&;OY45y`4GR zH~)ImKAD{7EJjs8SAZ*tE03wzo!^w)r1I%{nw;JJZgF*_x95j`=ZL(-JQ6)AzQj$b z?S!Vpudy98em!riyNcoSecY%K!<&74Xg#{F&fgirW$zl)sK9Nn^^N{mkC^YB{6zTy z_ko{;PpT{1o#IYlU%`;&Zf5i4zx^!N%kAdK=D4==_0B8&o&0{?5PnQv#13BTe$id8 zX>YcV-E;dIxY8s>R4)~ejSSbTG96QuUx|3Qe=qVMWNUlhBoKlq;iN`s#LF?(Jd3PpSlodkm>i?tc9J?cFqcxmK zCe}>s%uHRa_%UA3S6T>HA;XtM%Q7b!aB zFI$o1^owmBcI}{xw{%0u_!)m>n9_50t>u0k3HQ7YcrB7rcgz3 zCeeKoK5XzXPvm&uW#*aHNU(kdwslCbrjHxfF2B;G%apM+)QA3etX?b(uoZaKhS}&* zh=l*|E%Bc};}r8kzfKf?z79oUVd3!quHydu>Ev*5I{+TK-8}EY^F8T4e6#SFIMbd0 zu8yT~eC=sAK%-L@4j@3n7VaPrWnITQ{v2*V!CV`eDhC9z{e!cmy2u5KLR|+0W;r2U z26@!A1Y8vSC@E>}{rTwq8w3fr1Y})MmiJ0d@ofcKsk`zvLm;gC4X97?y!ap1q~%*{ z6(ia~b^)vF7JHn2jxf;+Rug9maAn)^(G*w05w18e@1Rt1ESjH^E%He~LsJ2#e%mHq zR0YT>!|p@p;k^Z4?~vR3-=vq5*jY7IsqqCan5j|A87ZU`EgAPNAiEVeB@GvXTWJ@L zD7%o2Us}~d{_`z9W={Jp4oh))H?HdeZ?xyL}(d!DGA&c+>L=}NR{;h~VF{f%{E)mZIG z^6OoYY{S;1W69L=Q`?;k0>Q%-4!Q=ur(K_%QNK#qxe#VChqG%FEG~`3$rTnueLqxd zGLfxjAoL-H{R$hYb%x&^K9m;(MfT>$(x@l#jL+gFoVdC)Ga0di9mh^&1FBH2%(!k{ z&_LHvicRq~?XhBaz-3au%DupvND*6O0Ac_xc&I#x+pU;UZlF=21ff*D#?Kd%l$>Fc zZAWvQo2K=|O;S!T>;4+A7g4u+$xppkG=7`vch z*N@4TZnnG#z*&AEYPV%UEuGX=+$2`VS;N-KkbisxY6(T#$S8xLvBXYd(*fEj`^my6 zG+)9g50e@S-Pj|$L$L-8HNVgLfv8L}q7RQ(&3V5`_ZO43_7kxQJidQEvbEBd^BUNUDzo&as@>GnEj?DRV4!#b*!qtz&X zrkd~Brb+dA*m*P9n82ID=FH005PfcKqv9f*Ih%I@Xq(twgWi5|5+Y_|e6(toq1(1T9l%UTvt*^(bL^kymrNv zh7{Yq`P>@(n~t^QCw&Ix^E%a}tBK zwfxI+S>g+VHSzq=r5K^*x||370eFgP{wDMnsb8X`?9UMFiM8+QqabiMunJkOLou^@ z--OdH7Jot{P6kRQ#U6$98OQg627)LCDaupU}lhziU3W3U_itmdi zYloce)wQ=piG~i77@A8Pob?(Uq}(|CGjDM8Kbn*LnnrTBTDcqvz@Tc(oGHG37NNXV zf#wx3NzY;uQG4W5J^Oe(!@n0`;rmxrxt%?$X8E@yTQiFqP7gTA+^B&bV3b~&-of=D z)H0%T=^4=~@y;<+o$v{EWB1k<W?W0`^$dh{rAQ6YDJOrPa>*$+(!G_pVUu@6aB5eh3un`X_7)8BbS ze=Buey3mV(BrCSWs0t=j;ah|DgdcHLA(VpziSfAFy=R}S!>s0t5u_5lA z^!Dpn^k#Wa_NHU6`!6*|{2SG^!%l7Aejj|;APO?c0%~=id=7&h&xgWW`IFT3_>(&~ z4q@mg>MO^C+u8Y2M0IU7wBc#BZ5h|Y;Y{OK&s|5|-SBwtt5Pn`Z^+lnS;rfaaDP%H|fLv`SQkeTX=YxIXMBcfZI)vA`&KI@M6TSD+ul3T$ z{Qu?R(x=r1u*li1vtE|mC~LJSt16dSW*reQb)cOcv%9~oz_vgSxDv90}`v ztiUA-9R`!L%>eXTubBWnQGmrslxuf0gw4!m zXs*J}&W^Z9zST4r?c^;6xZgG_a*kqfT#-y!;v}hXHdbpkEN6xw}9-iQrCzB!kt$XCkqQ-ygf2dMaTNKtHtU8&+aZ)wHUa zMT=H0Z+Q?TDD_{Mdo4Z3N>Nk}o|Ud$&>)Xub4htNnb>$(mK=gmJ2*$1qhrTxrXDoO zZ8oIyT3%fXw%#=wIf+Ggn+_8~A1!ClG8?u&@S=vc80Zk|8V)mH ze{l*qc8VbH_J9w0+p1bK;IuJ1Ro+W1?>v$I%T?Pw-z=IH>F!f0(%rZvenLv{S zUCqiN7Yl8mQO2Bl(6SRMTdP4qAbd5poGnQS5_j;3Y4rCh7EK(N;UTsY8JO&esx95! zz7f$gF^^<4gJXmsUVEK1qP86OpX17vI9Dr6eJnuan*3D}V=EW-ngeXY3@oMd;LHj* z(LUvB`YRmttAGj$lB#lv)q}mk#h#Q8rz)$e@D8>K zuhUps2N5uOplBu>gDZc&(y+(rdC|eIlU-?&2!uZ}Q72&F!Gu{~J87|7X?W4HdlhD- z$gAvhnKu+k&@h&95bJ7A(Wg-aeZ&7&P-r*^BeP|aOu|wx5yr-*GiDPw2JyqFkuhp(95q)F|1@dNVW&-x$>C#nqf~v)5?4dM_GR} zctdxyVZx_QdI5bY#exr7WkvCA1-9sK9JqD?4is80#ZtrAEhL)~cKf2FZv8h0EDJ-+ z0%U|Zb~l{tP&zxybq0CM0q`>EiHAWJaBQpsThQeF_fATqi^=kTjQ7^n!=SgpPc`zH ztG~nGMRDND!cWC!rxa8(XY-iPSn}*ypVZVs|FS1Jq?YtEx9&GKcn)F$FUIa#hwsssw!C}`CdiP+5>VFbB-H}Z_-a9u%Ba2OpGv$fX|+hD|? zFk4AoXgo*x^UgcXraO&EkNHtbDfR6E-eCWDeS)wp;0?qqIXLJH0t`Q zQR#Q(_9G_a?bK9Q(v?4vu0KV8u;F3^q!sNr8vwd2(q;17y_`Sd{;eKyZ_Oa`UgHjK z4`8}XMO!6~hr>TBOs4lB(>WJDs)O0)lp|-uE?|ta%v4p`a zIe0@XIu>2rv_)v~x)8!@pGdPCq^elobR3pjn zE5u(->*QVL>3_!=Nh}1n48}nY6FlPO+lKzzWv;DMPHsb=M#w&?JBQ#%NVLb0A`z|0V3vwvhBKvJmUDMeng0DvN+)3$*KV zNVlsboCBv;i(=P_J(95~AgTN(7*`@j+CM&?f+18& z-zh6W&su$hleTc|P2rJOdD0hZ{kloNuVmQdgv4PuY0T=J0 z(;odhPVPyM-bgu^mI2@+r_v!BG6?3xZddVX6w-$ z%n=?HYONpZRq#IwysN$k)@)o6BRYQ_p7p%^zN8=>Se^%SPE;czkaPxJ;O`DfMl zOjR-e$FUD={z7_1l*UhK0^thWR(gIxC?EO|*g0ph43^;S(^Hd*LZ^+Nz_v0%k8~kf z8wX3J)~t>4t24|vWfPrK$&xv=G7e-Ke*S58-+Mf%x~b{q@X~yoqB>b|A#Y1T-V_iTsfwbtskjO!rKc zFO)BS?DnbQqj!9n1o$|m%B3kP@~bn-r5@8u6y zhO32FVq}u@Dz+7>Q}c3hD8D&8ukM3}p-C6X_%yvOI2fO2Zcs@xTf>zdt zZgt{CP%HgI-%?n0E}+vW7qAY{GqH*GPW0k>wSO2sJ3b3oKF!UHCKxxaNfCWt#uOza zv>MwO@&pw}Zm?+L;p18nygTik#vzN*MtLT>5#7!VEytNK#YC>LDB`3MEDv)ha{M#H z4Kx10x)^<6-^L8qI9R-MIi8&1>U~7)1bI_9S>E}y~~}1%oP?l zDplrXh(8B^;=R}%&1NSQw<=yDwX-`~lPBSbgYXQCGZS18(L;KK^vJkEJ_x=FeB-YI zUoIdMxr6IzWb3&bv?nwyG%RfIa5bc_$H;kv`{uPZ*_Q&LN46o?mgwrS734PvMIg7v zCyxVzz@2DH@YOhy^jz?%{!IHg?o<8LyT06UxgUJt3w?dji{vBl`Sqc-x#*Su#N|bF z&1Xn==Y{V<_h4vQYQyDhb7rd7RQV%4^u7M>ToJIDlz-YjNe9wxu{aL+GFuZ#4Z3rs zct>Hlzqjj&rh8Nh``owV@u0rXJVsxVYzj@2c_P0o-4cxjg0A zrxXx8xZVtGp42YYHjvvf+b7+8fSY#&H^FVXmtwbkjh(Dt((3Hzdbdx{*|)><%ev0B z52>Tssr;!g%TZn7ir4vdCGZ}Cp*_FO}@#Be9z*w_WpIj=YZGRtM>k7 zjj4RKT>4d0kI=d0%^fCjm)tiFxsb;hzNzjpunYrg?` zY{L?1^2OK&ssq+@_08ZG+HVp35TcTeZ(bV~XvvGFZiB?;!a4{J;m2pvk>O@@u=rj% zslEDnHwOLs1w&=J0Y3fG3LnLc36?L@p8;P$6VVsVFQoh6Gi-9Kp5&&*o>YayyrON8 z7!R2n;b>k+Pyj?!On z!m1x;<*g?TAJ*$yl&~}M^^;-2NPPYKxx#s$`PEVTd#|oS8qdsg@cyEfv|1+CO~8lL zr0P#)qt@6^l3uU$M+w)>V>?bM=~EDHG2A#K%dJ39bG zc8Ub6QW_5?#Hwb{G)_Y=%>ave3iB^zqXbbyhiy!B{4d1GFRZvtuOQpZK1n&ker`>c zf|gUJTSEb=!Y(;5!WLn6`vBDB$-(4_gytb}Iek#hLgtn;x|s)fD;1u5Lb15uywvZm zKaD?d;Hb;6xQ>gyYroJ~6!5|m6*E+njJ4hY0Hs?{<##Q0M8zTI$MaZiyn?y)?ntF* zi3X*!{-Y`bSVFCFa7qhe(2;YJkw0WnU=?ji{*hm%%k55WILFjtQsUJgoUyq+-3dqf zI|P9<+4UKfWr0tPK)>jh%b+VUGqv3cNDYA;o@h+{e_l&opzkIkHMX}Ev-Slj7swk^Q6!s?FGMJYTe`mI^Q`>X%P6d! zl8{t(8GPuOQvWkB=REb#IdOgMB|x=lgg~epB=ps3E_!HNxb$gQ$=pTy2L(NoRPB#8l3an9r5PmPh zA2_VpTyS6!%T26gYvyrHy{uBTC}is^ARd=%^Fw-_c?-gQ*`oGMk$6S7h?oj@kiaU6 zF_ZiZ7x7`rJqNEyo&@tnR7iI3RNNKy*jLuwo@?_~0C$EVI5V03H0xd* zg0S~Cw&)iRaxEjHZlh&xs_xM;yRb2~Dnw$(uX&c-Dr7O6HXDZ?)03OLLvMem#1C%I zdd}2vH^m;_RE^<>p;sWPH7m9U;cvYPLO(YQaxOxIJblOCR`Qs$-5ZVyA>3FhNg~`> zU!soC&LcmhZNx;jNr|yh<}go5KNy!G2NTkpMRR*X!*i*acP=%)3NXB)zcOZ6qea42 z5=ui4@cm8fS_q$$KGohEDn)qE_GGHTy`(QoXTN@PW!$XFrL+Elc}f3k5w=mHQfQyA ze&UUsTwENN32t=9G60$hhZLA)E6L~|`Fm4lyKmuV=ttrzUDDVux!yO$Aq0nBb*%$Y z3ya9u0&XE3J%dP1LRF%=*?ji_Qy3x4_qwcV;R2TCcQ`bzP!j=z@^QI4+PGrOx;-m| zV>-mjuFl6e1-0hvtrsNaA2V=kIfnUGcYn@GHz~COZ01xa>@-`oCZo~${_HDIqrO4> z<#5L%X}mi%YD{N%T$UW>VB}`~+dDgW;OR;$!I6qRb=!EZW2i)0#me>J{7++rw!_6I zKeKAmTx4q=eB}qeb`*C8i;`%&P~eNH!Cr=4sDWGoYRZ-&v=U%}z8$Mn2HgvPWNUsd z%}I}K0PY*E<7d7-<>K0FIS?re)~&u*HOHK5T)e18L4XTkuaWe3Wt>gGenFh@Px=B? z5xPSKJhlvi16kgk7$GNx85!g;G(npnUrwJnnGu3WYqj z=j=2Zlda;Ar505tvxr=h=MUubd#Aa~%E4*x8fATCyt7kJtDb|(^j?bPeL{Z|ey8uU z6<=h7KMMYOsT~A8<i*zD=B!yG|lT|asQe8nZ`wG z4^*Tjv+@$w3XpB`Y=c-PITP#daatC2i6{g=a2bb#0O?gL2vSWE1Un=A#^m@8pPuR{ zHofe%9KNsNmlCi+9JkB-hy=$PS+^%|-&y?=pvo{4J-|H8#sx#RdYXuP*FqVDsm=Jafa^5!w2iWj$p@MYDMA=Egmgkv3d#S)DJAjfZorKn=q5 zK$y<6wz9ZhLKmz7@20O#(&#~*{~~+3*2Wc97E2ap_v=0n5MxPa2*24d6CEko3X|V4 zlGkN##{1m_yKte4o(lN>dkw>fD_vsf4IOKsD$7Eo=AY()ewWJsCU!gqCfX9N6li{< zlGGcm89QH-G~l7`POZs=Z>a9y-%7qPYWnY6I*eDBO{&A4@=1nj@bl&aBOE&_YwP1t z?!MT_pd3DG=`$C((N;#YBAIFZd;z^}(FH3brSAzg=T4Vt#W^VtbR^m*^)`VoyZ|8+ z)3L%;6it=I-UcX8tpML?^jQnwn)J=~wRrlcn3am%!K}~#!WcY)mZQ87-;D)TWkqWx z#|sHb!4X!iFCE9BfE&$kMh|wJRs|t4b{S9Qv$lYrI=|azm%??OSeNPN*STyn%ZIGW zS9YK_*Z3>>vNcDD|AYb>j>@cXdm>vS1(wXcXx7waqgYzLw|6ZBUHRMpS{+(sY>%5c zu){_l(!iR1)w`14@?k4w6pz^OJ@-4h@QG1tY#wb_Laz~)i};l4SR8yUX}jjGc#K;w zr*v^;9WDH}Pv-D8L}AS&r#8Bpqv}Xoqb0^mqlUKd^KS_Mc97k-Mrx}ldB9w`Vgxh^ zo^Y$U`|s@))TVyO!hS8FW35PZ&Z&08Ix*s{I0;$?YjXEGLETW_mgB)GKa z8A2{>qd{4{xXy{ZpOvya4|B*;63?(r;IN5U>$h8d5q#+h*O+VhQh{Rq9zu5C5M&_h z5c!{Mz_|P@^>KUG(KEQ0V(o5XPKNQs=5e#7i&+#M6n|o9gDO@MoWbB&M5JW^4jZo! zS(xi3ulb!sdYHf^0;5>qT2WNwNmul~SMgERh9xf9?Qc6n-smZ}L zdF9Mt*af?1xS7_TIWI-s{}yueyI#{W6nS%f}7}0dE^! z+GFo7>IL!}CCZ5S5%>LQ?eMbE7{7ZU*l`k~pT0mvi}*A8`5-gke66_+eYaD1R%L(= zxeH@^z4~dSUoOYCpG-a=6!YpxZ--}?UWQY+Yn1M_03s)r`s7`y@U@|W~ z3|#xn6VTXPT3QZ|!{kCY0W+v=ZkIOgE+WK{VTOJ-*7fza&Rcavb(8rVcFtT1ek98c zP!Q7C;Fl77rrh}%HeH0sS=pDv(9w^!97?Z~(mtjliQrbuGKiURi;Y!Eqm6&Sc{T=3 zi%yk=uPW@>MszS2?Rx$ywHVvST(b-s2Rdc5gRc1R@jn82M#v76Iu1>$>+k93qOE6_0aEs7!~w9~r}Yg^13?3;zGYLzs}I%!l6lvE&YfVQpj%YTzCMp~ z!!+ddjvGC?5m}s1cE5t9E_mX;GyNMOh#+ERvcC2&M_B**o7WvQ?ot*qswb1nO1S&S z!4O8Hz#)M-Yi&9d=(C`#;f`AQYV;zlRy!ht?Z4*a>%$}UovMymkz-J(qP&mcQ)!(4 z=;gMoVsWd15&6p!+$5Kkw3C=prcUhMv|8@vfHkp}C$xl!o&^Cwe0e4ZNl{h6oS7Sv zyknUlh$b9EgdGgm7Mn?N{YJRd@?}`Ws@%u$tEXyylwub(_6hjVO^+lUA%vnsn~LYc zz%OL*z%iL~491G`@Viw0gEy129ZJ^*UQlllH~j?C*7ZnjQ#M{U3!U==(Jr&Y#DUZ` ze-5_JFYr~>WksCj#zjEaH262+O$4COpF3lhX>VmSc~i0$>Qb`Ukz>~?_fa!49_Wy8 zLSr|qqYxciItoSj+S{cmNr$hjqf0sbFmQM&5$g*3(d}$h1N#nVJP!0rnP$jdWr|$O zOXP|7|E_4dV}HY@s@u{#=AEND2*YN+JGKy?)MG#6_pQ95R`KsR@G^%3Ii4e=$Z#5I zIcnwi5u~w%oO${_JBb{BF@q>kN-}=~qkWF7A}X_8A^yO#!Hg(!KfF|;Y;q;ItmVqs zcC^yZ9E`nZ3MNYIeiTVW5ixW$Rf4UVLc2bxG5k=dRM3>B*k;PV)B3KLJ zaGKcl0#!zd3_`M(1g|jqAj1U^+yDn6cjoFjOR}0(XGQFFDLW{GY>#Tx6*peT_Dvp5 zY{1-gtlYCq8#UeBSWHvIwG6-hTQH09IID<5-~|hZ$kAQ=pnngHfnc#}Nio-)!q@1X ze_IM|RlXy@DlR6w7ZhyQX-jR^)FnD*Wogvn?bzSE}610G%NAa-f1rkUc( z6)d=@oZ`ySDJknwfi$aGsmAwj3wS(9VJTp&15 z`7*(d9oc*@WDfiK_(m#UgZoaoX}n8WSga#K6G$-Cn4{QAk)(Nn#{@AQUVFg>r{L;D z>nTshD%GMC$PTV#y&<2dz!k0N=kR8C!6?YG4;bw$hGWZ6{i)KNzhi)QDhEoQIz~fI zsN*N9EQPP~hTm?SU@PeM{G5efKeZ=1iyRH9-W-51+VK@b5b+~FEYzi>&vn2awkTe6 z>sRgVbbi&sStvXw^A21G_R(!_xZsV&-|2zTS-x`bj+cv*h|;go=T1${FB+lZL*u2w zvEH%xPSC8?(S$+DaT%)X6q9?Y*% zI@HlVQ9~b^_olnynRKa1ny+r73q>Py!(81R;BNLvYZ^1xBxf)u-}KiC-n-lNXuk`F zp020i_1+FBBCfxqKRvFJ=u%8K92muorV$bN2NahMze(d+bM?|Q3!e`ENWF(aim}ua zvrX4?8B7;A859!~6NtKu)wWkeyuTbFrmmTx4}2a;13^xQg0x=pGC}hK%{HB;YfJ`)2e|x=d9| zH*qn>%kE*yN_yeyga7o)_O~ssEu$@SEc_FMjNhhl2k@)qHlJ2#(qMc6q8N)Dt4cea z&6L&4u#T?%|I!ElK?e#)u9)uF65D%H~}?=PLh#M)KIdbX|0(! zr7_KEc!BFM07lw$Zadv5%tFe0t*eFMESl4u@XX~`l)Xc)MhuI?9R7 zY+OKz(;YYZF-yyg;3ZL`DfbA*RKG!66~~g4!*d)Hv2E^;j}H$Vj&No)Up0o8{t-1gIR8FiR3V!3G!EfyR_|l7CApnFXXE z@Sk36yMO1GM7HD00!?K4X0GIL7?UJ^#u**e{qr4Rs<6F4nP|_);&dauA~%cirk@7R zV33T#GR^w%FySU?1mAXjbuls-1AA*4IS=aG52{cblx!p6WC@WtvWXX~2qHLlS@-s` z%sEbra&vUGtpD7_i{(p2H~+Bx2d&oCuOa*b9uSBd$2)|ltOkx)C@HWs?M1nc?4XhH zfYaxmi8>O*wv`ovvW8)D8GB@W6sg8WwGU`rUA=j>iRTioLE-!~1zFV7zsa%phB_Lj z1gW^tA~6Q%@o(ob2P_%Ulb~VbvF?LIr$z2yGh7)nQbI1qyH7zDlGx!WDQpMg<#>u3 zOP{P2cvntOqtX_y*G$$O(!6$odNYF1fyCm)VNva?ft3exvB)W5@4@IQpIN zkcmj@EM5ZTiB1{WDk(Sugw##)TH?GsY!(Low7|S|hybR!D!f36MmO}9@B1kXUJ%95 z-8i<#Y(w4yK`4?h3Jt}w;i$)GWvv}JLV5|@XRH_YMp2Pk-1hp|&JJXGD!a}uhK_Bx z2Ung6-E|fSFx)MpONHAWZD?liNNC_|NfU|$v+SvTTv*8ne$(dPDmHrBvZN~uA{n>g zBY(fr*`1WYxmxT05}eSp*!TBjs2qUZpE)X1DaI%;kK`2VOM>UOmwoYbKejcKB9BJc z4ncGz>2C9z7NFl5ZmW{=64Xh3zNp^1ORh+_NXV92F3Cb?EhW$_{xL zDrxi6I>&vvw<9WYk zvI8RaV`*YgZ6Pqs`|wPH%#cv&QQndWI^Ul<$j>1t$A&T+Xrx-M=*G-+V;;)_u`gNb z*RzKIsrKAizf0CAClKy%L_Om2J(up%TCG_;g-xUf?nbaeI-onLFZzvSWMzh;>a)a7 zPh^d=HL4DFrbUHUhu6Et9Zz@J8_f(`;W8I@9u>Dut!4v!Unnk7w}oS9_ZA0%T!7L3 z+h!8y=v9WVpgZ&rg}vQ+jixBjDzI~E*v`LU8XSqlym7w2UH~GE>RZxHX+mkLQ+%Jb zav4{*DK8ix?cC^)y|h(|)@hh~K7=G+e6obaE)4S@r+AaRA;+wA^OPRv4iPPlitI5p znm8?q=&(`lXef0vbETjMV)Q8SkK)aJ7fsrzy=NHNk^O%T6^Yu$AOO^`(DN9AwvItV zOq2Gs*^S*LWs;H!0f&gF1cZUv}pq_sYig*kwRgf4+NT|ejx<7-tWvzk-7Xq0{EkUr>3Tq<${Is751@6S)!k<>4( zCF^#L0)&&B|3~uhQCjoKE@1f?t z;T%}mRj`C#Q+Rcci-Sh~Z!Ov6w|r+)o+WAHPa1Q7pve;RR!oT=s6ANE_*_DgA8L*ALa>*yv`Za(59(Z0u4r*k#B2ROS%u z2HSt_g`{Z@+7lcx> zzOAWdIpWmAO^T#yYtga~iI_*K!NZxL*{(Aj^7nw1rRGamJ)y-zCaJTwYtX!~e}`OB z>^lra5o8=(J1GJFGqCpbpUp7RIn+|urH>*-PDD1SucM<+lt|LgzlUV}rnk1u+B6DgyZ6_6s+HxsI$jp{&GUo8P`*>cu9C6!*eXOq3%ep(WM3i@kDO@@jOlaH=L#k0b-#Z`|% z36Fvm!$;%8^Lc#VB)Ku=QcqV4iMEmEQ}Ze3%6dDgr&%!Hw&(|iBc}JHPg*C?hr)-| zM~^q<@lfL|pHc|K-cddTK3pGyo?PB8+^B>RwNEkC$cB{z z8KIz1{IGsQujT+CcK5yJ4~9MtK8qsa(8>W=oO zogF{~GBtrV0!;(W2SGE28{duQ-tmy}o+&ZjapYUe)qGDQUO1>QRL}=^W;Z$Hic?+#&4*D3go9EX8sf%!K1y#WIGd&UhRf;2ScNAAZ&mmbsR#NRj6598061j0#A*5`+@ zQ6?L?4rVv&gSa%NghyF|R0P=<($9ZQE(PqU!0QQ5zU%J3sAG%SOW96_>ytYsR4!Cx z89dOC3^3E%^kwR9O4!m}|0T2dRLOGZ<*88VQd%d~@c)X&{%>&_G|8#(OP1!8oH?*~ zT&Kf12%W*y@t{hp`&dfope#geyVPU>RJERk+5q^oX@vtdDL{{wn=J1J7T&>1M}Mfc~}!&BjRG2VmO?(%hm z?ISbTBmN0oZ2@KDV#1b@ivRVo3GlqsaynGjddYB}xi_^n0Dw)6s z6|vJDPw6FIu^VO_;83m!!}kdw+p)pTRD%=i6lbH6PO|?lcVYP^Zks;Vz*YGPt^elQ zlE7nP8%$%X16M(~-)Jq>iyL1$u!}H)RTH)-fX@7-2X*QZE$N#Z>GugYkyM=*NDHvf z_wyXZ-e&RW`lK3)faH}XY$;5d7=blZda^X*(CWz98wTkQIX3%_a5d#wUNFl zdx%E17aGEh!OK$6JtQ8;AGR%tfl}Z`AC){_3-8@GU{C>id|rXfQCD0kS9^F_pWdpG zu@y+g3no(~L9B=m7QSM!%vqmQe?Oru-Je^}vl}7%&|vD>h)Q~CrbTv?e^+w+tzh=? zj^{~jUZ?Zl5?s>BcohB%Z;Y=b3!(Gep9+QV7C>*VRs^V)NfREyJa4r#`iyNY1aaeX+I7{KrVvh#fQ0J9VrCPqz+M zfHh)zCT5=gJ|%#nI{HVEu5NcDCD&wN=B@7ySGr*8G8Of~uYQvW%8~kb}Lohg8FX zVR%bD)?NprwFhSx#s#uV-U_DepP}w;kHH&tkn>JMUxuTqR%sX*bo0fn3s|eyJnT65 zJpm!Mgx}^Onmod_ngu~{&;mC1t0f8=ui#B)7oz)484+2_bsM7orv@WC@A+0mVieR2 z`~IOX5~NCpRg**t*gKmv`}7w-8G{$wwPBR9X>A!MjY}|S)2ti#&~-)7D66QYZt0zl z=cTHNiXbw-i&d`9)=gow7q+XHyV#qxI0)L^(L}Ff$Q#iB2wxtfNM0JP9}1qdLoGJ{ za7V0rE||8!n3%DM5ay-v4BB|Y;%6@_G}h)Ano&5?u1(qMW=Q|7XtM1rgMrT7nPtX( zfo1KF+&YnA&vbM;3Z=!09d7HjWdoiYhlc`1g+VN`h(lGl<^dxsxK>YNi&lV619YT> z2Uyx0+I9p{Eh4AHSbf1KdqMRCYdgiW`{SR=Y3gHMS=>+52>6 z2v5w?6}Tq!g%)iXf2cOY7j*?IHPV$~t8P|xJ&*(WpXXA?&xfs*1;f$a;{S?NU1=8v z)aAZm6?s?lKJ27@$zQVgM~SREQKj+{wR{q9!5dcoR#ok?nRoeLy{@jAWBQxxep-p8 z(oE52F^nQCfE8u7G4I#N`TqySI+S%vl<}n$Se%*N&+nS1t&SCy zq;nkqQTwA)VFh=6>3kU@57xz7Dhs?fFO$+s0*CoP{-R~aKoEKNP)PKR;kK=Y`{3lM z;I(Y}SLg`|f5E!XH8#pTX^H}Crb4i$lbRk*tcUQUbz=u_d(_~on(vAdBD$Q@Qwm{& zCsn%IlWr8$T=&uHLon9j)?s57S15g}OYpP3BiSgwyPje^KF3RfKTN9Wh(UxK4ksbY zmkfx3?nmz7m|LRVG($rbh#Cz_sDSdne;sIH@!UF7tgTEX^iqEvW6fem7B6e_vyL;4+S?|$-F;$}Z3^r54@ z%`N!XVs2qdw@gVaZ9uqApsm@*BI9OQvn(YRkLDz3uU)!O= z+!`xMZZic|4R8cI(0^e^wWOrAL?k*dNFH|Nm|>Itvtxx3vj0-3coH#HSm`Jv_+48z z%dcD63#OyC6zxrs#~UH5Yx z9I>%mF5#v{;#q4oOeHD~yt6hsmzoGxyvO-z<+(6xf2XUp&#+=DxgG6F3_z0OG%p8& ze{dgc@mf`=A%Lb~sM`(L8z+NV6xXR<36Y z*Nt(HTN~+nS?Vkut0jt3ZG73Os)$trL!Gv zm44O(mY@k+qj-V_VXE7hsOs;Ao=DFbzsrvsgg|CWzQ{g>6dv(J9Ble;6+2wzTXI5a z!cUzlBaIKEL71Ru)&=z_Q3cxxUAE83x$+lS&H=zc-U<4@oN3+485H~q+qMgZdUXT2 zJ{7GYC9GJdtmfq+Un$7=I3|FK!>@h30*6^w0G_o3-JQ$)SKuwvH{;=i;4m*tN$4xg zc-bOJgkMTaVU=-SQ;r?at@E8G-oR(egORbp(#pyVopzu4s+XCusgsE@5Yu$pK^mVV zpPUc#`^(+)ai!E{66Neg+-9^F zv$NS*esyk;^!D14V|KHx;n_uX=8fJSJRs%vqdMhA24HfV^hTGf7p+Uc6Yt~3oRm4l zwm%uQG31rzLG>tjalIDMe7{yb4UwCz4WMN|!GHJKOF45PQWNVK(H+*sbF`#J}U9YYaZLSWwguc>nnw5XeC%+hM-TFO3eMo=Az7an=-z4od?+x#*4=+Mx z;FA-}A$?DHL_mXXN$B?LL;*qgJouzMHGKAdsJ&*MY@U1)+i}?0d`5)Tf7tq%d{%mG z-nDN&nr|w3#gGw5mXYQB4`t^R99bW=`(!e)CN?IvZF^$db~?7z6Wf_gG_h^lw(X8} z@}BSV+?=oaqHq4SyKC33TF+kVS-&T8qO-Gq!#BM%zEf~_Yv_wmjQd&qSiGs=&G^{4 z?p^fQvl-l>Fi7cz=k4$$-`{mj5NU8zxQpe3cH<&|DR|OG_{#R6e*V1N%DBH4 zG;$nyd~jJXgVmLWL>lc*Pk}k*c2EC- z-N6j$>Qw3e>7Re8yz3Z+ZUfd%$r^1Dlq3+ zRLAZ7=Pow%4Bp+ifBJb)q~tC%loGQMlTZ`;n6EZY$MW>8VJ|aAfoJmqh5oF{np)rC>_!D=S|C4+c$Z#D*7b&fxOtU_!Mn&Ky{WRCE%UDIcaI0Fsm3Glse={Qsj)idspplsI?^19srR~YbF{jy#;=IV zDZjZ}rLn8PG5RfNeXD1>Yt^yjZnv?rKxmzqdQf zWkbHs3K4SE}=F(TQg%R&d|*=YU#(HX%G^n30=vG}*pjdi+rwo=k}v49MsK zcsEUk>@$~cGIqu|JmK19hcP&~)i-6X4P0ZMM~70;$7Xh9l)$;L!(d7uC8n zjbC?%T>HPDNObG2U16uBW3~)8Q+^JBNEu(h3>c^jzo-Uq%Tcf9J~~#nNIQ+>a`*tRZ80E=E}=Y@&sfT}ba3ooZhv zY@E(6LX8v|DASsqBwGtGUVxG=d|V-p^aw^o9Yb9CS!47~x*AY!8cgOKZWrg_l>uUz zROZfLRuW0PsYgVOzDI?z^mf?|xeC>cC+G}^xRrQRM^L{Q$hbqw#vWU`?{+%O(&HSA z+EDpH$*I*XRC1$8}JXV`BCm7DJ`*uADh;lYkFi<7&zdaE(PyC&9d3S^hA)r z01V`pHboh>6t{(VnqJLj<)ZD(-+H5JS^8;AFmVY|!c z{_BOFYeoHz^`w^DI&>R1a)GLO)7XZbErAOl+nohc;#wA%gO-#D(ID5aMnfPq@%vrf zEyu(t1ol^HtsE6EnPdk8wxO~8TNvsg=Yrl4tu3w!+Kk*?WnfsU>+X`-*IuDKxgJ#! z=ien9p1C~75O%}UlM@m>8C&|QR>S>8$EevguQd4j@5|p@bn?{p+}$&5>GPr6-T;2% zE_&Ntc^i9fzZV$w(>e?xkG$Jt0jheq#RX!B3|s_V@@*iciR}8$sNB`+X zN!NMA`x?=1`q}kS+b%`xb(8S2Yh-QsYi?yS#qSw^x|Evr@K1=C9RKWxGfN($ws$b) zs3B^|@P)RB@BIx4OcWuX>kZD}po~$h7~Sm?nOnCI>q1Xh?!ESE$1or!C9M-NlN72+k# z*0?S72>2ap#%;4~t=GmZr<+uR3nc|2)qAi)At*a#ooRO+|K1v7uv&&y6w|<{tj(}< zkqOyK&JMc5Z{j!^SisMuipMiBZ6$obyg-qv4?k{a&o&#?95`OppP$F(a+W_!Jr*Kw z7=J$=pG_XQz6rcm>RBeb-=&x%EP2it5f-7Jd?V%3HmQ^?E!dBX;&EWSv| zvCKJg(&xlOlvzJm-u0gS9Dr?4sFG6Nc92BRt^ zW>M0*=Y341S9Cqp%hSSAC#QH0RXa7N21{|5f_KuoTjjkWO6lqHmRErd7}C&%bCj~i zdmV#t+qwb=cNY1Q(_m`mquDm}!34Fu zYdDvAXtcSVJRF_`lQXLpW4Em4?8@ZNyRFlQf}z;lRX@PqQr;%H?!iU`3!=h$AlQ+0db4tIg1TWLj*DP`_yL#aXodd6n14Dcv#A=Cc z8jW&abZz-Fu==~x68QDw;Bg+MtK&o_uxiI#vtu_J`ho}VZk|k6``C{8FfM@p280ej z8Yig}2z5~|^3w>tQA3!3Uo{4-Wc|6rn@aDS+E5&U?4k;3TKM@vGnSQn$#1S}iLo@8 zxq!MPM}%ANSR90ffc%n4#_DK&vV|3xB*s8DL;w_wmG+zFzwh{fc>*anRXz^ZqY(05 zq_lNFakXaaSD6d9VU8mYAbV|Nl;UJK)|DEJ!|vYt!Cb|G7GyimPE88B#OqgRjYghOxXCX~Oa zuM!L_ph#d1A!(0;2}hBl6DHvEE7sVd z#YNZe!ntj-3sg-$W)s@f;lWv9Ub@7IZY0U{ZkF`+im6CBo=w|NJ03=)5BndQ95~KN zV`M|;_l0{S@?Sb)Y-+_nGYTHO%1DzXGpi|eum5)SI5oloP01VYMvTVz(~d6%BGwqM z&p7?Oq(hf3dpyJW$5cWw?XD%I_`o;TzON#@@m~$CbCV)53GScMyPF<~6PUf4JYwgVd#q6R#LHm(*YWzy>+uKzD$g3Ok6_+Mo=Dd&DD>E**r z(f*&v5a4(?jiSm0Uw>%PI+(0hfR`+}NqDy=2oY5@UIxP((ibtA(WoQ>b4%WWi0*=< zG*qOWTzz)U=u=nVY4kkGx`$oNZA(^(R_c&u&t=Z6r zgh2-tfaJ;NjQ)%vJL=g-`xmjMIwU?FJGUoNqe^&d?sb-mr;YZZP4CZwPeT>adve#= zpJi=~iwv-IHW=JG^BjP~flw7N@QYVh6t7hJ*g~dh1 z@P=&K0Xr6M)WLppda?jPENNNK&#;F8GQveC;@>XE?X2w%#BG!Gbhu zgtkN)=@MSyHd2N#=e(4Hb-uMK0T09m=66IT{TtWQ*R4IbxSiQgt6)1{U6G`2S(ySB zCQp+#=4h=^H;uf=RpiM*t3{_IT_vpR5+e^ez29nSMt8}9l2x)SDSihcA|+v9nluN6 z6j{kZiR%x9l*wiALj^;uP<6)cCer{f8XlFa!LXaD`WnH8q-agd`(gAqLYcGNK&bPG zX>EG4Iy%@^$fWia!{ZF&H+T(L4TICa%xOERk!rdrI7^0u#}kkvVlQs54)!wmA ziFs6qDBX*g3enMvy4&409eN^1A(Vq!IQ*h$LP?DtJW5t&q1^*EotKz@O&YqST>-|G zNB8`C(Ri$3XDZ`pq0W&&E@Oma>&xFTDLnnrB}SbR^7V`fYQ)-ZtF_?Tj`wwe7>E^W z78smgBhiaE0;GiX$*da-Ss7vCp5eTEcjVvBLGX~BR;&6=ohc4i7FrwI^35PebK1sAZ?kQ$XHrrNx ziH}1~3U+EXcxEmS1qy}=SzHZzr2^U;>SIkm(!&vxysU6tbSJX1btw78Hh(y(ruXYL z?W=j4(5jWI;CfCof=f6@?{bxn>(cGe^I-CfJ3{s03OHfbrdD45$#4ZEq2=TNN=_{M zD<1Squ9sUoHk}XLSh4hJJ7Hsvn{iPGq<0`Z$LBtzR;(KDEo7mxkho6c*Wuy9mIZMt zzQ1Su;Ug(XQ<|-;l(nAr>#ZQUZ8q80e~fQS6k`B$q$(Kz1l>*7he z^H=rVQ$cYfQA+8oE!+O*;_qJ>4X4zybpUr8PhqoK*g8gA-@+T5)P zI1&nos3l{)&a(xzLMVh)~i{Rii)K)NeVT44={;7jLxBly|ZFS0kyn`R$CK?w6_S zsV&GZST2nFlZS*O8_8q)-?$B84*W*^65q*Q?H}&X>%JB|lFv6*8f>}NQq5NI(yLgn zEEr64M$b(zC-BEZShCZ*=DPk?6$;9ArTa0wd$cA9WXKo3pA>-U#No0eMGfv%fH7frWwvTM|_gTR;K4jmS+R zdhz0diJ4qXK^-59o96-6Xk>aqZ6E^@{Ri4J^R3pt^ZxU&N&G9RP5gK1QC=?j(=U?nLjp!aI?C zvAsw^KF2lF5c&6tRP&fGu9tN!v@TWS*_mIqjHNGIhV5nU_Eb-7aB6VXcOqm$ctXU% zAm5%Y^QtL7f!E3NvAQF3qI7;Xyv&Q74xFc-*V12%vFC2{?al5)-w!d9p#Lvq*#AQo z`z+}4#gK|Eocg>7hUnVe?%@7hD$CpFL(LKvO~>X@!|}yNixD6+KKz^5ALUX*1+N+& zRB)tQ?h*u}LeFK5In^NP1$l_2J57)lq;vo z4{%_3EYDjZo$1jOo_fw)+u#gnPon3L$o~RE|*4yDHoflkv)e)J0r_btfb<_ZIGfwJp2s&b=yAX?~nx!(@LT~4KUYz*^cZ7 z#L@1+W{F-quz@9R3fNGvXhxF^ZZIHE%WsyC2JJNZ1Pvt-$Sn%FsSY^%Ntet_l?^z_ zyGO-UTiai(E()T(<-z?-KZF>W%YZ{PdukCF<}4!Ax)l^&-_)|ax=9Ez z%8S80jh$!fMoN@nb1-aSEe8K=R3vLGuv_#_2bOCC&vO2ltCmeumvw_c}!ElVKEn-Q38{eO1>w z0s1);f1VaT*P<(U)$9imqYG_UAiH-Lq&DycYO1RcEfTft{X5vtL5GT zN@41Wu@`|2;lo>A`tlBjdJN+Xa|X7Rgw!?s`Q%<~r&8I79fEPhf`O?S2XftlFhY)X z2}A~!ytdLJ^)msP&5B`mS!uE+Wd99P#YZsh*2yu*gVdG7w&s;|5^>M zYL-ZpyD~Ff!)}#T1AUFhD-&lRx((~UC_Xka*ku`xnOe_XFwcd|4l+V*$8;obm0zUz ziza2V9=D$DBDofOZTXGtz;A~s)`Ii}10m@-pczydZ2L9%DqT^RzS6arEt*Sl$dM0A z311FA7~zO?or3RW%LH=F*g#S5hTY+LZmFBWo2(5N0S=mo*Klg2wL(x1yQ=%lzoD45 z8R?r1tKpT+b+@ycJ!`f7q2-#>nP>i0j55NjoUX;k(CHTllL~ZXe(y#_6s?R{+&8d+ zL_Tr~+50$I>O{Tiy+w^{6r;bg-p;kP8Iq5?XwvSv2)3<^NXQ%8E zmwLae79xoAR0dVqlIVryE*8;7L?u6epCD=|=`DEtV-QiM{D%mbUam%u8(tgj1u0h+ zA&sZ2)9bo69D?yz>u7Yh<{ao-Z{Lza|0GedB`@tA1GAFjDy$%!Ne-~-B2dPuRZ_dO z10w)p5tGG6fFddgJ)b9{Pve8Y{(S$KsA#-RR^e|X)KM5Fr3DfOK@kK2#M^qcOIerB z0nCx>%og3Pb^}OzFMx#yn~?_!Xn`wNo+;y9#3nF8S=`%1L{u14k1Wz3!&|QKrSO|P zx2!GSI2DKYVE8H0S7xv62}amb!uX=dVs&vhF*63^&_NtwHVVQDJ((1YScQ3CLZ=KJ z)R-=m*L0|d9w4O9qo_bLi-!QJn6O(^nI3XlFCYWWj9q8I=Ea#I`GoKViUPn`4e(Xq9Ygx(_L`x2rW0K-)d`Dla6>08;dS+ahn5$SB2j_3 zh4Q&qvfoOR3?xcR5}9aSf5OMq)_7DQrkIoh>b_d~-r3dTKPl-pP6NnU z;N>;fj{*insmI|A>gGnr#@ZUEuNi=q)0;s0Eas?sN_z~kQKar9>{vao;f-!CTFQk}_dZ_8&!R2~ zH~+nrk+ZVgDt&!lqx*4dv-`<~_0#@;7WYlFsfF$`RY^HC*;E@^qe&Dp<)FLKV%n>6 zT}|#TXXAru8OuVK1M{1X@ET<^AXHQOw0BE6hLxJD=V$vIc+hY#U2E~Y z)Z5S-9vU553`Yqs3AZC+vD5TLcYVC8I+{9C*k8Kq@v-=rTIXy*cCI>_Jd!^|Iv{D8 zTz6Qhs@15)sy(WstaCRrS(!Y)O0QzMGGj2EG$S0v9~EQi9G?I+fRaJ*CvzvfbMXM3 zm!y>LPCX&#Hhknl`hIIna*qOt;2C@D> z{OLc$-U6QjJ(w;dw<3IT+ECkm@DlN2j`c$s`_uV$fHr4C3teP~Qjq9EQ&v;_qWw@h z@O_zIG(nw~%(xMa;{FD2Za1g9c+pc|lb&`L=7!~_C-?ofVaa$-5(P;PQg?BZ_`u;U zctLuf_OHdMMED`hMa(eF4Bi zI$=5pyqEzz0Cy^s+fe&SU(Uy#^)eTQ@$59(NytOoBR>Q`Odr}Ox$F4t8AF-%#ZUW( z)ANB93k!v*P1ejzg0wF!#o)+(EwP>crKKi?NT#u#Tn;nSku5gUJgtjZTmTfDtQ>P( zU#!+uQ1~2Hu|bBP&r3gOV=n!qyZ*~ePjgj;9?xxt#kCupEU@vmY*Mt(BLqrggLg z&NvUt8M8RAr*D#J9UZMl7vb*=CE6;mUfQdMBrsezm$IghQ52mkmuw2E_}ruAlN+Uu zOX<1x6PucdV>e@0PJcjO4xM+X9sv?7uYYSV>NoKfuNjr;tTS;NUNvxaM9WXO%apj1 zZDlo{U}C>^m>~bDb0tOpahxJs*DJz*_&qu3vU26;?qD9)jABRew{XL09P`+Ir=Y@f zCi#_bqDStui)dy{f{rPUnf03b4N8hRRF@4iGD0&W0#zY2Zg$nln8m;yp?T`@=tb&1 zusZ$iGZt*mVZRjt87^J_VUE{RF8-`NSnVblSBw4&CaR5MOZ&qSnx9ieb8DEcAa$BCy+Z6-7D))N2N)b(71ixXOZs1IZE@JITWGIV#>l=<^YjvOGD!1{FU z;lul;SKt}_^(oTxCjARoaYX6qxnPQ$&!{V=vguA4B4|?zPAuwM}uL^toP78 zJr@7D#^ym5JNJ*M51 zEfk5%c9f6rkdOsA8xY8T=W*6A&G3}Qp>+%U=Uw+rNROU1IrEDaf_#UBlX1j!YTxl}&u16sIn{COmb@g7<^|(}hvd6Htz^2Tx!*wr-XxBHzK%7D~ zV-3x5>%_aJ*+r35bHr?Vx+6Q6+b%20WAq z_C5b@`}x~T)r4r}AMd;fL-8r0>=4wPoRiSZggVI6+tu=&uCd&80l4R8w0NytVW5qHn5`Et%e5NNBjmcWaDn}4lb+h3Lo z2!WfTkE!t`MVCCE{1dFBM$M3I$cGSquhO2PRBzvqs>lJ;(QZqd&#ytuYALh4_{4cY zrlu_A!`sJ!Zc;dbYyQNb)YLJq9<*YnvG)WBa0OT2a4u|`r?HWjU-FoyCH0<2nMc|_ zRp+<&uqp!s50kW-HPVJUgU%?3PFh90{Eeogz~Q&R#jsc5ZPJqTc&_Jwv(eP>Ui)Vj zoGufUV-)rk)?4??JNd`$D%onoSRzBd+1NTZzBBdD&&|#ysvd@~WRjS_a$CJpeysVz zp^P*t3~2;YNWtJlNY7{bLEtuq?a(26Z>MIW&6_L@63;CMtu&Uw|dVf4-S z2IVD-RM51*gHUjXWE5nn_I-UtbUZ0!Df~li&BQEbfkK9#`uI@2GvyYTof36FaXc!b z+!~p~rV6$Q`iiNn&A&bVSr@igv~Wz}$8j%c-j-MFm(=x-M&si-6n)>?7Zz1zXC$w> z`6P4&Lbi*?wIRlKs5d+Wp4~9B%>9u#m=48GF^^P|fVR%QU-lzML^@APZ#THWY@~4M zv`bq_bTssJIcr=Cve{!U2s`SgodY+)2Aq^?_vR4@2dfN%=~fsW&T24d`XMdfEWf}> z2(bp{H}*#s{z}Ym?12e{kYns?%fO}88#Qb4qcJ*H7G1AbSvRdC!})=s9ziRm+B1K_ zswr#Ak5CoACI?$T$UJ?DQ@0hK*K4Kg-WNd@UhwrSViZx_qV#@j3A4h7trGuz)iYP; z{=JRUxMI;W+L6)t*haFb{ELI3H=m>6hI|s$8l6={lfKasa#prxmd_!mB}o=~}@?(v_Hmk1qy+t&(eUe{5KJ$R8$tji7=m(an@{B0R%;>ueZ;5-=5 zUTFg1KqsG{bDj;S9q6tVa3!27qV10p)LupM3Y0E|iEavBHM2LoyU z0PdIJ1HWz|S0drl3BP2*GQeZ2>X{_LJemYHCJYX;mEH(jiJlIU0=&)yuQi&iXnjic zHu0WbSmg$;JOazeCGkzNzU11hR3=@n0OSzV3rCoTmeGQU%2BYamz3;+-gbE(lD5kU z5#TmhpRS0YPRN%M5%P%n99!=oX{06Zc>srUhO~Mgf5n$yM@Bx7YEOO=jCvw&OX$Ol zoG7og-#$)iM@&adZPWEa#PwPv!s`Y*M{PL+m zA(1RmYS4d*=eg>cF?I8?OVz9Iv;OOIGYe5z@BvW>6jZdP`~95xCRpK9`?XLdnV71< zATPlCyu5!$J4Df>ajD16rKJm7DMryjo1NH&QH4>3{+ab;zFQF4M4_Mc#cD0weKt1R zJr*=k+#uFkb85V)fkA zJNYEKCpqY{XMnZI*sktTS*_wx&BVXS=9lMaa$Z)0Jjf6e3O^NdkMS{LFy#=t!T+En zz1MJTxKTeK;MLjlZUIvQC)_5Uft>61FndrFc47^4u*?Lx@6$ z^@`da4;p^a>t3{9qI({p42XQ3uVr46K_fl9&{M_m!IX1>J&iqr(3jFjvrEvO*w@9G zvx3E&xKDKVyWQ!5?kbN+?Md2r9&{Ll*NA*^FLj@<9}TKx2eR+kZ_Lk1pOPbEBhl}S zAi>2?ncJR^`wz;5Sh9k5cZ$~+){&Yo%!1BtRVUGlz@>NF)mr~hWa{hWJ^!+M-qnRz zNma$wK>x(%S%(2A=a(w#6{vId)%HsL3ZkQMlNT*;Q`eYhCgq9s@}h(AbKz0^?c#~{ z#=(c^W%D*Wy6c!1rHfY|n0o`cxA|mHLu8EnBoIq*tB3iF!C~?^sNAV+JB|fx7|8JP zXV^Gy=>$E`Ht}nuR6exVfX-)4WRV#Nv|E?abZ~X7b-ePAmX0}R>iDlDw;}|ve5)-w zDA!#mUJUE2!te1TKh}LKUS-@Vcu+c~-&G&YAN4jOFT1eX+PZYwTc2!}Ec6`cIydf> zm(ZezFsA?;Yih^Z1_*R7ZOgm&FlR6McZrj-*+ddKXFmMSt#>~AlatUSg%N_Nr*Gc* zHFsV^Lj+s&?Ol#WJL|nQp%7oG6r6oqA&h`_EI;x$k0JS|%NXyzcq9c$8a|Ydl{M^- z_1C*Mi#vyjE~d|vNAatbXM(H9XXp;-j-L*hk4+D}M+J9FtNeFY4PC1rVz2RvY;5Sp z`H2_q4aKYBtN!V)#TT!Q3TvI|ZOaeFYm_cWYnxeXw%$&cH!b0ZmsY$i8n&|IkgF7j z8EdPDe%9}`_cCWT6$%XkvoAK+YugJ!k6< zJjp!Vul*FL({4k=iz0~3w=xNCb<^I<4$jZt>jqXLR|Z$k0+AuMn%W7uu$otIg^1BS zD$Ix-I}2x>T@KsQAIZuyYKEe9b5nyo%w2(!m{r*tN2#eZOtcLd_iXXux ziUZ}PJSV9>P@*-nt0Ap2X9y`h=6Vc@gzxzbK!gYlRc7bi!G{- zZ9%{gaX;Bh+BKB1XAW(Oj_7;pAWy*K5~`{zBW39&zyeLWaag^P^ia*!`HV)FFwmy0 zi{58il%cVs;_OOvORMcg%0T9U@!Vs+2o?66<#hkLn-$fg%x~)3}MJzRE z-a3bE9S6$h9%`fr$yeyO>sw`=1Hi>9*l%Fweb~R|&s!SGV&jS-%=I+iN^WcL#BI2O zbN{v-Sdo^VCtw#a9bc}6C3~U^$pWu4$C=^yn-`FdY)lmq_RF(ZMVritD%N=s)g8e` zMk|@DzrbBzaR?o_h`T6XA#0+{b1uOs2{78JIo}pFspCQS1fg|DS$3>x^rW!9nDZZi zzyU#AgEiq>^Ow9Quth;qsH=u+A&Y7`zc5hAAP(%hh2PjsGpo4@pVcU6ZIswk&_ zCc}x4Jx5pe7Mx$zl64uF2f6+X4*iw(M##0CBaXji)kjcNu!LRF8`oLvLC(<0Z+(cz zY=a=3-UBnp>2L%1k;SB#CwL4GQ6dIuDP_RUw0wFA(#NJI(5Lhj^1MVtBSD+Xz)qzm z{f)26Yq}?+_wCp1&qti3!Ku0-y1+HZF+yKPjX1h`aEY4WNP-3$s#LMOsR|B;KMA4E z^Gz^x77zgWL0ZG7Y|61kN3Qe2U#niE9q7)rk)p{DHcIvExUzd3*2f2d{{d+u9Tb}= zQr76IZ>RMKp(-v{e?^46d+8prFf{ zJ0<+ed(@Gd(H)_>0uJmaEpd?=y^1m;s8p)lb7QE2l}T>-lNglOh*6ANKUoZ|Nz$Z( z`UQj^oi!b39Fjvew&oE(1*g0UY~pu3M2fqyC2?T9c~7olQ2miltH{4IHL{!28i*ee zX_Nh>P%g`~y+2K^q!yn=O^5Up8W5AwAx8+=aBz^#*&OsG9U`N|>8mY4$`j)AO^erR zjG^zXI_9MXz0(}Cw2Fg+%2srVSh3j0T5!ln%OlY=DJaWm%J7x2XYhC8GPtdPw@yvc zz~zK+hNE+A6dnlF@m24_@9)HYsH(xGVaCHb4YJMLY;mRSp}e~ zkkjHQ~z-w5l8$f{_ zQ>2|Xfe6A(Qj=Uc4?K3VA+jdwdqHAK@$tmzOQ3Xg76<1ASwagnA=yL)p^bWHy3POn z>oh4%fC5`>Pds9fcMLX6iH)_*SKY*614d46@E3NYUne(Zq0t&*a@8zI7EGX=gu?g&#*{{~~GM9{&gEZS6LeSpk+s)!Gc*>bcNv%318Y-Wdr=E4o0oNxMDk$>v&5+g$x)c+#D!d3Tt^rCz%-%{|VLoOA=y z1T6*Dz|uCk9r=(QTfV!%-f`0lI-M4Bh>@Rtu0@%g%hTwWguILnm%QnIOCBNI0LN7r zinX0!I?nI-Nwc!Gs(!e>YfIi<+utmx4dTt#p1)h}b=qJi-TM0(S{Br}7tUYcnGX&p zt+09{7QnfbrI+*rz$T=v`=Hg8bh!gYyMObDw1^Y#Q!=mp<7xK~U@j+}w{bKiU!u|* zeX3@=oyE{ykB7)?vb7oTf<}5)xOUFuwkRWRWAZlA>AKJS_kFO9-@-;b&beQ}`km)D z^u0Ra>6|QI!O?KM7pN0z{UjLecZ%jrli+uG{tT-U^1Pr23H zsS=q|<@c3w;fi7>T!#4(E+Gd|7aXj|;P%npBly%M^@GQYcEzEI4@-U1l}WvEu808S zjh~l|GFGAPv^c5#fj*uiaW^^k8{rz~!T_&)X(YGc$` zQNy3ldV~MotAS%_A>{<72v$C!GmNi>2vrkkh})qlwUBX$QkMO24HnJ>0JW(Paupwo2%DI}2Y;q* zCO(M=W}>;NFqf=oEtm*M2(6wWKx)F>npmsks)H^z&O^3iwRS9nI;_m>RRJe@Y@)I_ zPHI`JMh?I1=FM^?{KaZhwOb(%bR z3%OBY|8C4ixM*Y&e$(GUxiEI%ClU(T73%sP}la`7R2ip-GrK{qC4TyowzfcWXS&q0w3v5H~rJCCL9b# zq&9+zt4X>xRJJnJw%8lKFq&z3)PKHBw>mHeax^+S7_G@I`z>6F9p))#mvj{jtb^@D zJ~=vjwyK1H@Im`I-1I2obJ%Me_^K7HyGQRdfp)YhpHSGGn7U(=kG~`ed1-;k-6QLB z(l}_pvwEjqiD|!=)R}$7fqxitKw!8_&nR;-i_%f)z|2Jw7ZnRGRcf_!qeqDjT~XgG zRCcW18u={aYa;KGrdcy^MVym~RKD4?Y*AgLZe=?H>qda1HdWkq-=_Q1fjqaw`PQ)l z{A_)1t+?@D1EN#1>D+8484?UqTnmarESGgG`WNRQqXX&s*F;0c}h2`FwYl9l>;X<7c9-WpR zFY19?8xy+A#5-nk_aBO=q&E@3xthP4=KR=o$XO3T)A4Zp@?RP?EeTL1C|#{061L0) zf6$+ai=&X#sqIZ@QWlzxhHrX5ol{ zg>bU`%ey}rlhw^=N+9tNd;oKjI2xM?nXDzTEY@J(JZW*-;KG8M9+?7z7VL~0YeoEg zA2NhvguY41kNs;Pfrx8a(PW23`>G91f6r>Dk4zVCn`kLY?u50ZYugtbXQZDJo2* zMNZ`y+`hm59C-{0->b7&hpe1H*}}qC*|AtNEy`Mh6-%F)WtqtgvD5ecF0YG`pm_~^ z+tCmGN#Q*XEKSx@GV|4Cu`kK1(Yc+@qko7@xXokD)CXeFGP^{H4rLC^YjUb&Qcb4B zs>H6%id!D7MnKndHglz$=^QSYD zTjYmv6r8SEw^D5V4T$Y8@Bx?Bfe(y8_z(zu&{(4@k;66#{+HASFtw%-o+rGIBTNr= zJu~GHm($Y0Vv24901jjXahT&QG)yo4-k6lWx8XFB@tnfTFp0g;koFs6PFI%4EWsfo zoR8twy@_nZw0K(GKDsg!9I5S1bFawH_ys7ZyMd6BWPsy&7>QyO2YCdPB}q95_UF11+IazQL<&+dWL!k_V>PWiSwDw z-`Y_v0|4REGPUl#bS^Jnq=PU34o7UYWYy{Jc_f1+q=*>j>U+yOpWo^kLkY zz2&;WSEB!$as8&h*4;LVJ8FUI6F6bdPxqGsJoo0yfPpl?^Gy3ov2PGgGL1EUP#C?oDhhaCcb+u0jh`|HlV4SqQ zms!q0)mny*;jwCCI#1I}Yk`EHXqT9+E$$iW@V)$7*)FN+hibis_k+;2bK=G0tp1bU zph!YE&u+yFpuc2{D^Qi+ln^k!)PLYbU-EkKk0Zpu;+QQT7$f5pWZ%~d{kCDZ3Qw&m ziZGT5Jab;8O~1v&U&MD*H=mO|L+J%)`Vq&s!;3b65oIyA2$3(MfuAHUlXO{e-0U!d z*h))c86>o8W6H>ulm6s+AHCqNVW7|7EpT1Mcr>I%2y{cIq0i!LNGqp?0&;PmM;BJgE z!E|$LTzyQtwLx8GX+oQHxt-0(6{S(|5D{0I>$)EJo){pXk;8gt22F^>)SNr)9iwgDjtayJY>d zI+Of(I+6u4VhUepzjmzUJ953)@5N5VP7P_&PUYTn-Y7iTJ(-{7ZcT43SrS0xpK`jg zy3)E*eDPcBO&Z;l7e^PXjGx*s-C z@2#MNS#Q}Y2L6Wn2EdZCy_Hv~)rkaX>^S*YcE)7}ZKhyG6+p6z{_gre090+hCJ)=2 zV`Ez|uy_hFlM{JYs2Tp%oi>6Gb1KJ6MVckQ=q9i3krs*>?3}Y{}sK; zMBr?GD5V*@e$FKP=D3CXpYnuBO~d#BOp(1`}X)7 zenyFwgS8i}je>1QfQjH`dQdcT-(OoTlLhQE7ztKU3mtUK z$;NK(VYSr8ZDIq=Sm{1;f&k8bM6~EH?H&e^sg#PlQx6<*p0V3L&628$OBRD1a4AdQ0H&w9z#rN#%VTPpfYN_x)4V z4D()?SoWiY>ucsI3JK3(;)kLb{T@J##2QveV4xMEE){l4@40ggb{J0YgUA$Z;TKKe z%g)GKdeifhYwdMUW-OW34`L3LK1TIkB?Cft17lcOMtURsZt}#d|3*8Bw0U9HSZ6wM z?RXtrA|t>FS~nvAb39ZgGhfy-!VA@c#agpkk1L9OgvkM>dHT-Ll;yxu2RHJ|gnL8X zL|Ab*PKKTZQnA3|#W{o^AG^)Nvn=2~OakcvEQXIs*oh$Q@1YW*|ynrZrpzF=zdr~U`1qP zWPUm47?{GDovb<3rOUg-K}ruSUM ztn6x-!?dTD^b4EjNOp6$&iEC$nsAN@_(ztaX~94B`Frl9(^YGp_fUEX2+MoDo8{J**xr9e;G44U83wNyoTpG{-vnO06Am0 zw&-uolPg}iB{7Q*Ngn*^$Tp{m`@Im*Eg``eJu3eV4E}+BBEb~aEJamoN9Y*vOLx?S z0ps({Z#3;U5+l#pu{<8sLd(5L!GD~&E+$r2stj% z`3c5Z1+0%EvJ!n>H0cL=DGAtJ%ARlS9Ts;y5sHX+B0p(Wdhb zG%vAj9J{6E@QhPJILz=xsv`wV(dn9w7on-)GBBJUJ>y8_&3gJ7`jqp>v>w$bJ{u59 zTTrWBUAP`Ln-Hh!Zo&=}qa(h?p&8fLxnoO`G@%npJHLulAI42F)T?c!Ho|3XQk#yI@ruCWl(8 zg5lsfTijGgW(Zr!Uy54zI@tnd411!M(STvd3Iy`!4-!E#LmfvhcyM>HLqrgDBnjs| ztK1lF#jb0sKGoTUGd8#cW|pB!c^&P?GL8qX#@8{&nnBXabNCYFP zzE`FC9w~AE0ShOMnIwcyS9ln+$=OiHO-wxp%eb*wLEGKvYVPkipxp?+5n4B}zrNjl z%I4AhDa)yK#Bo+#$TNTormnoI)sLI)wjii8vAH3rHxbZ}Fn;O!K`UXlG7gMnte1Vt z=wK@@sE~jsM%vUZ^1+@9Moq$gBt}TuY=*xW?MNj1VaR@-nUXxS5+d``W+ae@zWHAG zZGVqepbyCj7H_iIf40AW)zFNU!+c=VwfZFTtUrVd3p0%Dv{P<(3(aaud&mqSnYsXE-{VaLTc=r zNEou>sZv@7chdAc=|aFK|FwSdbK4ug;~+#hkcm$tuX2^s9$rW6X3}-`4di5T(Hko- z8`~(~rmrZ9xXg8gY!Wxo9-3kby4)Y zl480P)C|H_zH!}1GVE51be3Z#G3iIFtz(4JNmWCtP8fsB~c)jgDn0DO>pnF$(LP%O+8^$X-B>#Y1Yj3jqZ&WWHF! zKI{!}X&zrFaq~WCI6`c#iEqdlw3HFv+^lkv6Nuzun7NAP@B(roaBlU7i2B6TY`i@` zoy+iQHKgoY@KJYcWou5#DkiW;QcC3bf+|MF@{wb~Poc*}o2QRE9 z*QVFV@F{X>b})Dr9Mb3JKBo9-ad%QUt zKbKyK55h;l(~1xLQF^GlbUkVusvPR(*zxFG(7tg#{4NLU^&e{EbMZfT{5gM&dYIhU z{9R~RpAC94@F(J9>g#)*SDpA{{kY!kuB+$Ko#S>0+_S>N{bH}~)E=2GtL@&m7eF;$+a7@rRY{(_gVJWO`I z?hg*TflC8hljunXf%5`)-e*bTN#jY3CH3#Mex`@OztBHw$$KQa@;!QfmWP-lbHA^K zz9O54fWHbu=W^RfJ7z9g*@H_&#Wd(C_} zo!**ZkDa(e&Mmc^H@~O*_$IGjGwpw)%bi zpGV5Xv^dNAwYr);tS;)a_b&RFz8{@2FV3>=@atx6TRzM0Y~@(lWqmuIFIGw`15f*3 z`CFY|eBIpmJN}GUj=uPJ=x^-4zx3W~t_(HkZq5{_!+J4@ZD2Vl_DKqRpXj!w$kkOl z2xExbB#k$$#7;&$b!$GoLdb0-iG_XQ2B&TtMW69vu8M&M)G6g{86Pfh4Ia0e$e@~1 zf5}u|cg^@gmt$;#GAxq09|KSt*zUqCl<~1BjtR572Az?&PoU`7v_5t&SEz;8QM)_H zcgf6!7!iblVecbkkK^o^8{U#tM2cc>B)u>h1Sg64yHGMZ4$?KLYQy~wK`V>>y|m#J z)$nBnts~@M#TCo^q|_Y+K#%K}->6w}^@eltHqd0iZe>Tw*l9A0=A*DH!Br2k5%>#A31$f05jiX6PFcY!Pi=xmI#aPPNIu-B zNXVUyM8&oySmW%mZH?U$z9_4UO?BnNUO9;a21+b=Ke(E5)s@Rwa$PmtzHU_t#?vwznd9A+M51Ca-#_LyHZC(0p)(CWnY(M-Crc)xD!{Ilw*du%~QaoQPq)^ zAS(QoX*ZHnaq=#QXW@WPBlbFD5@JB7Y^KfG1gO6*)oh59vMFR%$i*1sG|ZX`8i6nb zcj)UDHv*fFt+NYZCL2D@(+g<@NyzhK^{AI(+6HWWk1C9%hi#M~a#DjlziGve+4yQc zE#6zPw2V$?36mzai7I7iwa8g1N}j4lHyVKJGz_tT?KA`#hwpsq>WEge!=Q67=6yyC z>$B*p@};X=RXt z4@A`_=dnsf;PEnm$Fay%d|`WwTPm<@#5Y7Z`<>b#f2Xc2?khM&%k5Q-`oSKNh~=BG zpsDw>&si?Y!KyBikyDX~eOhj#)cF;rfQ+eZtV5AEgflvmD+qy>0|f(SD$VFsKx+I0 zZdBynF9~Qzv%35{WTUd7J5p8rGe!^JU4L3fW66noRJ+yZ2a@WUR2K0Z^6D`Ph#RfKv}E*PB#$0VYYRji{~@R&_Z zA-;8d+d9iC#={w*#CH0#VRjes!M$S{h7V~AwuxZ?JBZYIv)Qaf70Yt=5qKQ{EHt=8 zbIW^sOYp>9LPU1jZR}i;e#FXpaXdxX`iw?d%#NnOl`t|Y5;PormTMsJl7+!@Y3Zm{ zXcZe;+}L$L2Wi<7ViFtnlF{g>Z>sY}GQ)ItH^#fu2|aTI1_xLKJDbjy`mf_q_v)@{ z>m?)6Zg$t61QT$Qvlxng(FYb87wOOH#GGBjz!<19{d>J#DgHDrKYo6z^2U_c_r4Wga{dC4K-4U2L zQw;%!x63+SgbEC}EYHcd*XB92leU5n2i?h1R)3yzS4q2l!DiYjKh7`F67Yhhz{2wR zsrod(M?2Vj=J!Zl%HHB~Nw*J{+L_q0MnpqqAdBJrA=R7Vn$x{xFYvrUlx@{W zKV`#d<+cn*8D%+9v0igpc86|)0;BVb(gX|9?BfAM6mS{Xh~8+ZffVp2*giIG*}@%- z-qu^nj~Y7#o??nK_ggiQ0?e5YYT&Kchl03;m`v+g+p6cTIlijX?f%&wN-$e=q#^*A zqM`^?UWqFiLGoi_M_1^QuiZ_9?N$VV*AyHj zhXG;%^EjPA)}(i?Z(dKe6?7Fwbf%Ch$BAV=5;$0xv-lL#CKO!`w9a8BRt;MyJ4-_! zJ}Q!`v&fwRhZscF{JOTV&2l|<;60|-SMIP!V7?R_%{oYFqT>lbW9l-6kV8EWfRTD# z4~4yXQ)*7PyqZgNNDd3r`KNd*$a!mH$YcY4@Pg@_Dihr@==Zf$CrRnEV_w7xO>WPS zTxaTEN1*$z#9uCZyCSPj6L=Vu2d+l(AlDG8cymFcIaA%}7nvpGV}KO-%mpc^-g$d2 zr6>xX35S-}tkgEJt$-Iv4#H)K!n-ieh#)sMl^Db=Y<6o()b1s9Pnhr(m(TJ<1@ZDc zKsuJrV_hiYL{yvt;VCa}HF0708;0T~2JM=*EGZn|hNi`kkJ_AaPH*~9jRfW&RvcBr z7eMRBk_1lDVl6OV=pq68spS~liG%FF_EvCFdM6NJe=^-tuN@rTC5%x*PXLq!Zg>6#yqX&T|U&7#+^y zV(f+L_20r&2q>`6#D;FeFwJllvP1w8Z(K%>0wxJ_60ct5vj&dsnCXaM%L|3TUhS8> zha83mpv+b1j!HTC%c*|zrFohdo`V?5u$Gg{&9k8qf%yE2o`)S;8O-*4P!gDNpg0a` zjFt7eH_j1MH|2}{)3J3m@?>0$_JnmuOG821)W-ss9YrQVc(tX?I6E0}TC)TB&(bMI++4efBBC!#+y!2B2GXob&?Bnc}yZ zawX0>#TMg27s?=n2KF#EG^iytJFXW!m>-9NMJA&+p73(XLGetWYWWKpTV?Evg0sp;sHS znpF&buuELF?wp=L^jL-|&ihX)QWW7nBh)HAwYd3D={;|O;w`^GgTX(ltCFz8VZ?z4 zEcLA?#wIZ4+bo2VB<|9tJ(haQSiodZ9@~bYN8>6+@jhP)>9X_`BJWPM5M|ru0-9K0 zu#4N#>gF)iiu@06Q3o#3zvoJrD83W*12nI`Ga~o0nRWYewaDb>p!7D& zsT%~1x-g?KkZFxgoQv_|)oO&hDuKpgt0rYCoX1>7?b=FCxb?#x3;?Fx!!#y$N)nDOK0MOlYc zQAgTZ^{2Qf4?>DdSreq_MaPJ4*dI&AyIQ;AD%fGg)fz1`LWlLn zA&Rn0KG5K!#fFuLDmaPm*biY6r`Ya&f6WUjV}jA@pq9zY6|3+?(@GPnI^qVoTF)cI zrAHyz66fFqx#PqK)dV&jc!x5NR`3&GX@&vIt_q@*rA4H5068axgB#(rnd;Y-QYcc^_8H?(}qTnZm|&+8X57ZR7brK`UGcf^@@ z_5X3_dFQ7r-$yoYlOsL-b#H~|P-Jm+>F9tm!CPiC$b4D%jfEkcWx!rE$T0Cd`>8=J zkvN)+EgP*(ZiV7r>}<%G2o}oPIF?E9u4XD5^vw@D*%2*THp2g@4(x;5>z)vU0$sXZ zLyCuXutOu*k4kL&AYK0B7LQenos0v{+`7 z7#VF$Wx1p+-ERH-ts)d*U3yR}LESK=}BU*te*X8BsamiN7>@Y3kgaw+aoV^aOI{8W9l9=+cI z%H`AnGk$GXPdT=9sXv%)Qk;H zylks%FNZhzpTwh!@A{*WBka&xXna&U^qyPzH+{NKzBiS<&hc3?cPMo-{d}K_+s8vM zLe&g98+kq!PZK6~%$&>YPxJkg$e98CH)~{da=rZD($>i*F|z(i?eR@^TZ%ttFCSvI z2a`XSzrCN)0CW9sfr=;V$MN&;-QNS_H|c}C&*%gHW4|fiL~izPzw5(S>~`-F_!#j~ z@saG&{K?$$t$gm3|K?3$f3SYoK4;g%k8WdquzW16B^#HI;$d*Jpy6U;cP2f>d@;U% zono(_G{&3Do83&;nDXPtnu#~4i|J$g+ASsf`ZO_lty(y;@NhA_i6>g% zcIA1pIg>pMj-@6dXL|qQ6I#!hiN(c7#PnqOv%kGxPxhG4q<-zWjJQ%SrM+0+1{tOH zC4AD|oS%BQ%*8NQVE`M+R;E&~B!#~46?OlJd`}EpD_pKetkL=xf zG20n`-MVXjh4xK+ZSM)>h~A0dU3ihbJXq`YwI8J$%8RRh5q-yt=0%|%4tpo6FYDRy z?!Cc{WF6(Z_;Njyx|!_Wp~=1Y@;FP}o|Jxh?CEM+!9K%1|5|(rzJcCJmM%}+vwXiybyco25}h+d21Z|CkxxEV56e{YLWq0b%cgJ-zc{5i#WwT9tfzYy`7!?EHmm=Vb_V=3dKckMA5S*^ zEz?cy)lRej!ngJ5*0a4(ZHqsWSId{oNBwj0Ri>N&2Y2Dj^QsX0KR+vPIv1`-gVV{D zmw$9ExUk)GZY|XaT=Qs9Tgz$|(9Lf`G%Rx~YtP;n?$!BC-&`+t7p=~GhkB-*ywP$Ai zmhbf@vuVlA^X=Q+pLHh9CcA0Dwc%#S%)T1u(`MFtfsJWxY1y-D+5Oi)m3Ie^tplT{ z@j1F%gy+uh3|24MP!c>ZOzVfve236 z*#Jn!2@@>F54V9Wa)i+#NJa?BYWeL2`KjT^5RmjUxfL3uYfeM?FkX|Dh#21 zqTEgJQ7oIhk|xTXKEq)XXsPYLm_VJ~0Dh9<>mhZL?Jm-NE*pwdRQabGwix|D!m7A?VkEXF4xC??c;IHXV3FewRF&gZ1r{d;@WlH%P&w*fvvM{q83j+zl zHy|j%bFaw|>dd1-fVvuD#G$gll~{tJ^ohFzBLkLF2i+<(el@f282~*PL7ncN%KB6mV))8-swQ(I(RT$ zR#IliQ+?nB*MBJ>gqy44{)7dB+ z$84KsU^saGX^+IfJ@Y-5i&URTVel}GQc^&fFQpRkp1!i+lD-41_wkAEeJ0HO$x?$w zsOm?a2GznPt3u;2$xb=6g*X7%GMog=7|PRUl5;p43^*j+A(r9Cts@8_XXSu~m+!tH z4l<`S50pnwa0(f7Zm=agMHBz-HUss6#RM%H?Lbf=*~(QW2k_g5NV=0EnRkh=sE109 zLsDl66m;>7+Fpn^-1{H5cVw?oA zj$85%V`uM8F-@lS+Q2K&9F)*(IfrZ?Y63M%-ewU=}T~ATkN(OvEwE z@Dc#S;FA2h-~~g?K{kw(7*QEyM7>4FU*R%D4}m2$f;=L8;0|_~E`^=PKG06Sk;gba zQJFB&l_Iv=4!;1dz|;r|HrOAai;9^8h2Z3{34mWc zC}T|h@ERO87kkN&{0y}8al67+%;2;2ZLsEk!3 z;P0mys_5<%OIq7nxTI~u@DCdS+kk;d1p6QF<2u3l(Rw+_3BG-LyVa`i2r3Si(dWB@ zVhwE06Qre&eF#b!MUvyWiZYcXkL&QMt1vK4O&Y+3PD1}$lrbb?@ll4*f&kIIEjjl_ zt2PW6xZwAKLJqIkbmTm;KLU-l#@5$iPzsf zKk9X~ntBBa*KJ@*7(}dMBe_J{{{HI|C85$^!Tw(xHEUN%IZ5A5j^Bp+3R(;TIRd}6w+ zS*f1T=L(DhnQB*HpGPMINX|SdP~Z%zW42^7m}nzaVeC+Atn&*@#de*H7oePkFm_2lIh_a?!4(M9 zQZ(}vF#u>iSuvtFIwm*Bm7wE8`(&yTH|(v}o088BXARpTENj2AZM|@awxyKwsDfaJ-Lgdgne^7 z=6F<~@j64C_dq&Uohp@EzhZmSh7))3zay6&`wZ?%GVzAy+)7Gm>9DjZo=Hs%3=C#_ zC`P)vtE^bVcr0yt4Af)N=g1e8XJtC?!ZMJ`N_E9QiHmD6LHXZq=kMEhR$iKl1G371*AEF zB7c*i%c)8qY}6%JDNujFGrWe&u?<7sLNNh^JLKL0vX5ZFAxX^t_3zdjX-Kec3QAeh z4`%h3;5Dm)J&G7d2^b7LfB?n#fyBeRAr;n=(19t48NV9|3LZ;xlv)A-w!U`tlX$56+O49CLe z5EIoJxksAX2w}8Q5UjXUT zrp>se{JtB0mJd-MxXKlv9XMtN9N9qTnz3cYK3;sRN&8fWLotiENj3y4CV}bLV+gmX zL?itvqNlX`#m1JXq|mM(GZqC7)Z0S2u_fgPpN17X0bI}K2ynuHCrXnY)*T;)Cxlwx z0TgeXu>|2aMuUgF*3Ho<_#FF>(CA?KHcw9lD6pFS$K;L78eJxyi_$Ij%fvon5UC&o zOW~h_l7xN6z);?qpbRFQvRw`(=!DVEZ{!xi5dZEl5OT;bb8Y!jSmBNtffIu}!6L2+ z0^9@R8Nw5Wk=Ce@biqt6$D%E`P*eK&P_U%9q^FH5Bak?HAR(H!!V*N(FIZeg5Wkhz zPtN@|6>4*mBW7dv*l-G5;$xj9p}7$Cg_+t7ZdG!J*~p#AJUGxd3K|SN zyg+dEKL_Cz?-V=`%cdf~MTql)U%<>ToWsPd0~mjwr62gYIu*Rnvh;h_#2{Bh4!a}} zHG30+gd8Ts1NZInC%C2V{$^JEz;5}69ihs>gu*K$-iDPFE-X_o>U+xGlr$5lCocBVD)+gElk(^2eYkHVn-ZN}(hV zGWt1@Y@ts0p$f6+l2fN2cF-IOU8}+ZnJ)1b&Ty{h>jmd9gXDub93N zu~~yCvU^L#hCMZ?qm5BFCovJS2dXTaSdf)$S!0i({W`cNLXR?d!QnX8Q?EIYg7Pys zU%52~C$zXx=7-&8c9_MaI;G83q0{tK?`lvTI%vgH-@_qOqQ9*Oupl7B_TdEG@T~n} znf@s)6mERjD~zmCp6|#uHb$()Mc`~&A1$UQQ=ktg!u z4dmu=Z=mmu7+fMEIkFLm7Qka={NxErrK;piNs0FH4Fk*l+UQgCQ(UZS*@%jRqR&(- z)D4Mb}2)HozH1Dj88UcLjsEL z2w1SAxynv6GPsLK-GB&HDH{4%*&775ZgtzDFb;f@Hznr3Y*Old9pO{mogHhUMTf`)3h#J4s9+CEOmH?O*u*y?Bd5n9!YY%A zUrlUq^*~N<#Z{&% zfrrC1Cpm%R>_8yav;IH@k}S13ZdO%_C(emD7ho3*3z!c}a6_)o5QIX0gsyfG6^YGo z-bZ9bj7HlsG4F@Kp~W5C1&)y6r90F(m5@u3zaVMgp&^bvPmV3|IdN&g7zD+z*PuZ# zyfK%cmO9_jiSvlLbDPTinMdqIVtYfVp&fBATc8>;Gq_K12KzmDoFru}oA*?0Io&Q7 zgez{k7eWc7y{ncly@5ws*5&(I>@N%I7-U@RU|gDLeT|K4@{C!4R6>?0Q5;!<*w)o9 za=lVb(-T1{A_}%mD}cv%E&?u`RB=f|Jr&`yM4**26V(W@NWl1zIpWNp3y+k-2i)E_ z?~|-vQg2e>2o+09`}zX34(poNd&>H(Xk^_G9AmWi0?vi#h=DMA|L{>Y$mzv zrGeS8dbbV3I6O65t!y(DrILKu08B>CarL+p#9gt-?4}8}L{EYB_xy=Kh2&~pGccI7 zBdm*Q(=!5%ENtP3jDxLo4EVBFJi!^Q7O+cjCIE5hht@EiU-G=#Uct1M-_-kLq*Xrf= zl(^if?)-Q0Z|+}qY5uexDqjt+;zyk`nKO~IyoEreCKeSdny;?=rwMUt`|7SHC$poW z30-Q@M9|&*{rF+}XnZt!RDHTUdONgu6zBi%wTL~cO|PZX*Yopyt30|o3QvQl5l;(G z!%p|7_R)TAawxqO+Z!HeA28eN83+_AVNkH3`b_&F`Vst|{(N|Iev|sX{?y{j!5?Ek z_Mi04|DbuXyKdf8ZeMH%ce-d-u_Ibd`QZIca3^2>TUT$8H*0NpF{^EV` zJzo6v=wfG{Fqtzuw0;YaWt`1?Cif(Mvwc~JiV zOJY;epQ{7cm%_`u2?18bcitb(p2n~9wX&`F`?q&8ERJ@g%~OY=oulzc_l^3-dGC67 zFvgt-o%oQrnCMLmPCP%RQNsCK0ELIvr}3$Cu-rQeeSpq}exb+F8l!nd{~e=;)rS8X zGw^#b{3t(7Jh(p$cktlCZ-;mrTmTPc{?zY_!L~-6V zjIQiJ>86L*Mtu45B~PhN7h(c64`*c@BmwmO)?esfdfLsf6k=4ktB7B3p864`_ z@TE>hyGW6AcC;m3?u5Sy>L`UNA31RD5m08U+GaOcx)Q6B2eVd}E+iP%8z!`*DMk9g zkn}s~fr1(;0tN0>uM=LP@#q3GVNFeej`zJ1`8nl zL0`d%gqCvXKLMzuM?aeNN7^x{x#rII%#VP z4!OELNKdyQD#1lWGCv&OJ9H1pGeI61SiruTQeU>*0y%G4q8#s_J z-xu1oFFs0Q^G!llv(FF+K<0MjUp$v$3f#qs?dnz`qs2%(yd5hQ+1~jFxg1go#qjQh zTt_n#*6*K<@-jA#P266>)}v-y@DS?Oj=KsbBkQDo6cWuC?42`KYJdZKV1g5Aqn1#T z{Ec-AEpyyVaI4(r1p>tw_K}O=St!Ammg7-7#xHmglE=7C>qHKc@kz zstrOFeN9dsS}DlM!o>rPgV^#==8ZWSZ3)KDbL6&=fh)=WVZyTyDEbdLM^z8Q2X7e^4bo`%#0a#GlGpN3<)*PZZM!+E~D@|oIK17 z*#tF1cNZfWU7!>o4r7gr=;)RyX#&XwC+ct#qk$>}U9tsLa)#7)ZW3icr07Ic5RJax zsmi~}YQ2qKrz8(TY|z#3(j~DrYC2hIl55p$A^E)m6QxK4>(Yz*qX$Co>eCX$^`~rCZM^?@=3U4U*Ud(KtPv7 zp5Y|a`mi~K-BM(YL@#PVX(*a02+hBeKC_3@&^OyYO`QfG*jz%Vc_M?Z8r20V_aJ94 z#MpCb1pT51Gn)V82}6=*1!X{bcPQ~2k&|lITFC4O#t+iLVIuu7CPkSL>wI)s20P3NA|KL zJuJI9OoI8}5uDgVC3|CxgM9~t2!Em_LEnjTg+AtW!RWop&W$-y?uBg|JOCmb>v(1A zcw`eH39>9%dQzon=+g@Naeps*WS?P`0TKK@=1hb;7`WrK5Xsj=6G{X@3t?u}6w1n+ z+~jbk4DnGt0i$U3poffrkb|p)*@(Tvd42mJAPAoQ7~(8t^=n5to@t5MlHS zlh!=IbU-OQL-t9Eq&f1yW{a|g7*EZrKtsIg5OgV_40|HL>P=y(%K(D1lt-=5W7B7( zxLUs>C>^)dhTI)5%B}^MInmA-2POH#4@&_KTNmveLe7qwF%6-L40bhRSxWIR>-!Ny z25Iig#?&7PAqX>Q?~Y6ps^4O4k|k3zik?Qq-n!%;lC3bX0>~$95|LWR;3ULsCP}DR zawJEqdQWZ01PXs{OKJW_0fHrRRC$2NkSMjUyAqGQr0Y;e3mKZC6Um)GzFn`=mopiE zqq?alpk~oc385+VaJ1b!(#(FnGdU$gqu0#v)Yvei5M}Suk(Zzpcdpv;? zfZ}&ZM$-1M&8cYAwhBZf>~4INxO<*r1)H&e7LbcjX{xdWz65E&W;jU1 z+k=6Y36k$at7AQuGLOE!@7L$+%TLQ|Ry*hXrT9|#X?oPWs=pP!Migq;lyAK^o1=jV zUut0LSZekteRO^*J*plJpDqnbJj${tWKsFM@+126TAJRL&)0k3N#aR;^cple)L#gx zUIofE%^}nxsQuo*Pwls99~S-;^EZ5F{FCj8?i_EW&0YSX+|~Nc+4YwO^Tx_cjWknx zlb&&(g!T+i#s`t(!YQpXMrT%E$}gjLn~tO&=3I&RNbdLi(wY)I*<9t?ZE)q>%HE`W zQhiyz3>Z>h$+p-Zwf)i|)^FQi$(Z(C`o8=Sek6V*`9ASo8OP>; zKmD`9r2-9Nb_DCFpJqpki|g6K!h^JVnvgWV=uUJGo-dadjTc2$M4Tw^{0@38t)8Zj z>)qM#^l&G7boBRd*|0wG?e≪t8=i(I}Dl@4-nSL==uD1icH|Kcp8@A5tHJAC>P{ zD(1t01N##Cllf!vef9Qn^7XL&#xJ}RQWHiO=9m3pcYVHD+KzrQGo72$=}pJ>;(lx7 z!2K#vhXMnd9n3%L$K>t!{CPRP&TX#TINhiZmIsLgO9BKO*lX64|7LQ0KPAolqwjqK zggxqb?Tr>L?8Je?C!)A?OX$c^L=j0;nYriidaM9HxpMG|!N;dJ$j&b3Tt*qknqz2| zl~gelxH@JQOz`{lVS@-JOvCzghp|Hh;n*8sgt1#69=_upn#J1e!Zw8Mu>uW_$;Q~k zB^qP!Y3r-Hy1ULMlwERk_I&>!MIWliU)TJpsrggeQ_JUWay6DK4u}6{{8{h8=f{~B z2|uEb|2=%WIJkrPy->&cS-CeksE^skS;xk}or8CZ?w?hy3C76Z?H){PI-$ z$dAdr?N95*_-v$4@hr1*6OMQD;k9G_;m6yw;ofueR#@M2-9sMU!({^NYjk>bIx%-w zs9xM(h%c7U=ksveT9hAOgI6QMkJ^nPD}FFvbNUb5-}CkZoE?4!KL)?cqwQe#Uw7}- zG>%_AyWeYP_jGvvfARI!L2b3qAL!e+Z%Ye>mQtX_Da9qlTeQVJxJz*f?oM0Wi+gZ) z2^In^?iNBI5GYP?OK`Z{xik0o-}lU%nSEx@oH@_gGiP^pKf52XvrtKs1K8|!{rr6KCb0kE-G4e7{!+0O`h)fX-ShjO|Nr=X-D`~IC%khCHd#eH->4<8sWMQo zUE2nq7?2jOPl+C<8x$ARW6+fhMU^t4l_b4jN*@YSf;6%U|NO-un*1{ysy9hGlw8M$ zhry>Mcb$C~-u<}Z2u69jPEv_QK!H9jqNDJKY01@={`p0v7fn~n_|E#DOH!;SY^8V{ zFVw56qzy_TmdZuWC7je?H~f>7Zhj5Uzwrx<>RhSph8K1B9N+0C9tg7Lagn@%JU2c= z1p7YIg<<~XeQh%|IWg4JQ#wG@n<{3~QsnG9=WHt8PUti0jSSzw}t>`2lSiXneh5GUOoPw_4p2X~LSSMMkV@= zpMn>2E^@4QMtaD1MzAE;?rkjF#7??Xt&XY-q#HGuLX_T8*&BjlcI~!xn0iR~n=xf` zs0GjMixDz<>rURyBBk0&?bL+|-I?e?G>^4rO1ObOq8nw@PM??$bs z{z;%T*k#DYb1_R}=CD<4^t9lmoT1@m(=2oOoBe}8L)qD#gFz1q<(WF8>gJp7X1l|% zDH3?XmOlpZe1G($=G~Ct#be z{sCpb{Y`$)CV@fE^qf1iXT8;%9~+U?d2Er*J$MN9W^y>si%oBWkNF41lKyEKJbtI7%U)SN%4j z3l&R9&tud0;}0i2fyQiEXg>D4VDP?q6@v&PY(;W4k8VDbclB4E<&wZe^ttfr7`sd^ ziSL@W?wDY`Tb(vpdo04A`SICv=kwnhhk@0qoT`VVEzB_SYIweTTS{YCrVr`DQ4 ze*PXlA^Q8&Sa?Y4PN{8vYMrWh)#X2@dvP;8XQ#T|hQ@3eseSGv1XpGe#HpVD;yZ=* zBO0WQ$hAr?N=7;n@U2;l35L{ZE0q1^7uwZO1;9AZ{iKmvFsCOqbM-G z^Pgi>D?iNU3l-u^I3*RS16B!kC^0AcXPm)_!px(Kdo?+$2Mdodl2P-(tH6s>6KT&P z)>55uM;i^Obc|#4Gdx!l89>fntEY#a#T+t^mqY^%^*il+&o3Zu<$C-A)h8YlqW_5; zj|dVlF28M$RjkcmtZW!Fj}CD1e3`_4`sZKjhP7SOZN~gHhUuLw53r!=7HUJmw)z{{ zDPLg*))?xJ&I8#%$cvTLh7AdVQtlh3#pq8atBY{+dTo z*_8y%Xz^9XzoI^xn#h&$JdmFX9FO{X#@Le6?$eiUSDI7tWRe|-#(i^GEEC;(@N5B6 zk+4f{9v4 z<{s$649y_nf{^1{3vnQ{w>BW$j-*pl&F_`;LaBO1D)3PCUk)CKhTOum3cYeNNC)aZE1wAnK!S;mdNy5eC6DkY)?ur65g&+DuP($|A z>Y0uBZJdfJ2D4cuHYdgsZ_!1Tz&D%Iv9S$KCyN#R6Z<#&1cTkJ>o=L|`U zhx#rcbI3Pu1n``0odc&!|iujwPg^A?*+O%!q`>MdP(2!&kI;D5iEep>1 zg8;))VS=dcd4QGZ%%oK>-kC_2*p^~B|D4!XFk|?(W^cp%^l!T_N`mOs{Gat%Ro5!z zh8ubvg<|Cb>G+kOU*;g?w`ypKOWoDd{OSc-#`8EH)i&h_(fdA1y1Q@y!@8TBu>PX9 z1eRGi#jLDd`QRDE)0=a8j&D-?o|h86+TOm;9eCZ&VY;>Ur^td<{iSi?i#4p}> z*cZXHlRl2dOkVG=Z=z?&IWbNtFD#7}pC)eFXeEfTa&NU;ZDgu7uryokj_;1Gdl-DI z(NuyPyL+6bnJTrM{&lLR$Aee<5(*2J<^CNz>?_lhwwTC{RveL5t1r+%UQKenv(g?$ zM$1G}Oxrp+TGq39p_Ko~j-BbI>~8SH)6kr6Y7J9<76W&2rRp@HS4@RR8H$A$>IUvc z4yfV$QK9@|Lgcsf(#E(Z^TKb(TYBh4B7QJ5cf`8kl>_z^a< zfCPa~!8uHheW+=+17mx;hx9u_cy_!@sG8^p3R735uWog3lNg3$#9`?T*{bm*(~|n- z?@BVNNmZwR1^Ey3f~4*83U8$4Bi^Xk@3X)H*gOk$0U|7`a01W7F5S}JhRG)HgmJmX z7lD9EQZ+(-8a!zl`s3G!b5qW*k0M&_U=uBR8qnty8SYMZaFbv~GG z+_1^tsJ@7s`wd|&f7>-aD3GWBbMI}aUWNh5wY>3aVQlOo#o-f6>#1E>)H5A39lKie zcA*2;SoBjm!-7u`ixY}=?T@v7R?%;Sy|j8~?LD@|`Vf3UFFHKlAus=#tg%I}x**D? zvz9i7{`>+8?GELubC~F48kT@uEhApH7n^Bnv-k7Yz*x|WtNO1?Qa?0lR+1WIP8Aod zmx?gjBr9wzYTbdRlO!?toFZYNV{M|2ycayrARXU>7H0(AwbDxL{Xr>6yyTYa3S7-v7H{uvOa*>lM3grZqKHZtZlk*uF zLbZ6zU2TY?jTvm+L4ulsUohIQ;B zQH#TtyPjK-_Us4!ue}Te>bv<~xl0kBoo-}&HLN;#xC>oCjqZI#^|>?N7`F%k=F!xJ zYtC8@GDkL&1L+ASnYsscw#zd zK*V-;&osX{nLeOyPu5y9ogOkC~ccPqu(}Lzha22W6I{A`fTDuF| z(l|k-&evaR1Cy8Lzn>D3ML*%zdIsOJUrZnqgK5oKK-en3e@bo3(Lr)^pNu}cmD8?r zR=LtZcG!>YsBSr`VYu_7Y1a@gS(804uiyjQaCZqbIlqg3aEX<2t??+Wu)eSYQRE6) ztS-rTx~3vA&9^FTlxrKf$;U+0RwGE%_DD+9cCA%oAXBqdEVGu}?4A5 zwNeucI{_NvCzF7%y(E}<&XbQ6LraT3--|}t(r*MnDVNKPiJquzLs$22lJUT8_=HC-@=dTW?YUNEO#4g#AgBdO@sBG!epVD_S zeRvS}Rgy2tKaGifvoO)Jw*;&KDp(ut98nH##EagKGUKWiivG>VkW|42fMrnTtqjfz zH#X;wE1qktPuN?l2lNRw^v&g_nHb&8pCHqi&CfYy%}+=E%pHcUj&xD^T9uh!9_P2D zIlh3ko*{~XoJNMj909J|tE#>%oo|ez2w{-7R9EN~9dZ+)o12wG>buKTlIWMT#$r@L zH`qSGmjRx~8qr=HOk_~9Wxb2Bd?b0G=QcUPvK|Xp43~&&nYHQZB=bunF}@&{x{t19 z*OOVNwJ*g)FV=6y{>W~L3+K{`h_?#mzTKmJLD6iDbFKqnG({(Ja$mRsRt z(Z^kx1TRFy&*v7|Z(oipk7Fn5+sLniX6~=&mq!Dk_YaIsqj%o8`uT~88Z7NW7pS={ z$P*XaNq%kCxO_-3CWbn1>ogf(Ts{xY7~wq)2)|y6>!hO2y_7$z-%cwiwh_8g*~E+< zzU*{&=6AE+WX8+=bdhDd`}bu@7yoT}@Jp5D&jH8ZcD^0{IpY()fG*9Q)%s?Y-lqq$ z-X@*7!GLFC8G(Gah$a8_8NW}rS2F=HfC}e1&EY=g_IO4Zuag&h^2E!yBjFI=1d+%qpN; z-aF{qgc*Y&6XxZ)d;gQ`A*T-}i}yXZ4=cHo(`&`+(;>g}+CAEZd#P1W*qnEiw7b*h}Nu)^TfXA;}>Dv0e4eZi1_Etwk?AJ#7E z)oVf%XnJwov-~!&bYt@fb(GUn3mGL6U+VtnpKo|I$EH!D1L(i{wH{*m8|IJnMLFio z-*Exnn&k{(-_op3vl}vb;bOckAi@Ia>Q2+Tp1K@}+E2n5O$u(uVnVtm-gudUriixg zbY2pzW2#e%>DO(-`$FE{VkVu(-+W-2mhH@v3K=BBA_3NRF2X>N|BC76ffZY7%f%~I zG1g-7(gA>uvWDnPIzwgpk~AcO;v7CxZ-nu?>MJvGRG>a9iBV_sBPUWWBb(uE6GLUU z=5Q*VckK=`$l6&tC5DpYekFn`)!g_3cRJhtLiYBG?V;!wN|*<^%C%Vi9Zd$O}r zAp54D!8_bf(!=IpWz|nk69s2u9BP*G{t0mS2|OAmU2cvdZUo1FQb)>x-I}&mSpz{-49M=T~m_v}Yzp_JzyVItgWsT3xrg;k?Uq zkDuuizS9^6j{V$g1>7p^i={J};y(U%O2aXZmfzx8zwb=uX0u^?m6>lybR{8~DXyx6 zl4t?$zCT^UF-vB>L?j7>?rgj%(wl5(Z*Pn<3m2FcdmWFMM7-9yRLr9G^)bh3)ubj{ zGVqr?_N4Vd&}~%dCD(>JS~5wbBgNYa3G+$`77izZ4b1f4nd|T1jbt_XeG=J9hy+ir zS>OI~1*b}rVNbj#t2E{Ph!Gr2LTyc&Y*iU49{D{fTh4QyoS}9pnpXVK6^c*~Qz%$IR#ghkq_9tL-M*qsA>ud&LZmm`#_H%h12xc zS1py@$7iJZ8_exVycG{*C$0PYkcOineZAiZ>hQLh-FPX;AyT3GH~W;kCPV_S)M`>B zZLFe}dDlI4wrAC-L^Py{`ERm5Ihf?or9Z2)KZ2g6nP3Wxlr;0|ZkpqUvW$|CKjWu& z*o^dbBfjQ%s+j{;-9IR}8|5+HGOVsDEYzNDuGJfQZ%~KI zP-#LT?V)Pve9y98`KI#$u1=?~e&+&X#2&r*SVn;@Vtb7hS(+nu!B;6#k&EzU+i^RX zV|9Rk_GyIBf3<7ko1}oi!*_b|RmJ6Mvr%w(uMI)yksOp}`h#hOfA#sV>Dga?9<0yR zf`v=Kx(n>|lEH5ZY?FLoWXe;BDOGy&fl*|@6a5hNBAu$>sZwX9{eY1O@~;%!gijzM zs2=e3Mq+AVpnRkA=DX<+e;l=(tg+sYCIV$pBrN?BXCi5PcRv+( zpe48(=C^*s=8BBl+|RXGofW;-oQ+-d{Sgz=jh9~O2W_xaJ%4RRqj>O2gAz*-ma(8M zuZ`j$FRSMF?iABu>mU26S1*`ngX(@3>VSe87!n2u4!J!^s@-(bsw^myPHO6cdzQV< zZ2|2nWCX32H41ID$+Ts+0BFn)xc*6Wdduj?5WDTLic5;l@%M-k-M|~a?a&JbzgZP! zA|@paDA|UuWjlY|i5$7PRv1)mQEs{*G2`%5d2z#0N;T5gq1RwglgM$N zHB7W?9aTxD>u_3;6|42IPonC)o@aDpPls#Zm_&-e7|-nkJ)$zotv~6r?4#C(Fql@m zkpakRsmfW4lwT$Q?_I8G`B=%<_$F1>=~3-%S7wA;hTy0|T-Rc1-9VN3wk#ASjL7Q~ zm`NznP1_Uf&Q+v-Eap1)@k1Dg;wo|W`Epi+#)8A`16dsmhgkL0dHDPm+e@PIr2B9VPdO&8tjN zk`m*zkv!+q!11($)6&S-M18FZ(&}%GA;nq)N3zJ331#w&#tFsR@kpUO@H&liL zH+_o*Thh;B#Y#>}D(#l^EMl}Pl2wde7kM_3PJM`dqr?eJU*2}hX}<7tXsCz$slPdQPOWaZ&_D8l6^P>b`P@=~@(&6Nnx>OA07&qy!Z8*}wbN z+4!N4Ci_X(2XCmjr8(f#yZx6sH@3)1ov1=~J%Dpxx%w@oe~EjN53cYbAIF$#eq_aT zwH3Zg*tyaCt?kl2LXdO5Dpgw6UA~&W{H)$r__tqZcM8Yme#&Q3VY^z$RJ2Nu#Pqp| z#nql#-=6a_6{CyDqTedy?22)$^d_<2EiiIUElbU73@>QOBf~MCP{cODHlPnjC>hGl zplxFV{UbA(q!=M~n5r`?@^@B~GR=WZ!Sbpjdc4pkUQ+2^th>&-vZqepIO*CTO&zT1 zVXdlcCahVS8BD!~0F!CVolb+x2a+UX$RaRe;aY#PQ!+3vLM+dljuVoKam7L=zHWsaqcujAQJ!22A8FPrXS!d+ zQ|R$qZ5eRrdNf<{M`UntTB$?;tm(HSX@$0YXy`!&n6p3nKEGsC2S6K-643`fGW6&A zgBj9rmgilvGcI%2%cI@e9);PpjjRFLI7F*?UsoF$uH`1kyB$f0V>RC zj&GD6hKkfig;}J(UaGb8l=LNZ&GqWdXbqn+b8FeQ@uOa7l(ZYlupd*YApSO7uV|<` z0&ICM`-mLPpG_#a7Zo`@B26f|F&OGuS`z*2h^OgXl`x{(on+O`#$;1qzfFoL=N4>D z+Te)>rX@<38D!POE+m$v@Eet-JXrvp+BfAGU9PWc{yNZab9OHho1p z>Fjp!rDSv5h^a7q{mu6lzfE>lradQ$!`tWmVHCB_j=vorVeCys^W{q5O1a#Dabqn0 zzKXAQMvhF!WuMK8+t8`Kimt3ibrmcZu?_u+6N&^b$swb-TUyvsN3wVkjr}+{> zuA<#xd_R(~MTB@e^3o&g@-HMY{rhSYzuL2pswX$1(el#L0AwJx%<(?otm)yp z1(n&Phm5oQ<9uKB+p}#?&y>;JmSVAVpR+$az3lsm6nshN7L{goMLuEA7EEbaktsuH z7)pLgq@)eT8dKkMAO}q<%6!BPujD_-=bU5HQwoC6nw@1@z_=-b&X+G$Pp(003xmK3lc=BiL z?^|WmVl7~$|B?;5Ms=D=u)eQey}W9}Cb;sh%ydjE$!sTyY^H>Q&Vzpi{pztJ4oUr? zVK%huY3&rPFt4y2t)`_$FlzK%bL9uD`o9ubImJ!%3&Qcm4EzQY()kFMUyR4CX*o*< zc6ya|hZ7-zKy`L~UeY3cbZKgUOm9{8)dsgyMdtn}VS!V81nfXl%_5A2Ji@nw)%=W_ z!n9#RbZq@C!+I>-HQfiYtFY9nOR2-cALqTQPz?{^BBAJi>X|RY^z!~}NCLbor}r%s zWBf>8H{U-;;DKZsV2mp6Kn1-q$8o)_RjCSU-&}6?uEu5Z4$knCggu=C zxmR2cf1P?fX%)h{(Z1JURKpXX?BjXF$JavN=)WJERxjL?V#8{id9FwyeDnHv9xGSm zIDlhUqa&5mdm&*{*09>Y#O#r$Bj?ghZeed(w$Oai@2h8fiV>}|t}ROOpA+{>L5CVL z?`W&gmQRSV;ZEhCJx}W0H>*CXi*Z*d`aP2djId@DQ!=nsF=rUM=+!MqOGQs9U~H0| zv;59^iW0yBjhk}nXuss-2;oT;f~~6&&Xe4FroCfKTdrzKtE1kT<;`G#25w&8WPn8; z$?Y6ad0Rc(`P*=zDolB)zQ5fC&2j_?oN0862YD7hE&GFgj+CEPuE|%W0nYstq{TO< zSx)L$kBuGM@%{Qj4Vr-zoNhP8cUbC}4Xd?3HV_nWHJpNS_D6#oJ1omMMGH@e+f9M| zGC6iHqBh0_jI{z*H=4<5c`{2|76QPLJw=b2*W;*R1DonA?FB$~=Hd;@hTI07`=X}; zRscNEA&k0YV!}ZWlaxJ%8Z!5IAUB3XiTFtbulTj;4flau zF!OCk*#fC)g&bh*$>FxoSS$UK)K$~s%3u`2m+KpgmK>_u6}FZg66}Wg*kIseLrH4VBfS$*;NkXaJlg*DF*bU99)soYRJPt15Uk*; zv{eJiOnfcop;kMHNDFN{@^=`{?6WO@ieHYjm(v`5e%!#A(}!7*((1KytMu6@D%Bb` zkJDMDRuL~Owncc6*q)ROUAeA!%B2378JnGYpOBkOU)&=!PFYbGJOkM-(NYKUq7%%I8>8PbHH zdR-XKz2PlTXfF!zh1bj+v_6X*YY48uykvKJ*{H5KTU;DF%|=W}yodHX&^VrqD49x3 zwA%6Bi%e$zFj>pCM>S-%|Ee<14K1)_|MD5$de8Vp#8ois3a_=X&Esu(v|s)>CVeZ4 zED5ADBaMDAKt8!Z(JnnUbx@{IB3ZMBjty-)#`Nd=cJE%m>0^!(qM*-xPR@G!$*}Dz zj|dSVBctz$O>x(ZW(V7dso6I7q@C_|gloI~qvqOUpD7(7u}>CU_4>?*^oUahFVpA) z!+sTwQwqVzu|8sBX7||W%s620;hyTtzy1S79UIO1da50^Ml^bb`#Rt86|&%SD~~D` zL-pJ4&tg?%mpQr$N7tZmgtVXy?>{vr8$zPo8l<~Q#>y_q%}=NCoD9+P1zV9)+2hXD z9DK&?AUmwck6+L6=o}y}^Hs9VUk{$21$^ijb4(v@qeB~y#<99r@>_B3+YvL;3mrz| z7LjpX>Jv`Vy4k=PW`CJJMDsJBz(4PHUKW6<%=kX#5&!LAW7xV$%VRdSrdD=#qpo&g zAdJ#pY(+PRr2~!}Pgq}Qad{}1d)m%r&Qq|n`Dll`|6&jggzrjd%(RL-Pe(wfmVO`Z?mkz3#|o0cVp0)ag-`Z;Iq=*1XEKs99XU_*=zCQZlaPytABM z$6-VQVjSi7mMkiqVr=vIokX#MN$+cYrViU-T&!}C&!NdpUTs(wMh9*LMehMX{=4Y( zg~|oCiu7K}pYhJjlkd*3nf(rTv^HOBzx+a6{3dusxDjK|K{Ch>NrxUG^k}FN?e}^7gvJUYd;EUrnB#if7-?=O2y++1^3J#rSD+ zg3J#T!`Z?Q#Lq_BthYJ^&r{mm!kJ#8(+x8G{Enc(hvS{xilu|KgiE3qt9v(p;f;mC z@nd2ik5V zOQa;85?FV5(5Yn7aX59>gTaGmxQkIJhQqsZeO-1iKjq>jt9T~z9@PFV=bLzmNtgApLr=9(Erz8MVu8LUdjs8g%>~cx- z&}ay}4vj>T0GI-^FH}}nq{OBcu4jL^MVioh<9>fzh2rV1LNvt1JobP8{gLKJBaD4m z?fVpLO@S#(wSfUO)|WHem6;?`AJEQ)jEVXnki_(ETCQWam!a%fA4$iautB zn!LNWIwyw)i5c&7F5Y5~;|^MIu#Q8wwTVMN=bQ0k=5xmW>w&KN!9Xj_|VR@p|MDs<<+7ilgiG%Iv~x#+J4P&LNoVC3AZXm$@oL!e!9C`ZYn} zGHpQJ?bK9O;dygy?9%7|Nr6MqJ5 zl%ygSIrV$Jq*cyAmtd?w&UOoYH{D-D+S>5NH`;6Q1FHD94y?~Yy9HN};=G2>O6&+% zSjp65jodJb*RkSCfn&E_YLi7!?7@C~W_y}P+g#|I*yc`#i*kexJU&!?rP9mxv4`Mv4%qike6Y zkboESu^K64O%}Ve2AqDCB@fcI&5+&QNiDY0vr8$i_4CpFy?;fs`>|0jUm+}9qHw#p z=h;iv6q_e;SHC>5XKL5Bqp`{^-(9opVwwp;Uz}FiwQ(;Tlra-p@5oESutl*n0Lo@f zn-37u!H8$9nd*KEJ&PVJBcs*qk2SY)DjZEnvu>}NcD06>(rYo{28+#z?qUyMO0ocJ zk>d^tMgF@n%4YL@cK9rZ2=!DVO%@l5QuHn0OyJ>$~laH*X))U*b zF9;qsomxd+4Ky(Lm!E~h2Lp}WGSv0#ctsZJlK3FW&mI!rOYISZ*>-&;#6$+EtH+D= z7merpe2sSbV1p-i&gu_P;T#sWmm5lv)q-1yG^OIWCz`y%)ojk zAw-vqV>PrijL&J*ef)}6qppbh?eecM)vj^(3OWs(jg{R@w<{l$7Qh|>@F!Yx)9z!D zmDoC1rcjNZjYCoHq1Z;LpVWL@ku+3@VadeW$Y_%+n)~u#jC*+)^5P}6<*1QQ3J>u{ zaw%g5W6x_qGHC1f@)KTwKwCJZ#fr9abN2FAe|wwSLu_n7+awBeP$|<{8e1pkr))E9 zYf#H0A#XBnj1Et<^+GJ!{C2M$AD913cmQsFmX=!CfhEWAIvYd9sNDe$b3N?7wwm|_ zZb)!Zw8gFr<3x%_-+S!icR(>7rN|V4#8PfS_RV%@#35+D+03=e6&|ShS!_5eNq?uL z=CefwD zrs`9Yj&QC-$<8a!v<8_>dF@h~Og-IwB@bCbm$B1VoEg?q1Ind8mJa4-DsM8;B%f1? zFdZh-&T62wtdtXoD}rXaNRx7XT4KrnWjv2*{n%GK-}uu~<5$a9r?@iBiQ@hyNJf{O zXtym?q(sZdbQ&%N0t(x1VqK-|%jovs?>_`YXyuC1SR3fHyN``YfRAD-w^!K8aB$Ig zViTV>H$b;QwEs*47g7JkGwKAZph&@32h(rG*TE=0j-`z^FCba$e_82Ctwv<2eTu*7 zdI*M3dc%O*(KaD0vLAe@KeJ{Db?7u~#JXWO9_>^1?Ed-dD|8xVIw7$KkZm(_VNG&- zREmz~U1=goU;+C`d;6zPyKy>ENb@II$wHtkCY%(zmMedA+6A!SkaqwcJrVv|`K&n^ z7peWbj8~-{!-9g9_FaW{B4*9x|Ly#-G3fbR5}(nO9(y|qIddsmy)4sF;q}_oFxKD_ zC^%V~IbC-R%OF`&+%|Y;u3iuDG?N0YO0j0_&o3|Kd4zGF$GWU}{!+PaG7fgsy}rA$ z)wikhlfC7Ol{j72U?3k?mD>tXFgkR-?T|LL;biMoFfbF8@xyZ)_@zMBP`c3Bs>}&# z_-kKXb*x&=@z_NI)w zK8-~PeSCg8O{t3Si&}ksR{9*{Fq$CX1;Wx7xw5$b^jRm3W*=1?KI`TLCW5wxQCnM) zjN~hO^d%ByS5_GZk_vtJ#p%LaWQlpAT+P61R?>NMw~ORCgDe0 zUoWx_Fl@;DIUFodcKiMQI-H@xmP>d^D%*X^H@JrD&RMo^=uA-sSiJhcEPhU?&9n0H zjK|IE3A+yizw-iXFo5GrY|@e%(sAlJNO#iG$^~&WT7#NuR0_ zJZ7`h0Bez*&(sBYngt)+yY|d6A=T`-XC1XD-MuMti3az3Yqs*ZXw<7I+OX9Xwt8BY zJgbXoYV+EiA`rAr&al_N86Hv4X?vi-MP|p`hyD5wt&X~`<|zYL+wE#bTO%~9v9Uzk zmm~cC!YQ4J6Udv5JvCvy#!w6YD9TBkrc7(h1fOL*Rmq>d)nnE_q>AVa88t17lU-(q*2TC#ok6V5xZab}^ zHkGalZDJ{paf&t4keno4_X6YyE?kN;$C$A`o#ec)1~YmdO?a{H7AlY(eVJ}jIcb`k zy?!R?DHkH7N5L7zJJ9Z*|CM4vZ9`+C1oBZ(#vCYGP8AKT$m-B;jIHaL*G01lY!g)^ z4J`I-a(!yfK-vkA%48oZWX)B{?D{t4I_f(x#rSB52HgS&R(UZcSnN`^z zsxLR}jCa3ZJr&|uRF=<@%AJ*JjX$(F2(+SLL~hFp0k0Vo(oLrI8J>gB6d&r!TLtaX zvO4*QwMW)?O(H@%Ds%&&P1wmdHFc)mn>NG@C)Q}qf3W$zRUdf{AULVOGQEwLRTgCr zb#=$a>Dwe#8&?cfyEGg7o3`+406MKe3;BHfJK6dIm3>}@-hA8UuF;DS{c$EV3C_TJ zN($k*`qy9l@?*}{N}v(;wzisWhW)`U^bxM07SN&xJK{SR7l}0FFYWxKnITC+#Za#e zMBi9xe|5AwEdVX;B_yjxs|L?Yii@+=p0mm+M-cYe6VjoZ9k7_=Hpc<`l9%JzCKN^F zkUs#@$DyA~_hQHFYB;FQY_EEV%8^4Uiqn#cv~FH#o!h$%XpXJPBsV3+Uo7QmVB-o z-R&lH{#1-`Na_Di3$mHt$FFd3jEuATg2~H9AMCYmUB)anZ<>vZc0F+7iSPtkZXhJy z)>KS|PH)e|)T}>$j@ReXTqxnnD>mw`r)1k2kDP%^opg9Q;9>wTeIJ`pT& zmTuH`2Ze+FMY3;txPFoGrL&uVtb(Z}0I(4jj_wnptFH;kMJ{XMUx#+G-Dazh$aW!# zj1X2?JF=Q|>CMkUBHc}qvXa?t0V1CP>n7f8Rs)xH`_Wl0QzAzeI^LhXBBTexD6*_v7Brnw*|mriDN{3`L0{~^cAEmO zUdol}xpG>|F}m>A>=vv*RA#Y5`C=Yv^CO6m=Zs6jrwQQ(MSHgGp?s}t8C1ioAuEXo z6TrTVKwLIVj5C~LcF&~LnJC_P;G03laAR!l#==x!McVw}ZeMGq*5yGqyeaY=h*nCc zl@UaK+da%j$adQ^oFca3!eiE0XomUl+2J>!FuVPGPvL;c!*8SPvF_XfG}T2vth%Ix zA9toWII0meV#gRHDnLzk_M7FTa-1}mW-0NxHHZ+%LXvMOtxFGT9FZoNV^sy#u1=`w zQWaOG(-Mz8if1-Fm<@~6YZ!MO$OSt603_d7Mdq&M`e|?9Xbwzvawl$Pao( zXci;`Ri&QpxQlQdOV$+QD89YYyB3{1jhfoPjBs47fis<_hSXa{IdhJap(M4iuN(ER zs-A<4s&bj$n9)EbMLFaiuh!`M^f=#Mn3b)K7PtA1wcUCp<|t*KSZ?y0x`))E?* zvnO})qHKDmOq(>|bZs%(=v3TcoYxYmt<(!g(fmVQZIBbGs){)}k4D4<7WX7*0c8GRj1;rshPfQM_RXwX^39HWUB3pK<+f*7!_Oc}5b| z8|H6Q?--}?Z;Dh+m}YGJjYk!^5O9bGDx7TMKTvlj>a2NnIJgPCQeWiJ^KZdCid|eG zSKB<8m5_8H)a^gin6$pOUjOM*vPTE;f1wqn8hQq_#ExbIq37Cdvvm9pTR8!>Y^86p z$mWE5#(t`M6E?#arTdbx7{$FSko&4TlV0pyeadXc`0TJj+0o86nY3)G>VD7EB4Yk@ z_Jc}6lHH|ReVVKzd57{v{#)#pY9p-ptB=d>o8en?GmX*gQ`~2a5SqA+$w80zcBNClJ0r$=_1^4XyV_4+&8LUFc0;Q% zEcstWbKiQro3U~5Qga8+__RaawML0{1VCq*32uS%BQ9M1Lgw zK>oII!sTTpv_!pVcAeW0KicJVtR(0(?7>~>`8I!C8QPgh05Hm3vFHRl`t?U6A9yWA`G+X3`48iFPQebd$AMau9Q$ zmGx$QJcgmjgd+pNlY@N&IVZ949Z2yNYhPF4q}cMP5cP6hUQ^rnc#}NG?IKe}W#ZcT zh)p)XBtNSVItl!)v66!`#eJ>qSbGTAeI6PW`Lx*y+F|m-X&Y>occW%Ule?U-Mc=R6 ze+t2ZZowdPd@$(?W4ukmHgN<^5+^RsUpG#K8kb{F4&b)Ha%5*>j_ZU&`cjyx`SsI1#QzlV5->xCpsx zzJuSRE@Qg*a;=jkE+UGyZO+(L86Q|dwrKl#mvLQ@IPm2A`#5J}zN1P}@U0szbdPsl zup!>sWSHlFzKZ`-ddo!xEzE_f)#EdsGR^PizHVT$T7MVaAdf|!QC$M$BD>r{_aXR# zgu~Vn8(E$;ufC!8TyZ>AcaQ$>9{)q~@AZ+mS@^Af`uysvp_P%s+i$TPgPFR#AXEC; z;qIzfX0UIgTvzQ~MvB4UEmB4fdfqR@QTRo^j~@@y3OD z>b-xVgIveZwdeN#-tZI`>!juCb8+7K+ge_lH8WqgSw8(A&;Ng_duiAB3qiTkMm!^i zV^h-)OL;Lp4sd5%GL@3Q(|-*z{S+k+V?G`mzbQdm%m#qb^rLOa#Tx_k%|J5o)i~3( zd-3DFJRTwD;8Ws(NkfW^(q!AqY;*O%RP;DkncHR`Y0m*EbcQtRQVVr?XfY@D`?RP- zCGxryYO)d}TrRvgv#;XQbmUhAjXo8ompp%C@E0jQayTJR?hJ=SxHrr^)%|I*@h>1r zO_0lNElhgxi&4@wx1cmM>wN*5G-K!d82SsF_{s_kY%PCJOO5x>yHbaz7l^>-nWWqC zlZn*T5dfg;m*xNM_kT%_8Nz*CfzcN|nHxcJy4i}m-VK~-0(TZ&DrD>OsoJS)r}has zmRr;W$59VZ|138%pxBdkPzQc1ZkL&Q;@@^eI}-Yv5#9AAc)9EQKJ~Y*x-@3mN6ev| zH;L`{BkhbqlTJgxpwLTdn)P$fkY!}+kwRTB+&1-Quh4v`Ruoi))5`+ge3S#c;8h{O zKkCTM2(ZFoH-ytP9$E)zjSaK!Pi`QgCLA%G9o*;JhUe3#^Ip~yqoUbkfQQmgaIq6W zy!B%5{p0_&$Nx_^Dfh8K-`?`l)e+Rgp{K=4!~&P)2|GIx+_7#H41Wf-+N^T5>@L$B zUa*h9sg&5{{^Xi0sKj`jI@;#Vj37%!efQNQeAFKSb2$v@jd z_9{>tZr=~Da35*N&TSr$M?*JSg=~0xS>yGj{b=&HTV^r&<*tlqMB$IhiNWDkiz53m zk)Tla+$>xdTxL`5FlamnogFN-uJAe6{n2EU$UHKJ^dE%r;;@(ovLZY}+4zJf3_%CPtPS z;LMbhUu>$>fCL(8qq?W9(cj@%oxglK<=>Lm_I+19?6^vAH>EEBPJ|IL3pjsY(vw{d2ruscj~#Z6GnJQLdTEeIXi^_tT0zYEmQ5>Jv;D(1M)T>t`J zDy3+D>O1?G{=sSr=)L}>mh!JD?*TJ+X9rVvHr~0fJq^Z}cbx+W^Urti4u67fbLSf_ zrm0vbj{So{bxf0$Y^AD5{dadjk7c+sBRLL|rHFh9oHOW!%~tNadt*HsigChuHM)<- zW(%CBQr?7C)%^t}*~$rV2*Pe#AVx~H3qM~e(tQ_{|FUfzL#HHWT zc?Ty~g>COeZC(nww-^Jq0F+eanQ;SBnK4T!u+{Aol)d&OkBEe_2E& zg$w-uQ2MbpS&nU7|D}%=7_CxA%yos-O=dYldh1-()^$zDU=+Cwym(@u)w06J?DS?s zW`1+!El;<8P{!A!J^%(n7N}d)Da4JWI>gvvJe;{Y6Kd35hX@N<{MT@?5)MPN+!wRM zZtaZ4a*Q3i=h-CFIp@kOTzH;&U5V89AIz2}Vnl{rsXhOvj?MVSEZbtosp~J|vI@}a zf+b7j%ofgZU68+=RGG%VCU=lB`qfa?2_a3{R$qn+dIiyBUnR6OA~=AjM4K5dH*G?Q z`+w$7UZ!sz(W{ZG6xl0cqWfE4-Hto!!n*5{w{eK_RUYJjPRsjtWvWPuvnKn&Qmq*Q zFEUx0p{azoR5PSs(bpHHv)iX91qpe-&s-Oh5&!bl&}&1+3!;BywQ&27jSlZZ%tn{60LR`@hLqlf?drdu2ZvSzD{`X1!amTpNdPnf<&CXj-TiL1p6GDam}7pGr&xdf zUKEF3iYR;7U4|_4l1QD6@F}+>$3xd-xt28~OFg}dD&mr|{H!yMp!ydO8XZm=4TZ(S z)EdI~AeV+?Ckl|o*5rGpaKvD7LXb2rJ0nMRa+yt42uy5{gyXxEO5CsR-PUW`E0;sC|XE?2_`VQel?@;QU^SpcR7?Y+hQ6InZGsl}? z#;l;2JoA#Pa5sdnvc9e_7+g67HhW(<;w{?as5g9EEBV2S$1*YyRM7CImnXfHm64s5 zg)v>v4a><8xnPygX46BX8BBs!+aA4nILeQ?MI(@#gqMrJiI%)G5(`k{c@>$_8zi`E zNQks9ddTcO+N$Z8n*t{DULk^1JT(<0t(vu+b_&Mw33tOkGXA{hT=Up6`MpO9K2k|n zd?y#>8$|vtpjhKShzkEEI#78{w|#q|TM&3d z#r9&dXbsEa2N2P{d!na-AEnhQ{=diC$$jy+<^O>A`!|?D(z?WSoA;xriBDvhfp(~{ zd&}RiX%f=}??-DDx$IEk^%?bAkBLuAm_Zi*VN(4@qxbg&n1Nu3@LYrUcLvNrgeD;| zr+%Z}`_V#0u3)IJFij#NXyQ{}MXnLQ44120>-}h^A~zRi;1D8wVNN~h(Zr_@FoWI@ zVc%)>TJMQZRdQeGM_k|&=51%<1=d_d5V!%oj01wy4pjA(b5oO2uC{~gw_`r`LF15~ z|Csk@Y08o$v*KRdIxJrI;r^Ge18_^O&-Su}YY)QZuNBAW$`iC_hjvEQN^DDMoifSy z7$CS~8Vrq~q8iSeNgu5!J4;D;0KP@kDw{}%!-l~@B;;~NN z`T`WUx`?ACSvJhvj#5za{lc|zlv-+qEkG3CC-Q`%NNW_CE4*k{lfKT{8VEu|8ASFi zIg9xuO)@0U9bn1$MK&1SbuGYk8W74fK>o?4G5(+Jk&jO6_VVK7AO8b%hzt_C62UX+&(R`Xh(dD+k1o>F#N<5fRXCQw^Xg%_`X<6@=#-!iw$Yur!M9ou?~ zoR&~>?tN)TOZ=9Q1sKin*Y~kq)8rd*tv?rG2meOhKmIn3Aj>Uk;xO5n(?vbKyJ^SG-ehNi zW!DPKpMs<}BJv+Lg+rI@+RfYgx84)NyQaF}H+RseNmbdT6yx_{h|?>Vpp9Y!BwRhb zZ`;rnA*~f{hK$ftRmT~dA{v!an{X=)5P7kZwU#?3GApO7PLC-?M-0rSSm=vtCK^|F z41_<2j|K(?7N3?;JiA4|U@{vRzU4PsWSQ7=$0d_BS>m!Z?$(4%&@4Em8`@+&dqza6 z#i?rCEiOt)OtG@N$g@jt$@463Ud+X=I@>}awS2PV>ehVti)VK|wIR#uHYYJAJyKru z0Q<+WUfuW26e}{c@BxD)oZ?Eo2I-VK86!Tn5j7$**}A1rIm(K9ie2S`OD(TgCdZ$p+J6&Lp$0{$sVdGu+U~;0le4FUhuVwn{O06gs#gQ>u6`0?LbEsO zU%FpbdA4bb-TJa9>thSy-?MVOX+2Lj&L7uvI5ac7E%BT0>~O zzM)vg>Z((W)D`{I`}=7QIoKn^56!OenlBM(-b==>c9}gT7V2)>O=sOf=x-P9AynK; zSMsoBj=4%|fsthwB47O^a2!$VW@$l*Rji?{T3b4D$#?jLr$zQ}Cd_L#de8dGt;Ed` z2y$qMG?zwW-}B!!-3&fs@b(H|2iJQ9;cV4G8o#xlA%YReo7L%mQLZ|a2X1)ip~-Fy zt?3RG>!X&%A`$cmUR{m_(hrdo?KsD|`g|LXRH4gBeV5>KIF_K)^^FF!gjV7cSyAF{ z@>7Y&MNco{BxYTegXmF?05+lM%yQU|QEijUmVtq25Iz!P#0Oh7)6o|{2(n9VUrb3R z;iInDsJ-+UxPeU(tr+eS9#GTNpngDGf$&ky4faYa6`Lod0t`-N*4{QiDrbV@PnNku z!c?=Zbo(h{e*I5X=oRoMx_-^?oZe05f5c2O(3|O^LF`uMuoi{*Oemc&`7JczZs<_7 zus8A~eGFAsSR`$bXWlgMV=AZ53Ii2$;4T~5md_1pU~0Bf2~(=WXJ?;d@Rhph#nBnl zN}-KbCmn|CO${0K&V8d>xO@?o-{YNRACt#)_a00iTx~QmGr*LQ0irr!17y__z?-@& zSqf>WW=}KWL|0XWvrU_`NVVqdj&K3u?_5y<-Bup0&9P)hj8KV9)HaVxE5hEG%6Qsa_7GTZw<-oZ>^bk+9_BV^G{J2q*C3aQz~gwRsk&wlw+ znm5-X51nyeS*APl7F~xwHiaJq<>}RM{O6rrwC0MG3JZJk$EcK1DL&c%?9H(6jSBWs zOU_;f

cO=OG1S!rnH1sm4vNKr+Z!3;$?&*k|Tcua5)I)4@es^MeYq&RlxeW-O%| z3R7`-jca;7X&JkvlUb;}8*WKwU2k@)73sY{b5RNaF^f3t$M!gzwx^y|x+UgUv`$$>3E++aK8&5++53)A=E%wRHH(WyP7##`!+)~cZf5`hoS;P& zrIUF;R<-xiB~_EIk?6#Te3ShA3F{X(;d%O#&0jxtppI)_9#4YICv{6j09nyAqq(8F-O(hlbH7Lam#?;exB6}s)Wh9hjQ z$o=r1z2w8w&I$cB5EUMQ7TK<+$pkSk+8TGHzI~9 z*6$$cf1OQ_ud8G($rsTz&qc`;OBzOmY#T(j7Djuq8iRw4(g$kzkaErwC_|K0$SAX1~jtE9Xld_E>TeoXj}|HIBi4_)}hP9^fc4GsQ)g2h*&;N&wr z*bwEv40bP&ag_G&aiA2%^6LD}#bQgJU`Pnn9^rGzVRzg>d-egjq(Gz0dv1H}2F~pI z_`5~;I;Ahd=23^Jgn7Gw#GXGpC;PBYzKZD4-BbU@$f9PFr)lhdr@UjO_(W@>)q3Zc%L$W(4q1Ik&+OFK*K^!5hMdCo5f zFd9+M_#q5AUyr8X&dj3BG#h@%;fFedpkS@+TlSx9xFpS`SVtneQMkSK&=uZo^g9K5?k?9Pl}kYUqFu|)6|5%)InP|$Y)unaJwKh*uZ7|0%6_>a>-N4|`uB(Amq>gc^~2z-_2diWP6sc^95VZ8 zOXSNFYe32PMXD&&$?UGCF^KH_mXmy}!{IN)+b_bvCBl4?U u(v Date: Mon, 21 Jul 2025 15:07:29 +0200 Subject: [PATCH 165/312] EmbedderOptions::has_fragments() --- crates/milli/src/vector/mod.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index f64223e41..873693a34 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -841,6 +841,25 @@ impl EmbedderOptions { } } } + + pub fn has_fragments(&self) -> bool { + match &self { + EmbedderOptions::HuggingFace(_) + | EmbedderOptions::OpenAi(_) + | EmbedderOptions::Ollama(_) + | EmbedderOptions::UserProvided(_) => false, + EmbedderOptions::Rest(embedder_options) => { + !embedder_options.indexing_fragments.is_empty() + } + EmbedderOptions::Composite(embedder_options) => { + if let SubEmbedderOptions::Rest(embedder_options) = &embedder_options.index { + !embedder_options.indexing_fragments.is_empty() + } else { + false + } + } + } + } } impl Default for EmbedderOptions { From 109395c199e03f96344907513fb0327c4f6467c3 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 21 Jul 2025 15:08:08 +0200 Subject: [PATCH 166/312] Index::embeddings specifies if the embedder has fragments --- crates/milli/src/index.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/index.rs b/crates/milli/src/index.rs index b2ec992ba..9f32fdb04 100644 --- a/crates/milli/src/index.rs +++ b/crates/milli/src/index.rs @@ -1766,20 +1766,22 @@ impl Index { &self, rtxn: &RoTxn<'_>, docid: DocumentId, - ) -> Result, bool)>> { + ) -> Result> { let mut res = BTreeMap::new(); let embedders = self.embedding_configs(); for config in embedders.embedding_configs(rtxn)? { let embedder_info = embedders.embedder_info(rtxn, &config.name)?.unwrap(); + let has_fragments = config.config.embedder_options.has_fragments(); let reader = ArroyWrapper::new( self.vector_arroy, embedder_info.embedder_id, config.config.quantized(), ); let embeddings = reader.item_vectors(rtxn, docid)?; + let regenerate = embedder_info.embedding_status.must_regenerate(docid); res.insert( config.name.to_owned(), - (embeddings, embedder_info.embedding_status.must_regenerate(docid)), + EmbeddingsWithMetadata { embeddings, regenerate, has_fragments }, ); } Ok(res) @@ -1919,6 +1921,12 @@ impl Index { } } +pub struct EmbeddingsWithMetadata { + pub embeddings: Vec, + pub regenerate: bool, + pub has_fragments: bool, +} + #[derive(Debug, Default, Deserialize, Serialize)] pub struct ChatConfig { pub description: String, From 324666759032683e164e0dab8e1f52d57cb875bf Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 21 Jul 2025 15:09:31 +0200 Subject: [PATCH 167/312] when exporting vectors, for regenerate to false when the embedder has fragments --- .../src/scheduler/process_dump_creation.rs | 14 ++++++++++++-- .../src/scheduler/process_export.rs | 14 ++++++++++++-- crates/meilitool/src/main.rs | 14 ++++++++++++-- 3 files changed, 36 insertions(+), 6 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/process_dump_creation.rs b/crates/index-scheduler/src/scheduler/process_dump_creation.rs index b8d100415..b14f23d0b 100644 --- a/crates/index-scheduler/src/scheduler/process_dump_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_dump_creation.rs @@ -5,6 +5,7 @@ use std::sync::atomic::Ordering; use dump::IndexMetadata; use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME; +use meilisearch_types::milli::index::EmbeddingsWithMetadata; use meilisearch_types::milli::progress::{Progress, VariableNameStep}; use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors}; use meilisearch_types::milli::{self}; @@ -227,12 +228,21 @@ impl IndexScheduler { return Err(Error::from_milli(user_err, Some(uid.to_string()))); }; - for (embedder_name, (embeddings, regenerate)) in embeddings { + for ( + embedder_name, + EmbeddingsWithMetadata { embeddings, regenerate, has_fragments }, + ) in embeddings + { let embeddings = ExplicitVectors { embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors( embeddings, )), - regenerate, + regenerate: regenerate && + // Meilisearch does not handle well dumps with fragments, because as the fragments + // are marked as user-provided, + // all embeddings would be regenerated on any settings change or document update. + // To prevent this, we mark embeddings has non regenerate in this case. + !has_fragments, }; vectors.insert(embedder_name, serde_json::to_value(embeddings).unwrap()); } diff --git a/crates/index-scheduler/src/scheduler/process_export.rs b/crates/index-scheduler/src/scheduler/process_export.rs index a951a7ca6..0cd06f2e4 100644 --- a/crates/index-scheduler/src/scheduler/process_export.rs +++ b/crates/index-scheduler/src/scheduler/process_export.rs @@ -9,6 +9,7 @@ use flate2::write::GzEncoder; use flate2::Compression; use meilisearch_types::index_uid_pattern::IndexUidPattern; use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME; +use meilisearch_types::milli::index::EmbeddingsWithMetadata; use meilisearch_types::milli::progress::{Progress, VariableNameStep}; use meilisearch_types::milli::update::{request_threads, Setting}; use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors}; @@ -229,12 +230,21 @@ impl IndexScheduler { )); }; - for (embedder_name, (embeddings, regenerate)) in embeddings { + for ( + embedder_name, + EmbeddingsWithMetadata { embeddings, regenerate, has_fragments }, + ) in embeddings + { let embeddings = ExplicitVectors { embeddings: Some( VectorOrArrayOfVectors::from_array_of_vectors(embeddings), ), - regenerate, + regenerate: regenerate && + // Meilisearch does not handle well dumps with fragments, because as the fragments + // are marked as user-provided, + // all embeddings would be regenerated on any settings change or document update. + // To prevent this, we mark embeddings has non regenerate in this case. + !has_fragments, }; vectors.insert( embedder_name, diff --git a/crates/meilitool/src/main.rs b/crates/meilitool/src/main.rs index b967e620c..170bbdcc8 100644 --- a/crates/meilitool/src/main.rs +++ b/crates/meilitool/src/main.rs @@ -15,6 +15,7 @@ use meilisearch_types::heed::{ }; use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME; use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader}; +use meilisearch_types::milli::index::EmbeddingsWithMetadata; use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors}; use meilisearch_types::milli::{obkv_to_json, BEU32}; use meilisearch_types::tasks::{Status, Task}; @@ -591,12 +592,21 @@ fn export_documents( .into()); }; - for (embedder_name, (embeddings, regenerate)) in embeddings { + for ( + embedder_name, + EmbeddingsWithMetadata { embeddings, regenerate, has_fragments }, + ) in embeddings + { let embeddings = ExplicitVectors { embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors( embeddings, )), - regenerate, + regenerate: regenerate && + // Meilisearch does not handle well dumps with fragments, because as the fragments + // are marked as user-provided, + // all embeddings would be regenerated on any settings change or document update. + // To prevent this, we mark embeddings has non regenerate in this case. + !has_fragments, }; vectors .insert(embedder_name, serde_json::to_value(embeddings).unwrap()); From 01d1ef65c4b3a3b306edf6f2589fe029f1c0f83f Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 21 Jul 2025 15:10:25 +0200 Subject: [PATCH 168/312] Update search and docs usages --- crates/meilisearch/src/routes/indexes/documents.rs | 9 +++++++-- crates/meilisearch/src/search/mod.rs | 9 ++++++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 947cd153f..138f5140f 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -19,6 +19,7 @@ use meilisearch_types::error::{Code, ResponseError}; use meilisearch_types::heed::RoTxn; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::milli::documents::sort::recursive_sort; +use meilisearch_types::milli::index::EmbeddingsWithMetadata; use meilisearch_types::milli::update::IndexDocumentsMethod; use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors; use meilisearch_types::milli::{AscDesc, DocumentId}; @@ -1460,9 +1461,13 @@ fn some_documents<'a, 't: 'a>( Some(Value::Object(map)) => map, _ => Default::default(), }; - for (name, (vector, regenerate)) in index.embeddings(rtxn, key)? { + for ( + name, + EmbeddingsWithMetadata { embeddings, regenerate, has_fragments: _ }, + ) in index.embeddings(rtxn, key)? + { let embeddings = - ExplicitVectors { embeddings: Some(vector.into()), regenerate }; + ExplicitVectors { embeddings: Some(embeddings.into()), regenerate }; vectors.insert( name, serde_json::to_value(embeddings).map_err(MeilisearchHttpError::from)?, diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 93efad67f..82096e7b4 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -16,7 +16,7 @@ use meilisearch_types::error::{Code, ResponseError}; use meilisearch_types::heed::RoTxn; use meilisearch_types::index_uid::IndexUid; use meilisearch_types::locales::Locale; -use meilisearch_types::milli::index::{self, SearchParameters}; +use meilisearch_types::milli::index::{self, EmbeddingsWithMetadata, SearchParameters}; use meilisearch_types::milli::score_details::{ScoreDetails, ScoringStrategy}; use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors; use meilisearch_types::milli::vector::Embedder; @@ -1528,8 +1528,11 @@ impl<'a> HitMaker<'a> { Some(Value::Object(map)) => map, _ => Default::default(), }; - for (name, (vector, regenerate)) in self.index.embeddings(self.rtxn, id)? { - let embeddings = ExplicitVectors { embeddings: Some(vector.into()), regenerate }; + for (name, EmbeddingsWithMetadata { embeddings, regenerate, has_fragments: _ }) in + self.index.embeddings(self.rtxn, id)? + { + let embeddings = + ExplicitVectors { embeddings: Some(embeddings.into()), regenerate }; vectors.insert( name, serde_json::to_value(embeddings).map_err(InternalError::SerdeJson)?, From 6dc241f9dec9bc9c91e1959e84cdab68e238e280 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Mon, 21 Jul 2025 15:10:39 +0200 Subject: [PATCH 169/312] Fix tests --- .../src/scheduler/test_embedders.rs | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/test_embedders.rs b/crates/index-scheduler/src/scheduler/test_embedders.rs index a9b920bd2..791fed4d8 100644 --- a/crates/index-scheduler/src/scheduler/test_embedders.rs +++ b/crates/index-scheduler/src/scheduler/test_embedders.rs @@ -3,6 +3,7 @@ use std::collections::BTreeMap; use big_s::S; use insta::assert_json_snapshot; use meili_snap::{json_string, snapshot}; +use meilisearch_types::milli::index::EmbeddingsWithMetadata; use meilisearch_types::milli::update::Setting; use meilisearch_types::milli::vector::settings::EmbeddingSettings; use meilisearch_types::milli::vector::SearchQuery; @@ -220,8 +221,8 @@ fn import_vectors() { let embeddings = index.embeddings(&rtxn, 0).unwrap(); - assert_json_snapshot!(embeddings[&simple_hf_name].0[0] == lab_embed, @"true"); - assert_json_snapshot!(embeddings[&fakerest_name].0[0] == beagle_embed, @"true"); + assert_json_snapshot!(embeddings[&simple_hf_name].embeddings[0] == lab_embed, @"true"); + assert_json_snapshot!(embeddings[&fakerest_name].embeddings[0] == beagle_embed, @"true"); let doc = index.documents(&rtxn, std::iter::once(0)).unwrap()[0].1; let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); @@ -311,9 +312,9 @@ fn import_vectors() { let embeddings = index.embeddings(&rtxn, 0).unwrap(); // automatically changed to patou because set to regenerate - assert_json_snapshot!(embeddings[&simple_hf_name].0[0] == patou_embed, @"true"); + assert_json_snapshot!(embeddings[&simple_hf_name].embeddings[0] == patou_embed, @"true"); // remained beagle - assert_json_snapshot!(embeddings[&fakerest_name].0[0] == beagle_embed, @"true"); + assert_json_snapshot!(embeddings[&fakerest_name].embeddings[0] == beagle_embed, @"true"); let doc = index.documents(&rtxn, std::iter::once(0)).unwrap()[0].1; let fields_ids_map = index.fields_ids_map(&rtxn).unwrap(); @@ -497,13 +498,13 @@ fn import_vectors_first_and_embedder_later() { let docid = index.external_documents_ids.get(&rtxn, "0").unwrap().unwrap(); let embeddings = index.embeddings(&rtxn, docid).unwrap(); - let (embedding, _) = &embeddings["my_doggo_embedder"]; - assert!(!embedding.is_empty(), "{embedding:?}"); + let EmbeddingsWithMetadata { embeddings, .. } = &embeddings["my_doggo_embedder"]; + assert!(!embeddings.is_empty(), "{embeddings:?}"); // the document with the id 3 should keep its original embedding let docid = index.external_documents_ids.get(&rtxn, "3").unwrap().unwrap(); let embeddings = index.embeddings(&rtxn, docid).unwrap(); - let (embeddings, _) = &embeddings["my_doggo_embedder"]; + let EmbeddingsWithMetadata { embeddings, .. } = &embeddings["my_doggo_embedder"]; snapshot!(embeddings.len(), @"1"); assert!(embeddings[0].iter().all(|i| *i == 3.0), "{:?}", embeddings[0]); @@ -558,7 +559,7 @@ fn import_vectors_first_and_embedder_later() { "###); let embeddings = index.embeddings(&rtxn, docid).unwrap(); - let (embedding, _) = &embeddings["my_doggo_embedder"]; + let EmbeddingsWithMetadata { embeddings: embedding, .. } = &embeddings["my_doggo_embedder"]; assert!(!embedding.is_empty()); assert!(!embedding[0].iter().all(|i| *i == 3.0), "{:?}", embedding[0]); @@ -566,7 +567,7 @@ fn import_vectors_first_and_embedder_later() { // the document with the id 4 should generate an embedding let docid = index.external_documents_ids.get(&rtxn, "4").unwrap().unwrap(); let embeddings = index.embeddings(&rtxn, docid).unwrap(); - let (embedding, _) = &embeddings["my_doggo_embedder"]; + let EmbeddingsWithMetadata { embeddings: embedding, .. } = &embeddings["my_doggo_embedder"]; assert!(!embedding.is_empty()); } @@ -696,7 +697,7 @@ fn delete_document_containing_vector() { "###); let docid = index.external_documents_ids.get(&rtxn, "0").unwrap().unwrap(); let embeddings = index.embeddings(&rtxn, docid).unwrap(); - let (embedding, _) = &embeddings["manual"]; + let EmbeddingsWithMetadata { embeddings: embedding, .. } = &embeddings["manual"]; assert!(!embedding.is_empty(), "{embedding:?}"); index_scheduler From ab07e9480ec8364dc2555252e5bc8b32c02d7379 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 21 Jul 2025 18:22:10 +0200 Subject: [PATCH 170/312] Resolve post-merge issues --- .../src/routes/indexes/documents.rs | 2 +- crates/meilisearch/tests/search/filters.rs | 28 +++++++++---------- crates/meilisearch/tests/vector/mod.rs | 2 +- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/documents.rs b/crates/meilisearch/src/routes/indexes/documents.rs index 0b70ad86f..cf6181de5 100644 --- a/crates/meilisearch/src/routes/indexes/documents.rs +++ b/crates/meilisearch/src/routes/indexes/documents.rs @@ -139,7 +139,7 @@ pub struct DocumentsFetchAggregator { // if a filter was used per_filter: bool, with_vector_filter: bool, - + // if documents were sorted sort: bool, diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index aa2b06e76..fd5bc57db 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -4,8 +4,8 @@ use tempfile::TempDir; use super::test_settings_documents_indexing_swapping_and_search; use crate::common::{ - default_settings, shared_index_with_documents, shared_index_with_nested_documents, Server, - DOCUMENTS, NESTED_DOCUMENTS, + default_settings, shared_index_for_fragments, shared_index_with_documents, + shared_index_with_nested_documents, Server, DOCUMENTS, NESTED_DOCUMENTS, }; use crate::json; @@ -734,7 +734,7 @@ async fn test_filterable_attributes_priority() { #[actix_rt::test] async fn vector_filter_all_embedders() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -769,7 +769,7 @@ async fn vector_filter_all_embedders() { #[actix_rt::test] async fn vector_filter_missing_fragment() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -789,7 +789,7 @@ async fn vector_filter_missing_fragment() { #[actix_rt::test] async fn vector_filter_non_existant_embedder() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -809,7 +809,7 @@ async fn vector_filter_non_existant_embedder() { #[actix_rt::test] async fn vector_filter_all_embedders_user_provided() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; // This one is counterintuitive, but it is the same as the previous one. // It's because userProvided is interpreted as an embedder name @@ -831,7 +831,7 @@ async fn vector_filter_all_embedders_user_provided() { #[actix_rt::test] async fn vector_filter_specific_embedder() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -866,7 +866,7 @@ async fn vector_filter_specific_embedder() { #[actix_rt::test] async fn vector_filter_user_provided() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -892,7 +892,7 @@ async fn vector_filter_user_provided() { #[actix_rt::test] async fn vector_filter_specific_fragment() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -951,7 +951,7 @@ async fn vector_filter_specific_fragment() { #[actix_rt::test] async fn vector_filter_non_existant_fragment() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -971,7 +971,7 @@ async fn vector_filter_non_existant_fragment() { #[actix_rt::test] async fn vector_filter_specific_fragment_user_provided() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -991,7 +991,7 @@ async fn vector_filter_specific_fragment_user_provided() { #[actix_rt::test] async fn vector_filter_document_template_but_fragments_used() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -1093,7 +1093,7 @@ async fn vector_filter_feature_gate() { #[actix_rt::test] async fn vector_filter_negation() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ @@ -1125,7 +1125,7 @@ async fn vector_filter_negation() { #[actix_rt::test] async fn vector_filter_or_combination() { - let index = crate::vector::shared_index_for_fragments().await; + let index = shared_index_for_fragments().await; let (value, _code) = index .search_post(json!({ diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 8851d029e..8a701ac83 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -14,7 +14,7 @@ use meilisearch::option::MaxThreads; use crate::common::index::Index; use crate::common::{default_settings, GetAllDocumentsOptions, Server}; use crate::json; -pub use {fragments::shared_index_for_fragments, rest::create_mock}; +pub use rest::create_mock; pub async fn get_server_vector() -> Server { Server::new().await From ba0f50e5efe280c111feb2714b1c635bd07a7434 Mon Sep 17 00:00:00 2001 From: nicolasvienot Date: Mon, 21 Jul 2025 20:00:36 +0200 Subject: [PATCH 171/312] fix: update default deserialization for ChatSearchParams limit field --- crates/milli/src/update/chat.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/milli/src/update/chat.rs b/crates/milli/src/update/chat.rs index 2f364894d..a6c0b3fbc 100644 --- a/crates/milli/src/update/chat.rs +++ b/crates/milli/src/update/chat.rs @@ -93,7 +93,7 @@ pub struct ChatSearchParams { pub hybrid: Setting, #[serde(default, skip_serializing_if = "Setting::is_not_set")] - #[deserr(default = Setting::Set(20))] + #[deserr(default)] #[schema(value_type = Option)] pub limit: Setting, From 0014ed31145b1c9a4021ab6069362cc23c884b37 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 10:56:05 +0200 Subject: [PATCH 172/312] Apply review suggestions --- crates/meilisearch/tests/search/filters.rs | 21 +- .../milli/src/search/facet/filter_vector.rs | 278 ++++++++++-------- crates/milli/src/vector/db.rs | 1 + 3 files changed, 161 insertions(+), 139 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index fd5bc57db..4701e986d 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -981,7 +981,7 @@ async fn vector_filter_specific_fragment_user_provided() { .await; snapshot!(value, @r#" { - "message": "Index `[uuid]`: Vector filter cannot specify both a fragment name and userProvided.\n31:43 _vectors.rest.fragments.other.userProvided EXISTS", + "message": "Index `[uuid]`: Vector filter cannot have both `other` and `userProvided`.\n31:43 _vectors.rest.fragments.other.userProvided EXISTS", "code": "invalid_search_filter", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_search_filter" @@ -1022,19 +1022,19 @@ async fn vector_filter_document_template() { let (response, code) = index .update_settings(json!({ - "embedders": { - "rest": setting, - }, + "embedders": { + "rest": setting, + }, })) .await; snapshot!(code, @"202 Accepted"); server.wait_task(response.uid()).await.succeeded(); let documents = json!([ - {"id": 0, "name": "kefir"}, - {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, - {"id": 2, "name": "intel"}, - {"id": 3, "name": "iko" } + {"id": 0, "name": "kefir"}, + {"id": 1, "name": "echo", "_vectors": { "rest": [1, 1, 1] }}, + {"id": 2, "name": "intel"}, + {"id": 3, "name": "iko" } ]); let (value, code) = index.add_documents(documents, None).await; snapshot!(code, @"202 Accepted"); @@ -1052,9 +1052,6 @@ async fn vector_filter_document_template() { { "name": "kefir" }, - { - "name": "echo" - }, { "name": "intel" }, @@ -1066,7 +1063,7 @@ async fn vector_filter_document_template() { "processingTimeMs": "[duration]", "limit": 20, "offset": 0, - "estimatedTotalHits": 4 + "estimatedTotalHits": 3 } "#); } diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index e3ec698f5..3a0f86637 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -2,14 +2,107 @@ use filter_parser::Token; use roaring::RoaringBitmap; use crate::error::{Error, UserError}; +use crate::vector::db::IndexEmbeddingConfig; use crate::vector::{ArroyStats, ArroyWrapper}; use crate::Index; +#[derive(Debug)] +enum VectorFilterInner<'a> { + Fragment { embedder_token: Token<'a>, fragment_token: Token<'a> }, + DocumentTemplate { embedder_token: Token<'a> }, + UserProvided { embedder_token: Token<'a> }, + FullEmbedder { embedder_token: Token<'a> }, +} + +impl VectorFilterInner<'_> { + fn evaluate_inner( + &self, + rtxn: &heed::RoTxn<'_>, + index: &Index, + embedding_configs: &[IndexEmbeddingConfig], + regenerate: bool, + ) -> crate::Result { + let embedder = match self { + VectorFilterInner::Fragment { embedder_token, .. } => embedder_token, + VectorFilterInner::DocumentTemplate { embedder_token } => embedder_token, + VectorFilterInner::UserProvided { embedder_token } => embedder_token, + VectorFilterInner::FullEmbedder { embedder_token } => embedder_token, + }; + let embedder_name = embedder.value(); + let available_embedders = + || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); + + let embedding_config = embedding_configs + .iter() + .find(|config| config.name == embedder_name) + .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; + + let embedder_info = index + .embedding_configs() + .embedder_info(rtxn, embedder_name)? + .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; + + let arroy_wrapper = ArroyWrapper::new( + index.vector_arroy, + embedder_info.embedder_id, + embedding_config.config.quantized(), + ); + + let mut docids = match self { + VectorFilterInner::Fragment { embedder_token: embedder, fragment_token: fragment } => { + let fragment_name = fragment.value(); + let fragment_config = embedding_config + .fragments + .as_slice() + .iter() + .find(|fragment| fragment.name == fragment_name) + .ok_or_else(|| FragmentDoesNotExist { + embedder, + fragment, + available: embedding_config + .fragments + .as_slice() + .iter() + .map(|f| f.name.clone()) + .collect(), + })?; + + arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + } + VectorFilterInner::DocumentTemplate { .. } => { + if !embedding_config.fragments.as_slice().is_empty() { + return Ok(RoaringBitmap::new()); + } + + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents - user_provided_docsids.clone() + } + VectorFilterInner::UserProvided { .. } => { + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + user_provided_docsids.clone() + } + VectorFilterInner::FullEmbedder { .. } => { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents + } + }; + + if regenerate { + let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); + docids &= skip_regenerate; + } + + Ok(docids) + } +} + +#[derive(Debug)] pub(super) struct VectorFilter<'a> { - embedder_token: Option>, - fragment_token: Option>, - document_template: bool, - user_provided: bool, + inner: Option>, + regenerate: bool, } #[derive(Debug)] @@ -17,8 +110,7 @@ pub enum VectorFilterError<'a> { EmptyFilter, InvalidPrefix(Token<'a>), MissingFragmentName(Token<'a>), - UserProvidedWithFragment(Token<'a>), - DocumentTemplateWithFragment(Token<'a>), + ExclusiveOptions(Box<(Token<'a>, Token<'a>)>), LeftoverToken(Token<'a>), EmbedderDoesNotExist { embedder: &'a Token<'a>, @@ -51,11 +143,13 @@ impl std::fmt::Display for VectorFilterError<'_> { MissingFragmentName(_token) => { write!(f, "Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.") } - UserProvidedWithFragment(_token) => { - write!(f, "Vector filter cannot specify both a fragment name and userProvided.") - } - DocumentTemplateWithFragment(_token) => { - write!(f, "Vector filter cannot specify both a fragment name and documentTemplate.") + ExclusiveOptions(tokens) => { + write!( + f, + "Vector filter cannot have both `{}` and `{}`.", + tokens.0.value(), + tokens.1.value() + ) } LeftoverToken(token) => { write!(f, "Vector filter has leftover token: `{}`.", token.value()) @@ -107,11 +201,10 @@ impl<'a> From> for Error { fn from(err: VectorFilterError<'a>) -> Self { match &err { EmptyFilter => Error::UserError(UserError::InvalidFilter(err.to_string())), - InvalidPrefix(token) - | MissingFragmentName(token) - | UserProvidedWithFragment(token) - | DocumentTemplateWithFragment(token) - | LeftoverToken(token) => token.clone().as_external_error(err).into(), + InvalidPrefix(token) | MissingFragmentName(token) | LeftoverToken(token) => { + token.clone().as_external_error(err).into() + } + ExclusiveOptions(tokens) => tokens.1.clone().as_external_error(err).into(), EmbedderDoesNotExist { embedder: token, .. } | FragmentDoesNotExist { fragment: token, .. } => token.as_external_error(err).into(), } @@ -127,11 +220,15 @@ impl<'a> VectorFilter<'a> { /// /// Valid formats: /// - `_vectors` + /// - `_vectors.mustRegenerate` /// - `_vectors.{embedder_name}` + /// - `_vectors.{embedder_name}.mustRegenerate` /// - `_vectors.{embedder_name}.userProvided` + /// - `_vectors.{embedder_name}.userProvided.mustRegenerate` /// - `_vectors.{embedder_name}.documentTemplate` - /// - `_vectors.{embedder_name}.documentTemplate.userProvided` + /// - `_vectors.{embedder_name}.documentTemplate.mustRegenerate` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` + /// - `_vectors.{embedder_name}.fragments.{fragment_name}.mustRegenerate` pub(super) fn parse(s: &'a Token<'a>) -> Result> { let mut split = s.split(".").peekable(); @@ -151,38 +248,46 @@ impl<'a> VectorFilter<'a> { } let mut user_provided_token = None; - if split.peek().map(|t| t.value()) == Some("userProvided") - || split.peek().map(|t| t.value()) == Some("user_provided") - { + if split.peek().map(|t| t.value()) == Some("userProvided") { user_provided_token = split.next(); } let mut document_template_token = None; - if split.peek().map(|t| t.value()) == Some("documentTemplate") - || split.peek().map(|t| t.value()) == Some("document_template") - { + if split.peek().map(|t| t.value()) == Some("documentTemplate") { document_template_token = split.next(); } - if let (Some(_), Some(user_provided_token)) = (&fragment_name, &user_provided_token) { - return Err(UserProvidedWithFragment(user_provided_token.clone()))?; + let mut regenerate_token = None; + if split.peek().map(|t| t.value()) == Some("regenerate") { + regenerate_token = split.next(); } - if let (Some(_), Some(document_template_token)) = (&fragment_name, &document_template_token) - { - return Err(DocumentTemplateWithFragment(document_template_token.clone()))?; - } + let inner = match (fragment_name, user_provided_token, document_template_token) { + (Some(fragment_name), None, None) => Some(VectorFilterInner::Fragment { + embedder_token: embedder_name + .expect("embedder name comes before fragment so it's always Some"), + fragment_token: fragment_name, + }), + (None, Some(_), None) => Some(VectorFilterInner::UserProvided { + embedder_token: embedder_name + .expect("embedder name comes before userProvided so it's always Some"), + }), + (None, None, Some(_)) => Some(VectorFilterInner::DocumentTemplate { + embedder_token: embedder_name + .expect("embedder name comes before documentTemplate so it's always Some"), + }), + (Some(a), Some(b), _) | (_, Some(a), Some(b)) | (Some(a), None, Some(b)) => { + return Err(ExclusiveOptions(Box::new((a, b)))); + } + (None, None, None) => embedder_name + .map(|embedder_token| VectorFilterInner::FullEmbedder { embedder_token }), + }; if let Some(next) = split.next() { return Err(LeftoverToken(next))?; } - Ok(Self { - embedder_token: embedder_name, - fragment_token: fragment_name, - user_provided: user_provided_token.is_some(), - document_template: document_template_token.is_some(), - }) + Ok(Self { inner, regenerate: regenerate_token.is_some() }) } pub(super) fn evaluate( @@ -194,100 +299,19 @@ impl<'a> VectorFilter<'a> { let index_embedding_configs = index.embedding_configs(); let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; - let mut embedders = Vec::new(); - if let Some(embedder_token) = &self.embedder_token { - let embedder_name = embedder_token.value(); - - let Some(embedding_config) = - embedding_configs.iter().find(|config| config.name == embedder_name) - else { - return Err(EmbedderDoesNotExist { - embedder: embedder_token, - available: embedding_configs.iter().map(|c| c.name.clone()).collect(), - })?; - }; - - let Some(embedder_info) = index_embedding_configs.embedder_info(rtxn, embedder_name)? - else { - return Err(EmbedderDoesNotExist { - embedder: embedder_token, - available: embedding_configs.iter().map(|c| c.name.clone()).collect(), - })?; - }; - - if self.document_template && !embedding_config.fragments.as_slice().is_empty() { - return Ok(RoaringBitmap::new()); - } - - embedders.push((embedding_config, embedder_info)); - } else { - for embedder_config in embedding_configs.iter() { - let Some(embedder_info) = - index_embedding_configs.embedder_info(rtxn, &embedder_config.name)? - else { - continue; - }; - embedders.push((embedder_config, embedder_info)); - } - }; + let inners = dbg!(match self.inner { + Some(inner) => vec![inner], + None => embedding_configs + .iter() + .map(|config| VectorFilterInner::FullEmbedder { + embedder_token: Token::from(config.name.as_str()), + }) + .collect(), + }); let mut docids = RoaringBitmap::new(); - for (embedding_config, embedder_info) in embedders { - let arroy_wrapper = ArroyWrapper::new( - index.vector_arroy, - embedder_info.embedder_id, - embedding_config.config.quantized(), - ); - - docids |= if let Some(fragment_token) = &self.fragment_token { - let fragment_name = fragment_token.value(); - let Some(fragment_config) = embedding_config - .fragments - .as_slice() - .iter() - .find(|fragment| fragment.name == fragment_name) - else { - return Err(FragmentDoesNotExist { - embedder: self - .embedder_token - .as_ref() - .expect("there can't be a fragment without an embedder"), - fragment: fragment_token, - available: embedding_config - .fragments - .as_slice() - .iter() - .map(|f| f.name.clone()) - .collect(), - })?; - }; - - if let Some(universe) = universe { - arroy_wrapper - .items_in_store(rtxn, fragment_config.id, |bitmap| bitmap & universe)? - } else { - arroy_wrapper - .items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? - } - } else { - let mut universe = universe.cloned(); - if self.user_provided { - let user_provided_docsids = - embedder_info.embedding_status.user_provided_docids(); - match &mut universe { - Some(universe) => *universe &= user_provided_docsids, - None => universe = Some(user_provided_docsids.clone()), - } - } - - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - if let Some(universe) = &universe { - stats.documents & universe - } else { - stats.documents - } - }; + for inner in inners.iter() { + docids |= inner.evaluate_inner(rtxn, index, &embedding_configs, self.regenerate)?; } if let Some(universe) = universe { diff --git a/crates/milli/src/vector/db.rs b/crates/milli/src/vector/db.rs index 2fea75d68..d445b47c0 100644 --- a/crates/milli/src/vector/db.rs +++ b/crates/milli/src/vector/db.rs @@ -128,6 +128,7 @@ impl EmbeddingStatus { pub fn is_user_provided(&self, docid: DocumentId) -> bool { self.user_provided.contains(docid) } + /// Whether vectors should be regenerated for that document and that embedder. pub fn must_regenerate(&self, docid: DocumentId) -> bool { let invert = self.skip_regenerate_different_from_user_provided.contains(docid); From 982e9898861f4e29b0409cf3a51e610ba308cdc5 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 11:10:05 +0200 Subject: [PATCH 173/312] Test regenerate filter --- crates/meilisearch/tests/search/filters.rs | 36 +++++++++++++++++++ .../milli/src/search/facet/filter_vector.rs | 11 +++--- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 4701e986d..1a27bdf99 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -1151,3 +1151,39 @@ async fn vector_filter_or_combination() { } "#); } + + +#[actix_rt::test] +async fn vector_filter_regenerate() { + let index = shared_index_for_fragments().await; + + for selector in ["_vectors.rest.regenerate", "_vectors.rest.fragments.basic.regenerate"] { + let (value, _code) = index + .search_post(json!({ + "filter": format!("{selector} EXISTS"), + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "hits": [ + { + "name": "kefir" + }, + { + "name": "intel" + }, + { + "name": "dustin" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3 + } + "#); + } +} + diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 3a0f86637..7fbd9c916 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -92,7 +92,7 @@ impl VectorFilterInner<'_> { if regenerate { let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); - docids &= skip_regenerate; + docids -= skip_regenerate; } Ok(docids) @@ -220,15 +220,14 @@ impl<'a> VectorFilter<'a> { /// /// Valid formats: /// - `_vectors` - /// - `_vectors.mustRegenerate` /// - `_vectors.{embedder_name}` - /// - `_vectors.{embedder_name}.mustRegenerate` + /// - `_vectors.{embedder_name}.regenerate` /// - `_vectors.{embedder_name}.userProvided` - /// - `_vectors.{embedder_name}.userProvided.mustRegenerate` + /// - `_vectors.{embedder_name}.userProvided.regenerate` /// - `_vectors.{embedder_name}.documentTemplate` - /// - `_vectors.{embedder_name}.documentTemplate.mustRegenerate` + /// - `_vectors.{embedder_name}.documentTemplate.regenerate` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` - /// - `_vectors.{embedder_name}.fragments.{fragment_name}.mustRegenerate` + /// - `_vectors.{embedder_name}.fragments.{fragment_name}.regenerate` pub(super) fn parse(s: &'a Token<'a>) -> Result> { let mut split = s.split(".").peekable(); From 6d93b36279c75901a637fe8a4aaf69461274286e Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 11:18:41 +0200 Subject: [PATCH 174/312] Format --- crates/meilisearch/tests/search/filters.rs | 14 +- .../milli/src/search/facet/filter_vector.rs | 278 ++++++++---------- 2 files changed, 122 insertions(+), 170 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 1a27bdf99..12bfbe2ea 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -1152,19 +1152,18 @@ async fn vector_filter_or_combination() { "#); } - #[actix_rt::test] async fn vector_filter_regenerate() { let index = shared_index_for_fragments().await; for selector in ["_vectors.rest.regenerate", "_vectors.rest.fragments.basic.regenerate"] { let (value, _code) = index - .search_post(json!({ - "filter": format!("{selector} EXISTS"), - "attributesToRetrieve": ["name"] - })) - .await; - snapshot!(value, @r#" + .search_post(json!({ + "filter": format!("{selector} EXISTS"), + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" { "hits": [ { @@ -1186,4 +1185,3 @@ async fn vector_filter_regenerate() { "#); } } - diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 7fbd9c916..a59bbb5f9 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -14,108 +14,49 @@ enum VectorFilterInner<'a> { FullEmbedder { embedder_token: Token<'a> }, } -impl VectorFilterInner<'_> { - fn evaluate_inner( - &self, - rtxn: &heed::RoTxn<'_>, - index: &Index, - embedding_configs: &[IndexEmbeddingConfig], - regenerate: bool, - ) -> crate::Result { - let embedder = match self { - VectorFilterInner::Fragment { embedder_token, .. } => embedder_token, - VectorFilterInner::DocumentTemplate { embedder_token } => embedder_token, - VectorFilterInner::UserProvided { embedder_token } => embedder_token, - VectorFilterInner::FullEmbedder { embedder_token } => embedder_token, - }; - let embedder_name = embedder.value(); - let available_embedders = - || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); - - let embedding_config = embedding_configs - .iter() - .find(|config| config.name == embedder_name) - .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; - - let embedder_info = index - .embedding_configs() - .embedder_info(rtxn, embedder_name)? - .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; - - let arroy_wrapper = ArroyWrapper::new( - index.vector_arroy, - embedder_info.embedder_id, - embedding_config.config.quantized(), - ); - - let mut docids = match self { - VectorFilterInner::Fragment { embedder_token: embedder, fragment_token: fragment } => { - let fragment_name = fragment.value(); - let fragment_config = embedding_config - .fragments - .as_slice() - .iter() - .find(|fragment| fragment.name == fragment_name) - .ok_or_else(|| FragmentDoesNotExist { - embedder, - fragment, - available: embedding_config - .fragments - .as_slice() - .iter() - .map(|f| f.name.clone()) - .collect(), - })?; - - arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? - } - VectorFilterInner::DocumentTemplate { .. } => { - if !embedding_config.fragments.as_slice().is_empty() { - return Ok(RoaringBitmap::new()); - } - - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents - user_provided_docsids.clone() - } - VectorFilterInner::UserProvided { .. } => { - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - user_provided_docsids.clone() - } - VectorFilterInner::FullEmbedder { .. } => { - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents - } - }; - - if regenerate { - let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); - docids -= skip_regenerate; - } - - Ok(docids) - } -} - #[derive(Debug)] pub(super) struct VectorFilter<'a> { inner: Option>, regenerate: bool, } -#[derive(Debug)] +#[derive(Debug, thiserror::Error)] pub enum VectorFilterError<'a> { + #[error("Vector filter cannot be empty.")] EmptyFilter, + + #[error("Vector filter must start with `_vectors` but found `{}`.", _0.value())] InvalidPrefix(Token<'a>), + + #[error("Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.")] MissingFragmentName(Token<'a>), + + #[error("Vector filter cannot have both `{}` and `{}`.", _0.0.value(), _0.1.value())] ExclusiveOptions(Box<(Token<'a>, Token<'a>)>), + + #[error("Vector filter has leftover token: `{}`.", _0.value())] LeftoverToken(Token<'a>), - EmbedderDoesNotExist { - embedder: &'a Token<'a>, - available: Vec, - }, + + #[error("The embedder `{}` does not exist. {}", embedder.value(), { + if available.is_empty() { + String::from("This index does not have any configured embedders.") + } else { + let mut available = available.clone(); + available.sort_unstable(); + format!("Available embedders are: {}.", available.iter().map(|e| format!("`{e}`")).collect::>().join(", ")) + } + })] + EmbedderDoesNotExist { embedder: &'a Token<'a>, available: Vec }, + + #[error("The fragment `{}` does not exist on embedder `{}`. {}", fragment.value(), embedder.value(), { + if available.is_empty() { + String::from("This embedder does not have any configured fragments.") + } else { + let mut available = available.clone(); + available.sort_unstable(); + format!("Available fragments on this embedder are: {}.", available.iter().map(|f| format!("`{f}`")).collect::>().join(", ")) + } + })] FragmentDoesNotExist { embedder: &'a Token<'a>, fragment: &'a Token<'a>, @@ -125,78 +66,6 @@ pub enum VectorFilterError<'a> { use VectorFilterError::*; -impl std::error::Error for VectorFilterError<'_> {} - -impl std::fmt::Display for VectorFilterError<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - EmptyFilter => { - write!(f, "Vector filter cannot be empty.") - } - InvalidPrefix(prefix) => { - write!( - f, - "Vector filter must start with `_vectors` but found `{}`.", - prefix.value() - ) - } - MissingFragmentName(_token) => { - write!(f, "Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.") - } - ExclusiveOptions(tokens) => { - write!( - f, - "Vector filter cannot have both `{}` and `{}`.", - tokens.0.value(), - tokens.1.value() - ) - } - LeftoverToken(token) => { - write!(f, "Vector filter has leftover token: `{}`.", token.value()) - } - EmbedderDoesNotExist { embedder, available } => { - write!(f, "The embedder `{}` does not exist.", embedder.value())?; - if available.is_empty() { - write!(f, " This index does not have configured embedders.") - } else { - write!(f, " Available embedders are: ")?; - let mut available = available.clone(); - available.sort_unstable(); - for (idx, embedder) in available.iter().enumerate() { - write!(f, "`{embedder}`")?; - if idx != available.len() - 1 { - write!(f, ", ")?; - } - } - write!(f, ".") - } - } - FragmentDoesNotExist { embedder, fragment, available } => { - write!( - f, - "The fragment `{}` does not exist on embedder `{}`.", - fragment.value(), - embedder.value(), - )?; - if available.is_empty() { - write!(f, " This embedder does not have configured fragments.") - } else { - write!(f, " Available fragments on this embedder are: ")?; - let mut available = available.clone(); - available.sort_unstable(); - for (idx, fragment) in available.iter().enumerate() { - write!(f, "`{fragment}`")?; - if idx != available.len() - 1 { - write!(f, ", ")?; - } - } - write!(f, ".") - } - } - } - } -} - impl<'a> From> for Error { fn from(err: VectorFilterError<'a>) -> Self { match &err { @@ -320,3 +189,88 @@ impl<'a> VectorFilter<'a> { Ok(docids) } } + +impl VectorFilterInner<'_> { + fn evaluate_inner( + &self, + rtxn: &heed::RoTxn<'_>, + index: &Index, + embedding_configs: &[IndexEmbeddingConfig], + regenerate: bool, + ) -> crate::Result { + let embedder = match self { + VectorFilterInner::Fragment { embedder_token, .. } => embedder_token, + VectorFilterInner::DocumentTemplate { embedder_token } => embedder_token, + VectorFilterInner::UserProvided { embedder_token } => embedder_token, + VectorFilterInner::FullEmbedder { embedder_token } => embedder_token, + }; + let embedder_name = embedder.value(); + let available_embedders = + || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); + + let embedding_config = embedding_configs + .iter() + .find(|config| config.name == embedder_name) + .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; + + let embedder_info = index + .embedding_configs() + .embedder_info(rtxn, embedder_name)? + .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; + + let arroy_wrapper = ArroyWrapper::new( + index.vector_arroy, + embedder_info.embedder_id, + embedding_config.config.quantized(), + ); + + let mut docids = match self { + VectorFilterInner::Fragment { embedder_token: embedder, fragment_token: fragment } => { + let fragment_name = fragment.value(); + let fragment_config = embedding_config + .fragments + .as_slice() + .iter() + .find(|fragment| fragment.name == fragment_name) + .ok_or_else(|| FragmentDoesNotExist { + embedder, + fragment, + available: embedding_config + .fragments + .as_slice() + .iter() + .map(|f| f.name.clone()) + .collect(), + })?; + + arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + } + VectorFilterInner::DocumentTemplate { .. } => { + if !embedding_config.fragments.as_slice().is_empty() { + return Ok(RoaringBitmap::new()); + } + + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents - user_provided_docsids.clone() + } + VectorFilterInner::UserProvided { .. } => { + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + user_provided_docsids.clone() + } + VectorFilterInner::FullEmbedder { .. } => { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents + } + }; + + if regenerate { + let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); + docids -= skip_regenerate; + } + + Ok(docids) + } +} From 3362fb8476860efa36fb6a1b9fdf296e47b42d11 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 11:21:06 +0200 Subject: [PATCH 175/312] Remove print --- crates/milli/src/search/facet/filter_vector.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index a59bbb5f9..e2c00a16a 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -167,7 +167,7 @@ impl<'a> VectorFilter<'a> { let index_embedding_configs = index.embedding_configs(); let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; - let inners = dbg!(match self.inner { + let inners = match self.inner { Some(inner) => vec![inner], None => embedding_configs .iter() @@ -175,7 +175,7 @@ impl<'a> VectorFilter<'a> { embedder_token: Token::from(config.name.as_str()), }) .collect(), - }); + }; let mut docids = RoaringBitmap::new(); for inner in inners.iter() { From 776e55d2096febba707582273243721eebb7b1dc Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 11:37:21 +0200 Subject: [PATCH 176/312] Improve code readability --- crates/milli/src/search/facet/filter_vector.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index e2c00a16a..83b3adcc3 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -1,5 +1,5 @@ use filter_parser::Token; -use roaring::RoaringBitmap; +use roaring::{MultiOps, RoaringBitmap}; use crate::error::{Error, UserError}; use crate::vector::db::IndexEmbeddingConfig; @@ -177,10 +177,10 @@ impl<'a> VectorFilter<'a> { .collect(), }; - let mut docids = RoaringBitmap::new(); - for inner in inners.iter() { - docids |= inner.evaluate_inner(rtxn, index, &embedding_configs, self.regenerate)?; - } + let mut docids = inners + .iter() + .map(|i| i.evaluate_inner(rtxn, index, &embedding_configs, self.regenerate)) + .union()?; if let Some(universe) = universe { docids &= universe; From 2f2e42e72db0b81ad31510fc703c4d216bb424d5 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 12:33:18 +0200 Subject: [PATCH 177/312] Add test for issue #4653 --- crates/meilisearch/tests/snapshot/mod.rs | 49 +++++++++++++++++++++--- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/tests/snapshot/mod.rs b/crates/meilisearch/tests/snapshot/mod.rs index 32946b06e..b4e3f152c 100644 --- a/crates/meilisearch/tests/snapshot/mod.rs +++ b/crates/meilisearch/tests/snapshot/mod.rs @@ -122,11 +122,7 @@ async fn perform_on_demand_snapshot() { let server = Server::new_with_options(options).await.unwrap(); let index = server.index("catto"); - index - .update_settings(json! ({ - "searchableAttributes": [], - })) - .await; + index.update_settings(json! ({ "searchableAttributes": [] })).await; index.load_test_set(&server).await; @@ -203,3 +199,46 @@ async fn perform_on_demand_snapshot() { server.index("doggo").settings(), ); } + +#[actix_rt::test] +#[cfg_attr(target_os = "windows", ignore)] +async fn snapshotception_issue_4653() { + let temp = tempfile::tempdir().unwrap(); + let snapshot_dir = tempfile::tempdir().unwrap(); + let options = + Opt { snapshot_dir: snapshot_dir.path().to_owned(), ..default_settings(temp.path()) }; + + let server = Server::new_with_options(options).await.unwrap(); + + let (task, code) = server.create_snapshot().await; + snapshot!(code, @"202 Accepted"); + snapshot!(json_string!(task, { ".enqueuedAt" => "[date]" }), @r###" + { + "taskUid": 0, + "indexUid": null, + "status": "enqueued", + "type": "snapshotCreation", + "enqueuedAt": "[date]" + } + "###); + let task = server.wait_task(task.uid()).await.succeeded(); + + let temp = tempfile::tempdir().unwrap(); + let snapshot_path = snapshot_dir.path().to_owned().join("db.snapshot"); + + let options = Opt { import_snapshot: Some(snapshot_path), ..default_settings(temp.path()) }; + let snapshot_server = Server::new_with_options(options).await.unwrap(); + + // The snapshot creation task should NOT be spawned => task queue is empty + let (tasks, code) = snapshot_server.tasks().await; + snapshot!(code, @"200 OK"); + snapshot!(tasks, @r#" + { + "results": [], + "total": 0, + "limit": 20, + "from": null, + "next": null + } + "#); +} From 971683438028e3ad4bdc99c0066481c9be70cb77 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 14:31:42 +0200 Subject: [PATCH 178/312] Initial fix --- .../src/scheduler/process_snapshot_creation.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs index d58157ae3..431555904 100644 --- a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs @@ -6,6 +6,7 @@ use meilisearch_types::heed::CompactionOption; use meilisearch_types::milli::progress::{Progress, VariableNameStep}; use meilisearch_types::tasks::{Status, Task}; use meilisearch_types::{compression, VERSION_FILE_NAME}; +use roaring::RoaringBitmap; use crate::processing::{AtomicUpdateFileStep, SnapshotCreationProgress}; use crate::{Error, IndexScheduler, Result}; @@ -38,6 +39,10 @@ impl IndexScheduler { // two read operations as the task processing is synchronous. // 2.1 First copy the LMDB env of the index-scheduler + // + // Note that just before we copy it, we set the status of the current tasks to Succeeded. + // This is because when the snapshot is loaded in the future, we don't want these tasks to rerun. + // In any case, if the snapshot can be loaded, it means that the tasks did succeed. progress.update_progress(SnapshotCreationProgress::SnapshotTheIndexScheduler); let dst = temp_snapshot_dir.path().join("tasks"); fs::create_dir_all(&dst)?; @@ -46,7 +51,20 @@ impl IndexScheduler { } else { CompactionOption::Enabled }; + + let mut wtxn = self.env.write_txn()?; + for task in &mut tasks { + task.status = Status::Succeeded; + self.queue.tasks.update_task(&mut wtxn, task)?; + } + wtxn.commit()?; self.env.copy_to_path(dst.join("data.mdb"), compaction_option)?; + let mut wtxn = self.scheduler.auth_env.write_txn()?; + for task in &mut tasks { + task.status = Status::Enqueued; + self.queue.tasks.update_task(&mut wtxn, task)?; + } + wtxn.commit()?; // 2.2 Create a read transaction on the index-scheduler let rtxn = self.env.read_txn()?; From 6394efc4c25f8ec0ab13f880687d03e18aeb9a5a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 15:17:26 +0200 Subject: [PATCH 179/312] Turn dirty fix into beautiful fix --- .../index-scheduler/src/index_mapper/mod.rs | 2 +- .../scheduler/process_snapshot_creation.rs | 65 +++++++++++++------ 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/crates/index-scheduler/src/index_mapper/mod.rs b/crates/index-scheduler/src/index_mapper/mod.rs index 86fb17ca7..e6bdccd41 100644 --- a/crates/index-scheduler/src/index_mapper/mod.rs +++ b/crates/index-scheduler/src/index_mapper/mod.rs @@ -71,7 +71,7 @@ pub struct IndexMapper { /// Path to the folder where the LMDB environments of each index are. base_path: PathBuf, /// The map size an index is opened with on the first time. - index_base_map_size: usize, + pub(crate) index_base_map_size: usize, /// The quantity by which the map size of an index is incremented upon reopening, in bytes. index_growth_amount: usize, /// Whether we open a meilisearch index with the MDB_WRITEMAP option or not. diff --git a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs index 431555904..b9c4329ff 100644 --- a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs @@ -6,11 +6,34 @@ use meilisearch_types::heed::CompactionOption; use meilisearch_types::milli::progress::{Progress, VariableNameStep}; use meilisearch_types::tasks::{Status, Task}; use meilisearch_types::{compression, VERSION_FILE_NAME}; -use roaring::RoaringBitmap; +use crate::heed::EnvOpenOptions; use crate::processing::{AtomicUpdateFileStep, SnapshotCreationProgress}; +use crate::queue::TaskQueue; use crate::{Error, IndexScheduler, Result}; +/// # Safety +/// +/// See [`EnvOpenOptions::open`]. +unsafe fn mark_tasks_as_succeeded( + tasks: &[Task], + dst: &std::path::Path, + nb_db: u32, + index_base_map_size: usize, +) -> Result<()> { + let env_options = EnvOpenOptions::new(); + let mut env_options = env_options.read_txn_without_tls(); + let env = env_options.max_dbs(nb_db).map_size(index_base_map_size).open(dst)?; + let mut wtxn = env.write_txn()?; + let task_queue = TaskQueue::new(&env, &mut wtxn)?; + for mut task in tasks.iter().cloned() { + task.status = Status::Succeeded; + task_queue.update_task(&mut wtxn, &task)?; + } + wtxn.commit()?; + Ok(()) +} + impl IndexScheduler { pub(super) fn process_snapshot( &self, @@ -39,10 +62,6 @@ impl IndexScheduler { // two read operations as the task processing is synchronous. // 2.1 First copy the LMDB env of the index-scheduler - // - // Note that just before we copy it, we set the status of the current tasks to Succeeded. - // This is because when the snapshot is loaded in the future, we don't want these tasks to rerun. - // In any case, if the snapshot can be loaded, it means that the tasks did succeed. progress.update_progress(SnapshotCreationProgress::SnapshotTheIndexScheduler); let dst = temp_snapshot_dir.path().join("tasks"); fs::create_dir_all(&dst)?; @@ -51,29 +70,33 @@ impl IndexScheduler { } else { CompactionOption::Enabled }; - - let mut wtxn = self.env.write_txn()?; - for task in &mut tasks { - task.status = Status::Succeeded; - self.queue.tasks.update_task(&mut wtxn, task)?; - } - wtxn.commit()?; self.env.copy_to_path(dst.join("data.mdb"), compaction_option)?; - let mut wtxn = self.scheduler.auth_env.write_txn()?; - for task in &mut tasks { - task.status = Status::Enqueued; - self.queue.tasks.update_task(&mut wtxn, task)?; - } - wtxn.commit()?; - // 2.2 Create a read transaction on the index-scheduler + // 2.2 Mark the current snapshot tasks as succeeded in the newly created env + // + // This is done to ensure that the tasks are not processed again when the snapshot is imported + // + // # Safety + // + // This is safe because we open the env file we just created in a temporary directory. + // We are sure it's not being used by any other process nor thread. + unsafe { + mark_tasks_as_succeeded( + &tasks, + &dst, + Self::nb_db(), + self.index_mapper.index_base_map_size, + )?; + } + + // 2.3 Create a read transaction on the index-scheduler let rtxn = self.env.read_txn()?; - // 2.3 Create the update files directory + // 2.4 Create the update files directory let update_files_dir = temp_snapshot_dir.path().join("update_files"); fs::create_dir_all(&update_files_dir)?; - // 2.4 Only copy the update files of the enqueued tasks + // 2.5 Only copy the update files of the enqueued tasks progress.update_progress(SnapshotCreationProgress::SnapshotTheUpdateFiles); let enqueued = self.queue.tasks.get_status(&rtxn, Status::Enqueued)?; let (atomic, update_file_progress) = AtomicUpdateFileStep::new(enqueued.len() as u32); From c1aa4120ac63d319f28de56e98ff686762acbc1c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 15:18:13 +0200 Subject: [PATCH 180/312] Update test --- crates/meilisearch/tests/snapshot/mod.rs | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/crates/meilisearch/tests/snapshot/mod.rs b/crates/meilisearch/tests/snapshot/mod.rs index b4e3f152c..08623f869 100644 --- a/crates/meilisearch/tests/snapshot/mod.rs +++ b/crates/meilisearch/tests/snapshot/mod.rs @@ -221,7 +221,7 @@ async fn snapshotception_issue_4653() { "enqueuedAt": "[date]" } "###); - let task = server.wait_task(task.uid()).await.succeeded(); + server.wait_task(task.uid()).await.succeeded(); let temp = tempfile::tempdir().unwrap(); let snapshot_path = snapshot_dir.path().to_owned().join("db.snapshot"); @@ -229,16 +229,22 @@ async fn snapshotception_issue_4653() { let options = Opt { import_snapshot: Some(snapshot_path), ..default_settings(temp.path()) }; let snapshot_server = Server::new_with_options(options).await.unwrap(); - // The snapshot creation task should NOT be spawned => task queue is empty - let (tasks, code) = snapshot_server.tasks().await; + // The snapshot creation task should NOT be spawned again => task is succeeded + let (task, code) = snapshot_server.get_task(task.uid()).await; snapshot!(code, @"200 OK"); - snapshot!(tasks, @r#" + snapshot!(json_string!(task, { ".enqueuedAt" => "[date]" }), @r#" { - "results": [], - "total": 0, - "limit": 20, - "from": null, - "next": null + "uid": 0, + "batchUid": 0, + "indexUid": null, + "status": "succeeded", + "type": "snapshotCreation", + "canceledBy": null, + "error": null, + "duration": null, + "enqueuedAt": "[date]", + "startedAt": null, + "finishedAt": null } "#); } From 846d27354bd69a967d00e3e5d4af5ed3324453df Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 22 Jul 2025 15:18:21 +0200 Subject: [PATCH 181/312] Format --- crates/meilisearch/tests/snapshot/mod.rs | 2 +- crates/milli/src/update/facet/mod.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/snapshot/mod.rs b/crates/meilisearch/tests/snapshot/mod.rs index 08623f869..a3c78bf28 100644 --- a/crates/meilisearch/tests/snapshot/mod.rs +++ b/crates/meilisearch/tests/snapshot/mod.rs @@ -228,7 +228,7 @@ async fn snapshotception_issue_4653() { let options = Opt { import_snapshot: Some(snapshot_path), ..default_settings(temp.path()) }; let snapshot_server = Server::new_with_options(options).await.unwrap(); - + // The snapshot creation task should NOT be spawned again => task is succeeded let (task, code) = snapshot_server.get_task(task.uid()).await; snapshot!(code, @"200 OK"); diff --git a/crates/milli/src/update/facet/mod.rs b/crates/milli/src/update/facet/mod.rs index c40916670..71596530e 100644 --- a/crates/milli/src/update/facet/mod.rs +++ b/crates/milli/src/update/facet/mod.rs @@ -119,6 +119,7 @@ pub struct FacetsUpdate<'i> { min_level_size: u8, data_size: u64, } + impl<'i> FacetsUpdate<'i> { pub fn new( index: &'i Index, From 5dcf79233e8dc6ceb7a3f7004a4a0c0e3d2c428c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 23 Jul 2025 11:30:39 +0200 Subject: [PATCH 182/312] Remove useless parameter Co-Authored-By: Tamo --- .../src/scheduler/process_snapshot_creation.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs index b9c4329ff..0859974cc 100644 --- a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs @@ -18,12 +18,11 @@ use crate::{Error, IndexScheduler, Result}; unsafe fn mark_tasks_as_succeeded( tasks: &[Task], dst: &std::path::Path, - nb_db: u32, index_base_map_size: usize, ) -> Result<()> { let env_options = EnvOpenOptions::new(); let mut env_options = env_options.read_txn_without_tls(); - let env = env_options.max_dbs(nb_db).map_size(index_base_map_size).open(dst)?; + let env = env_options.max_dbs(TaskQueue::nb_db()).map_size(index_base_map_size).open(dst)?; let mut wtxn = env.write_txn()?; let task_queue = TaskQueue::new(&env, &mut wtxn)?; for mut task in tasks.iter().cloned() { @@ -84,7 +83,6 @@ impl IndexScheduler { mark_tasks_as_succeeded( &tasks, &dst, - Self::nb_db(), self.index_mapper.index_base_map_size, )?; } From 44b24652d272dadb676c1422cc8ab13a606b0052 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 23 Jul 2025 14:30:25 +0200 Subject: [PATCH 183/312] Change strategy to remove task instead of marking it succeeded --- .../scheduler/process_snapshot_creation.rs | 53 ++++++++++++++++--- crates/meilisearch/tests/snapshot/mod.rs | 46 +++++++++++----- 2 files changed, 77 insertions(+), 22 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs index 0859974cc..3b46d0359 100644 --- a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs @@ -25,9 +25,50 @@ unsafe fn mark_tasks_as_succeeded( let env = env_options.max_dbs(TaskQueue::nb_db()).map_size(index_base_map_size).open(dst)?; let mut wtxn = env.write_txn()?; let task_queue = TaskQueue::new(&env, &mut wtxn)?; - for mut task in tasks.iter().cloned() { - task.status = Status::Succeeded; - task_queue.update_task(&mut wtxn, &task)?; + + // Destructuring to ensure the code below gets updated if a database gets added in the future. + let TaskQueue { + all_tasks, + status, + kind, + index_tasks: _, // snapshot creation tasks are not index tasks + canceled_by, + enqueued_at, + started_at, + finished_at, + } = task_queue; + + for task in tasks { + all_tasks.delete(&mut wtxn, &task.uid)?; + + let mut tasks = status.get(&wtxn, &task.status)?.unwrap_or_default(); + tasks.remove(task.uid); + status.put(&mut wtxn, &task.status, &tasks)?; + + let mut tasks = kind.get(&wtxn, &task.kind.as_kind())?.unwrap_or_default(); + tasks.remove(task.uid); + kind.put(&mut wtxn, &task.kind.as_kind(), &tasks)?; + + canceled_by.delete(&mut wtxn, &task.uid)?; + + let timestamp = task.enqueued_at.unix_timestamp_nanos(); + let mut tasks = enqueued_at.get(&wtxn, ×tamp)?.unwrap_or_default(); + tasks.remove(task.uid); + enqueued_at.put(&mut wtxn, ×tamp, &tasks)?; + + if let Some(task_started_at) = task.started_at { + let timestamp = task_started_at.unix_timestamp_nanos(); + let mut tasks = started_at.get(&wtxn, ×tamp)?.unwrap_or_default(); + tasks.remove(task.uid); + started_at.put(&mut wtxn, ×tamp, &tasks)?; + } + + if let Some(task_finished_at) = task.finished_at { + let timestamp = task_finished_at.unix_timestamp_nanos(); + let mut tasks = finished_at.get(&wtxn, ×tamp)?.unwrap_or_default(); + tasks.remove(task.uid); + finished_at.put(&mut wtxn, ×tamp, &tasks)?; + } } wtxn.commit()?; Ok(()) @@ -80,11 +121,7 @@ impl IndexScheduler { // This is safe because we open the env file we just created in a temporary directory. // We are sure it's not being used by any other process nor thread. unsafe { - mark_tasks_as_succeeded( - &tasks, - &dst, - self.index_mapper.index_base_map_size, - )?; + mark_tasks_as_succeeded(&tasks, &dst, self.index_mapper.index_base_map_size)?; } // 2.3 Create a read transaction on the index-scheduler diff --git a/crates/meilisearch/tests/snapshot/mod.rs b/crates/meilisearch/tests/snapshot/mod.rs index a3c78bf28..98ce17b80 100644 --- a/crates/meilisearch/tests/snapshot/mod.rs +++ b/crates/meilisearch/tests/snapshot/mod.rs @@ -229,22 +229,40 @@ async fn snapshotception_issue_4653() { let options = Opt { import_snapshot: Some(snapshot_path), ..default_settings(temp.path()) }; let snapshot_server = Server::new_with_options(options).await.unwrap(); - // The snapshot creation task should NOT be spawned again => task is succeeded - let (task, code) = snapshot_server.get_task(task.uid()).await; + // The snapshot should have been taken without the snapshot creation task + let (tasks, code) = snapshot_server.tasks().await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(task, { ".enqueuedAt" => "[date]" }), @r#" + snapshot!(tasks, @r#" { - "uid": 0, - "batchUid": 0, - "indexUid": null, - "status": "succeeded", - "type": "snapshotCreation", - "canceledBy": null, - "error": null, - "duration": null, - "enqueuedAt": "[date]", - "startedAt": null, - "finishedAt": null + "results": [], + "total": 0, + "limit": 20, + "from": null, + "next": null + } + "#); + + // Ensure the task is not present in the snapshot + let (task, code) = snapshot_server.get_task(0).await; + snapshot!(code, @"404 Not Found"); + snapshot!(task, @r#" + { + "message": "Task `0` not found.", + "code": "task_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#task_not_found" + } + "#); + + // Ensure the batch is also not present + let (batch, code) = snapshot_server.get_batch(0).await; + snapshot!(code, @"404 Not Found"); + snapshot!(batch, @r#" + { + "message": "Batch `0` not found.", + "code": "batch_not_found", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#batch_not_found" } "#); } From 1f18f0ba77e6abb9224e506a8d823815b151a2ea Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 23 Jul 2025 14:33:58 +0200 Subject: [PATCH 184/312] Update little tiny comments --- .../src/scheduler/process_snapshot_creation.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs index 3b46d0359..4a7a9e074 100644 --- a/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_snapshot_creation.rs @@ -15,7 +15,7 @@ use crate::{Error, IndexScheduler, Result}; /// # Safety /// /// See [`EnvOpenOptions::open`]. -unsafe fn mark_tasks_as_succeeded( +unsafe fn remove_tasks( tasks: &[Task], dst: &std::path::Path, index_base_map_size: usize, @@ -112,7 +112,7 @@ impl IndexScheduler { }; self.env.copy_to_path(dst.join("data.mdb"), compaction_option)?; - // 2.2 Mark the current snapshot tasks as succeeded in the newly created env + // 2.2 Remove the current snapshot tasks // // This is done to ensure that the tasks are not processed again when the snapshot is imported // @@ -121,7 +121,7 @@ impl IndexScheduler { // This is safe because we open the env file we just created in a temporary directory. // We are sure it's not being used by any other process nor thread. unsafe { - mark_tasks_as_succeeded(&tasks, &dst, self.index_mapper.index_base_map_size)?; + remove_tasks(&tasks, &dst, self.index_mapper.index_base_map_size)?; } // 2.3 Create a read transaction on the index-scheduler From aa5a1f333aa01a3161fa979cec4a85da35c5b803 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 23 Jul 2025 15:33:17 +0200 Subject: [PATCH 185/312] Refactor to support less combinations --- crates/meilisearch/tests/search/filters.rs | 69 +++++---- .../milli/src/search/facet/filter_vector.rs | 140 +++++++++--------- 2 files changed, 109 insertions(+), 100 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 12bfbe2ea..cd2da747d 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -981,7 +981,7 @@ async fn vector_filter_specific_fragment_user_provided() { .await; snapshot!(value, @r#" { - "message": "Index `[uuid]`: Vector filter cannot have both `other` and `userProvided`.\n31:43 _vectors.rest.fragments.other.userProvided EXISTS", + "message": "Index `[uuid]`: Vector filter cannot have both `fragments` and `userProvided`.\n15:24 _vectors.rest.fragments.other.userProvided EXISTS", "code": "invalid_search_filter", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_search_filter" @@ -1156,32 +1156,45 @@ async fn vector_filter_or_combination() { async fn vector_filter_regenerate() { let index = shared_index_for_fragments().await; - for selector in ["_vectors.rest.regenerate", "_vectors.rest.fragments.basic.regenerate"] { - let (value, _code) = index - .search_post(json!({ - "filter": format!("{selector} EXISTS"), - "attributesToRetrieve": ["name"] - })) - .await; - snapshot!(value, @r#" - { - "hits": [ - { - "name": "kefir" - }, - { - "name": "intel" - }, - { - "name": "dustin" - } - ], - "query": "", - "processingTimeMs": "[duration]", - "limit": 20, - "offset": 0, - "estimatedTotalHits": 3 - } - "#); + let (value, _code) = index + .search_post(json!({ + "filter": format!("_vectors.rest.regenerate EXISTS"), + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "hits": [ + { + "name": "kefir" + }, + { + "name": "intel" + }, + { + "name": "dustin" + } + ], + "query": "", + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3 } + "#); + + let (value, _code) = index + .search_post(json!({ + "filter": format!("_vectors.rest.fragments.basic.regenerate EXISTS"), + "attributesToRetrieve": ["name"] + })) + .await; + snapshot!(value, @r#" + { + "message": "Index `[uuid]`: Vector filter cannot have both `fragments` and `regenerate`.\n15:24 _vectors.rest.fragments.basic.regenerate EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" + } + "#); } diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 83b3adcc3..91f138685 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -8,16 +8,17 @@ use crate::Index; #[derive(Debug)] enum VectorFilterInner<'a> { - Fragment { embedder_token: Token<'a>, fragment_token: Token<'a> }, - DocumentTemplate { embedder_token: Token<'a> }, - UserProvided { embedder_token: Token<'a> }, - FullEmbedder { embedder_token: Token<'a> }, + Fragment(Token<'a>), + DocumentTemplate, + UserProvided, + Regenerate, + None, } #[derive(Debug)] pub(super) struct VectorFilter<'a> { - inner: Option>, - regenerate: bool, + embedder: Option>, + inner: VectorFilterInner<'a>, } #[derive(Debug, thiserror::Error)] @@ -31,8 +32,10 @@ pub enum VectorFilterError<'a> { #[error("Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.")] MissingFragmentName(Token<'a>), - #[error("Vector filter cannot have both `{}` and `{}`.", _0.0.value(), _0.1.value())] - ExclusiveOptions(Box<(Token<'a>, Token<'a>)>), + #[error("Vector filter cannot have both {}.", { + _0.iter().map(|t| format!("`{}`", t.value())).collect::>().join(" and ") + })] + ExclusiveOptions(Vec>), #[error("Vector filter has leftover token: `{}`.", _0.value())] LeftoverToken(Token<'a>), @@ -73,7 +76,12 @@ impl<'a> From> for Error { InvalidPrefix(token) | MissingFragmentName(token) | LeftoverToken(token) => { token.clone().as_external_error(err).into() } - ExclusiveOptions(tokens) => tokens.1.clone().as_external_error(err).into(), + ExclusiveOptions(tokens) => tokens + .first() + .cloned() + .unwrap_or_else(|| Token::from("")) // Should never happen: tokens is never created empty + .as_external_error(err) + .into(), EmbedderDoesNotExist { embedder: token, .. } | FragmentDoesNotExist { fragment: token, .. } => token.as_external_error(err).into(), } @@ -92,11 +100,8 @@ impl<'a> VectorFilter<'a> { /// - `_vectors.{embedder_name}` /// - `_vectors.{embedder_name}.regenerate` /// - `_vectors.{embedder_name}.userProvided` - /// - `_vectors.{embedder_name}.userProvided.regenerate` /// - `_vectors.{embedder_name}.documentTemplate` - /// - `_vectors.{embedder_name}.documentTemplate.regenerate` /// - `_vectors.{embedder_name}.fragments.{fragment_name}` - /// - `_vectors.{embedder_name}.fragments.{fragment_name}.regenerate` pub(super) fn parse(s: &'a Token<'a>) -> Result> { let mut split = s.split(".").peekable(); @@ -108,54 +113,53 @@ impl<'a> VectorFilter<'a> { let embedder_name = split.next(); - let mut fragment_name = None; + let mut fragment_tokens = None; if split.peek().map(|t| t.value()) == Some("fragments") { let token = split.next().expect("it was peeked before"); + let name = split.next().ok_or_else(|| MissingFragmentName(token.clone()))?; - fragment_name = Some(split.next().ok_or(MissingFragmentName(token))?); + fragment_tokens = Some((token, name)); } + let mut remaining_tokens = split.collect::>(); + let mut user_provided_token = None; - if split.peek().map(|t| t.value()) == Some("userProvided") { - user_provided_token = split.next(); + if let Some(position) = remaining_tokens.iter().position(|t| t.value() == "userProvided") { + user_provided_token = Some(remaining_tokens.remove(position)); } let mut document_template_token = None; - if split.peek().map(|t| t.value()) == Some("documentTemplate") { - document_template_token = split.next(); + if let Some(position) = + remaining_tokens.iter().position(|t| t.value() == "documentTemplate") + { + document_template_token = Some(remaining_tokens.remove(position)); } let mut regenerate_token = None; - if split.peek().map(|t| t.value()) == Some("regenerate") { - regenerate_token = split.next(); + if let Some(position) = remaining_tokens.iter().position(|t| t.value() == "regenerate") { + regenerate_token = Some(remaining_tokens.remove(position)); } - let inner = match (fragment_name, user_provided_token, document_template_token) { - (Some(fragment_name), None, None) => Some(VectorFilterInner::Fragment { - embedder_token: embedder_name - .expect("embedder name comes before fragment so it's always Some"), - fragment_token: fragment_name, - }), - (None, Some(_), None) => Some(VectorFilterInner::UserProvided { - embedder_token: embedder_name - .expect("embedder name comes before userProvided so it's always Some"), - }), - (None, None, Some(_)) => Some(VectorFilterInner::DocumentTemplate { - embedder_token: embedder_name - .expect("embedder name comes before documentTemplate so it's always Some"), - }), - (Some(a), Some(b), _) | (_, Some(a), Some(b)) | (Some(a), None, Some(b)) => { - return Err(ExclusiveOptions(Box::new((a, b)))); - } - (None, None, None) => embedder_name - .map(|embedder_token| VectorFilterInner::FullEmbedder { embedder_token }), - }; - - if let Some(next) = split.next() { - return Err(LeftoverToken(next))?; + if !remaining_tokens.is_empty() { + return Err(LeftoverToken(remaining_tokens.remove(0))); } - Ok(Self { inner, regenerate: regenerate_token.is_some() }) + let inner = + match (fragment_tokens, user_provided_token, document_template_token, regenerate_token) + { + (Some((_token, name)), None, None, None) => VectorFilterInner::Fragment(name), + (None, Some(_), None, None) => VectorFilterInner::UserProvided, + (None, None, Some(_), None) => VectorFilterInner::DocumentTemplate, + (None, None, None, Some(_)) => VectorFilterInner::Regenerate, + (None, None, None, None) => VectorFilterInner::None, + (a, b, c, d) => { + let a = a.map(|(token, _)| token); + let present = [a, b, c, d].into_iter().flatten().collect(); + return Err(ExclusiveOptions(present)); + } + }; + + Ok(Self { inner, embedder: embedder_name }) } pub(super) fn evaluate( @@ -167,19 +171,16 @@ impl<'a> VectorFilter<'a> { let index_embedding_configs = index.embedding_configs(); let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; - let inners = match self.inner { - Some(inner) => vec![inner], - None => embedding_configs - .iter() - .map(|config| VectorFilterInner::FullEmbedder { - embedder_token: Token::from(config.name.as_str()), - }) - .collect(), + let embedders = match self.embedder { + Some(embedder) => vec![embedder], + None => { + embedding_configs.iter().map(|config| Token::from(config.name.as_str())).collect() + } }; - let mut docids = inners + let mut docids = embedders .iter() - .map(|i| i.evaluate_inner(rtxn, index, &embedding_configs, self.regenerate)) + .map(|e| self.inner.evaluate(rtxn, index, e, &embedding_configs)) .union()?; if let Some(universe) = universe { @@ -191,19 +192,13 @@ impl<'a> VectorFilter<'a> { } impl VectorFilterInner<'_> { - fn evaluate_inner( + fn evaluate( &self, rtxn: &heed::RoTxn<'_>, index: &Index, + embedder: &Token<'_>, embedding_configs: &[IndexEmbeddingConfig], - regenerate: bool, ) -> crate::Result { - let embedder = match self { - VectorFilterInner::Fragment { embedder_token, .. } => embedder_token, - VectorFilterInner::DocumentTemplate { embedder_token } => embedder_token, - VectorFilterInner::UserProvided { embedder_token } => embedder_token, - VectorFilterInner::FullEmbedder { embedder_token } => embedder_token, - }; let embedder_name = embedder.value(); let available_embedders = || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); @@ -224,8 +219,8 @@ impl VectorFilterInner<'_> { embedding_config.config.quantized(), ); - let mut docids = match self { - VectorFilterInner::Fragment { embedder_token: embedder, fragment_token: fragment } => { + let docids = match self { + VectorFilterInner::Fragment(fragment) => { let fragment_name = fragment.value(); let fragment_config = embedding_config .fragments @@ -245,7 +240,7 @@ impl VectorFilterInner<'_> { arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? } - VectorFilterInner::DocumentTemplate { .. } => { + VectorFilterInner::DocumentTemplate => { if !embedding_config.fragments.as_slice().is_empty() { return Ok(RoaringBitmap::new()); } @@ -255,22 +250,23 @@ impl VectorFilterInner<'_> { arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; stats.documents - user_provided_docsids.clone() } - VectorFilterInner::UserProvided { .. } => { + VectorFilterInner::UserProvided => { let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); user_provided_docsids.clone() } - VectorFilterInner::FullEmbedder { .. } => { + VectorFilterInner::Regenerate => { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); + stats.documents - skip_regenerate + } + VectorFilterInner::None => { let mut stats = ArroyStats::default(); arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; stats.documents } }; - if regenerate { - let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); - docids -= skip_regenerate; - } - Ok(docids) } } From bb4d57386280796928f311133eb14d1a4af470a4 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 14:56:35 +0200 Subject: [PATCH 186/312] Switch to a nom parser --- crates/filter-parser/src/condition.rs | 60 ++++ crates/filter-parser/src/error.rs | 34 ++ crates/filter-parser/src/lib.rs | 46 ++- crates/filter-parser/src/value.rs | 33 ++ crates/meilisearch/tests/search/filters.rs | 18 +- crates/milli/src/search/facet/filter.rs | 23 +- .../milli/src/search/facet/filter_vector.rs | 306 ++++++------------ 7 files changed, 269 insertions(+), 251 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index 0fc007bf1..af0767706 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -7,11 +7,20 @@ use nom::branch::alt; use nom::bytes::complete::tag; +use nom::character::complete::char; +use nom::character::complete::multispace0; use nom::character::complete::multispace1; use nom::combinator::cut; +use nom::combinator::map; +use nom::combinator::value; +use nom::sequence::preceded; use nom::sequence::{terminated, tuple}; use Condition::*; +use crate::error::IResultExt; +use crate::value::parse_vector_value; +use crate::ErrorKind; +use crate::VectorFilter; use crate::{parse_value, FilterCondition, IResult, Span, Token}; #[derive(Debug, Clone, PartialEq, Eq)] @@ -113,6 +122,57 @@ pub fn parse_not_exists(input: Span) -> IResult { Ok((input, FilterCondition::Not(Box::new(FilterCondition::Condition { fid: key, op: Exists })))) } +fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_>)> { + let (input, _) = multispace0(input)?; + let (input, fid) = tag("_vectors")(input)?; + + if let Ok((input, _)) = multispace1::<_, crate::Error>(input) { + return Ok((input, (Token::from(fid), None, VectorFilter::None))); + } + + let (input, _) = char('.')(input)?; + + // From this point, we are certain this is a vector filter, so our errors must be final. + // We could use nom's `cut`` but it's better to be explicit about the errors + + let (input, embedder_name) = parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidEmbedder)?; + + let (input, filter) = alt(( + map( + preceded(tag(".fragments"), |input| { + let (input, _) = tag(".")(input).map_cut(ErrorKind::VectorFilterMissingFragment)?; + parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidFragment) + }), + VectorFilter::Fragment, + ), + value(VectorFilter::UserProvided, tag(".userProvided")), + value(VectorFilter::DocumentTemplate, tag(".documentTemplate")), + value(VectorFilter::Regenerate, tag(".regenerate")), + value(VectorFilter::None, nom::combinator::success("")), + ))(input)?; + + let (input, _) = multispace1(input).map_cut(ErrorKind::VectorFilterLeftover)?; + + Ok((input, (Token::from(fid), Some(embedder_name), filter))) +} + +/// vectors_exists = vectors "EXISTS" +pub fn parse_vectors_exists(input: Span) -> IResult { + let (input, (fid, embedder, filter)) = terminated(parse_vectors, tag("EXISTS"))(input)?; + + Ok((input, FilterCondition::VectorExists { fid, embedder, filter })) +} +/// vectors_not_exists = vectors "NOT" WS+ "EXISTS" +pub fn parse_vectors_not_exists(input: Span) -> IResult { + let (input, (fid, embedder, filter)) = parse_vectors(input)?; + + let (input, _) = tuple((tag("NOT"), multispace1, tag("EXISTS")))(input)?; + Ok(( + input, + FilterCondition::Not(Box::new(FilterCondition::VectorExists { fid, embedder, filter })), + )) +} + /// contains = value "CONTAINS" value pub fn parse_contains(input: Span) -> IResult { let (input, (fid, contains, value)) = diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index 855ce983e..cf2419b01 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -42,6 +42,23 @@ pub fn cut_with_err<'a, O>( } } +pub trait IResultExt<'a> { + fn map_cut(self, kind: ErrorKind<'a>) -> Self; +} + +impl<'a, T> IResultExt<'a> for IResult<'a, T> { + fn map_cut(self, kind: ErrorKind<'a>) -> Self { + self.map_err(move |e: nom::Err>| { + let input = match e { + nom::Err::Incomplete(_) => return e, + nom::Err::Error(e) => *e.context(), + nom::Err::Failure(e) => *e.context(), + }; + nom::Err::Failure(Error::new_from_kind(input, kind)) + }) + } +} + #[derive(Debug)] pub struct Error<'a> { context: Span<'a>, @@ -76,6 +93,11 @@ pub enum ErrorKind<'a> { InternalError(error::ErrorKind), DepthLimitReached, External(String), + + VectorFilterLeftover, + VectorFilterInvalidEmbedder, + VectorFilterMissingFragment, + VectorFilterInvalidFragment, } impl<'a> Error<'a> { @@ -169,6 +191,18 @@ impl Display for Error<'_> { ErrorKind::MisusedGeoBoundingBox => { writeln!(f, "The `_geoBoundingBox` filter is an operation and can't be used as a value.")? } + ErrorKind::VectorFilterLeftover => { + writeln!(f, "The vector filter has leftover tokens.")? + } + ErrorKind::VectorFilterInvalidFragment => { + writeln!(f, "The vector filter's fragment is invalid.")? + } + ErrorKind::VectorFilterMissingFragment => { + writeln!(f, "The vector filter is missing a fragment name.")? + } + ErrorKind::VectorFilterInvalidEmbedder => { + writeln!(f, "The vector filter's embedder is invalid.")? + } ErrorKind::ReservedKeyword(word) => { writeln!(f, "`{word}` is a reserved keyword and thus cannot be used as a field name unless it is put inside quotes. Use \"{word}\" or \'{word}\' instead.")? } diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 1590b08fd..b5697f914 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -65,6 +65,9 @@ use nom_locate::LocatedSpan; pub(crate) use value::parse_value; use value::word_exact; +use crate::condition::{parse_vectors_exists, parse_vectors_not_exists}; +use crate::error::IResultExt; + pub type Span<'a> = LocatedSpan<&'a str, &'a str>; type IResult<'a, Ret> = nom::IResult, Ret, Error<'a>>; @@ -146,6 +149,15 @@ impl<'a> From<&'a str> for Token<'a> { } } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VectorFilter<'a> { + Fragment(Token<'a>), + DocumentTemplate, + UserProvided, + Regenerate, + None, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub enum FilterCondition<'a> { Not(Box), @@ -153,6 +165,7 @@ pub enum FilterCondition<'a> { In { fid: Token<'a>, els: Vec> }, Or(Vec), And(Vec), + VectorExists { fid: Token<'a>, embedder: Option>, filter: VectorFilter<'a> }, GeoLowerThan { point: [Token<'a>; 2], radius: Token<'a> }, GeoBoundingBox { top_right_point: [Token<'a>; 2], bottom_left_point: [Token<'a>; 2] }, } @@ -183,7 +196,8 @@ impl<'a> FilterCondition<'a> { FilterCondition::Or(seq) | FilterCondition::And(seq) => { seq.iter().find_map(|filter| filter.use_contains_operator()) } - FilterCondition::GeoLowerThan { .. } + FilterCondition::VectorExists { .. } + | FilterCondition::GeoLowerThan { .. } | FilterCondition::GeoBoundingBox { .. } | FilterCondition::In { .. } => None, } @@ -191,13 +205,7 @@ impl<'a> FilterCondition<'a> { pub fn use_vector_filter(&self) -> Option<&Token> { match self { - FilterCondition::Condition { fid, op: _ } => { - if fid.value().starts_with("_vectors.") || fid.value() == "_vectors" { - Some(fid) - } else { - None - } - } + FilterCondition::Condition { .. } => None, FilterCondition::Not(this) => this.use_vector_filter(), FilterCondition::Or(seq) | FilterCondition::And(seq) => { seq.iter().find_map(|filter| filter.use_vector_filter()) @@ -205,6 +213,7 @@ impl<'a> FilterCondition<'a> { FilterCondition::GeoLowerThan { .. } | FilterCondition::GeoBoundingBox { .. } | FilterCondition::In { .. } => None, + FilterCondition::VectorExists { fid, .. } => Some(fid), } } @@ -292,10 +301,7 @@ fn parse_in_body(input: Span) -> IResult> { let (input, _) = ws(word_exact("IN"))(input)?; // everything after `IN` can be a failure - let (input, _) = - cut_with_err(tag("["), |_| Error::new_from_kind(input, ErrorKind::InOpeningBracket))( - input, - )?; + let (input, _) = tag("[")(input).map_cut(ErrorKind::InOpeningBracket)?; let (input, content) = cut(parse_value_list)(input)?; @@ -529,8 +535,7 @@ fn parse_primary(input: Span, depth: usize) -> IResult { parse_is_not_null, parse_is_empty, parse_is_not_empty, - parse_exists, - parse_not_exists, + alt((parse_vectors_exists, parse_vectors_not_exists, parse_exists, parse_not_exists)), parse_to, parse_contains, parse_not_contains, @@ -586,6 +591,19 @@ impl std::fmt::Display for FilterCondition<'_> { } write!(f, "]") } + FilterCondition::VectorExists { fid: _, embedder, filter: inner } => { + write!(f, "_vectors")?; + if let Some(embedder) = embedder { + write!(f, ".{embedder:?}")?; + } + match inner { + VectorFilter::Fragment(fragment) => write!(f, ".fragments.{fragment:?}"), + VectorFilter::DocumentTemplate => write!(f, ".documentTemplate"), + VectorFilter::UserProvided => write!(f, ".userProvided"), + VectorFilter::Regenerate => write!(f, ".regenerate"), + VectorFilter::None => Ok(()), + } + } FilterCondition::GeoLowerThan { point, radius } => { write!(f, "_geoRadius({}, {}, {})", point[0], point[1], radius) } diff --git a/crates/filter-parser/src/value.rs b/crates/filter-parser/src/value.rs index 98cac39fe..345f0b0a2 100644 --- a/crates/filter-parser/src/value.rs +++ b/crates/filter-parser/src/value.rs @@ -80,6 +80,39 @@ pub fn word_exact<'a, 'b: 'a>(tag: &'b str) -> impl Fn(Span<'a>) -> IResult<'a, } } +/// vector_value = ( non_dot_word | singleQuoted | doubleQuoted) +pub fn parse_vector_value(input: Span) -> IResult { + pub fn non_dot_word(input: Span) -> IResult { + let (input, word) = take_while1(|c| is_value_component(c) && c != '.')(input)?; + Ok((input, word.into())) + } + + let (input, value) = alt(( + delimited(char('\''), cut(|input| quoted_by('\'', input)), cut(char('\''))), + delimited(char('"'), cut(|input| quoted_by('"', input)), cut(char('"'))), + non_dot_word, + ))(input)?; + + match unescaper::unescape(value.value()) { + Ok(content) => { + if content.len() != value.value().len() { + Ok((input, Token::new(value.original_span(), Some(content)))) + } else { + Ok((input, value)) + } + } + Err(unescaper::Error::IncompleteStr(_)) => Err(nom::Err::Incomplete(nom::Needed::Unknown)), + Err(unescaper::Error::ParseIntError { .. }) => Err(nom::Err::Error(Error::new_from_kind( + value.original_span(), + ErrorKind::InvalidEscapedNumber, + ))), + Err(unescaper::Error::InvalidChar { .. }) => Err(nom::Err::Error(Error::new_from_kind( + value.original_span(), + ErrorKind::MalformedValue, + ))), + } +} + /// value = WS* ( word | singleQuoted | doubleQuoted) WS+ pub fn parse_value(input: Span) -> IResult { // to get better diagnostic message we are going to strip the left whitespaces from the input right now diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index cd2da747d..67f9ebb71 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -779,7 +779,7 @@ async fn vector_filter_missing_fragment() { .await; snapshot!(value, @r#" { - "message": "Index `[uuid]`: Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.\n15:24 _vectors.rest.fragments EXISTS", + "message": "The vector filter is missing a fragment name.\n24:31 _vectors.rest.fragments EXISTS", "code": "invalid_search_filter", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_search_filter" @@ -981,7 +981,7 @@ async fn vector_filter_specific_fragment_user_provided() { .await; snapshot!(value, @r#" { - "message": "Index `[uuid]`: Vector filter cannot have both `fragments` and `userProvided`.\n15:24 _vectors.rest.fragments.other.userProvided EXISTS", + "message": "The vector filter has leftover tokens.\n30:50 _vectors.rest.fragments.other.userProvided EXISTS", "code": "invalid_search_filter", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_search_filter" @@ -1190,11 +1190,11 @@ async fn vector_filter_regenerate() { })) .await; snapshot!(value, @r#" - { - "message": "Index `[uuid]`: Vector filter cannot have both `fragments` and `regenerate`.\n15:24 _vectors.rest.fragments.basic.regenerate EXISTS", - "code": "invalid_search_filter", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_search_filter" - } - "#); + { + "message": "The vector filter has leftover tokens.\n30:48 _vectors.rest.fragments.basic.regenerate EXISTS", + "code": "invalid_search_filter", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_search_filter" + } + "#); } diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index 21a552965..4e67814d3 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -10,8 +10,8 @@ use memchr::memmem::Finder; use roaring::{MultiOps, RoaringBitmap}; use serde_json::Value; -use super::{facet_range_search, filter_vector::VectorFilter}; -use crate::constants::RESERVED_GEO_FIELD_NAME; +use super::facet_range_search; +use crate::constants::{RESERVED_GEO_FIELD_NAME, RESERVED_VECTORS_FIELD_NAME}; use crate::error::{Error, UserError}; use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features}; use crate::heed_codec::facet::{ @@ -230,7 +230,7 @@ impl<'a> Filter<'a> { } pub fn use_vector_filter(&self) -> Option<&Token> { - self.condition.use_vector_filter() + dbg!(self.condition.use_vector_filter()) } } @@ -241,10 +241,10 @@ impl<'a> Filter<'a> { let filterable_attributes_rules = index.filterable_attributes_rules(rtxn)?; for fid in self.condition.fids(MAX_FILTER_DEPTH) { - let attribute = fid.value(); + let attribute = dbg!(fid.value()); if matching_features(attribute, &filterable_attributes_rules) .is_some_and(|(_, features)| features.is_filterable()) - || VectorFilter::matches(attribute) + || attribute == RESERVED_VECTORS_FIELD_NAME { continue; } @@ -549,16 +549,6 @@ impl<'a> Filter<'a> { } FilterCondition::Condition { fid, op } => { let value = fid.value(); - if VectorFilter::matches(value) { - if !matches!(op, Condition::Exists) { - return Err(Error::UserError(UserError::InvalidFilter(String::from( - "Vector filter can only be used with the `exists` operator", - )))); - } - let vector_filter = VectorFilter::parse(fid)?; - return vector_filter.evaluate(rtxn, index, universe); - } - let Some(field_id) = field_ids_map.id(value) else { return Ok(RoaringBitmap::new()); }; @@ -616,6 +606,9 @@ impl<'a> Filter<'a> { Ok(RoaringBitmap::new()) } } + FilterCondition::VectorExists { fid: _, embedder, filter } => { + super::filter_vector::evaluate(rtxn, index, universe, embedder.clone(), filter) + } FilterCondition::GeoLowerThan { point, radius } => { if index.is_geo_filtering_enabled(rtxn)? { let base_point: [f64; 2] = diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 91f138685..2ddd801ed 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -1,45 +1,13 @@ -use filter_parser::Token; +use filter_parser::{Token, VectorFilter}; use roaring::{MultiOps, RoaringBitmap}; -use crate::error::{Error, UserError}; +use crate::error::Error; use crate::vector::db::IndexEmbeddingConfig; use crate::vector::{ArroyStats, ArroyWrapper}; use crate::Index; -#[derive(Debug)] -enum VectorFilterInner<'a> { - Fragment(Token<'a>), - DocumentTemplate, - UserProvided, - Regenerate, - None, -} - -#[derive(Debug)] -pub(super) struct VectorFilter<'a> { - embedder: Option>, - inner: VectorFilterInner<'a>, -} - #[derive(Debug, thiserror::Error)] pub enum VectorFilterError<'a> { - #[error("Vector filter cannot be empty.")] - EmptyFilter, - - #[error("Vector filter must start with `_vectors` but found `{}`.", _0.value())] - InvalidPrefix(Token<'a>), - - #[error("Vector filter is inconsistent: either specify a fragment name or remove the `fragments` part.")] - MissingFragmentName(Token<'a>), - - #[error("Vector filter cannot have both {}.", { - _0.iter().map(|t| format!("`{}`", t.value())).collect::>().join(" and ") - })] - ExclusiveOptions(Vec>), - - #[error("Vector filter has leftover token: `{}`.", _0.value())] - LeftoverToken(Token<'a>), - #[error("The embedder `{}` does not exist. {}", embedder.value(), { if available.is_empty() { String::from("This index does not have any configured embedders.") @@ -72,201 +40,113 @@ use VectorFilterError::*; impl<'a> From> for Error { fn from(err: VectorFilterError<'a>) -> Self { match &err { - EmptyFilter => Error::UserError(UserError::InvalidFilter(err.to_string())), - InvalidPrefix(token) | MissingFragmentName(token) | LeftoverToken(token) => { - token.clone().as_external_error(err).into() - } - ExclusiveOptions(tokens) => tokens - .first() - .cloned() - .unwrap_or_else(|| Token::from("")) // Should never happen: tokens is never created empty - .as_external_error(err) - .into(), EmbedderDoesNotExist { embedder: token, .. } | FragmentDoesNotExist { fragment: token, .. } => token.as_external_error(err).into(), } } } -impl<'a> VectorFilter<'a> { - pub(super) fn matches(value: &str) -> bool { - value.starts_with("_vectors.") || value == "_vectors" +pub(super) fn evaluate( + rtxn: &heed::RoTxn<'_>, + index: &Index, + universe: Option<&RoaringBitmap>, + embedder: Option>, + filter: &VectorFilter<'_>, +) -> crate::Result { + let index_embedding_configs = index.embedding_configs(); + let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; + + let embedders = match embedder { + Some(embedder) => vec![embedder], + None => embedding_configs.iter().map(|config| Token::from(config.name.as_str())).collect(), + }; + + let mut docids = embedders + .iter() + .map(|e| evaluate_inner(rtxn, index, e, &embedding_configs, filter)) + .union()?; + + if let Some(universe) = universe { + docids &= universe; } - /// Parses a vector filter string. - /// - /// Valid formats: - /// - `_vectors` - /// - `_vectors.{embedder_name}` - /// - `_vectors.{embedder_name}.regenerate` - /// - `_vectors.{embedder_name}.userProvided` - /// - `_vectors.{embedder_name}.documentTemplate` - /// - `_vectors.{embedder_name}.fragments.{fragment_name}` - pub(super) fn parse(s: &'a Token<'a>) -> Result> { - let mut split = s.split(".").peekable(); - - match split.next() { - Some(token) if token.value() == "_vectors" => (), - Some(token) => return Err(InvalidPrefix(token)), - None => return Err(EmptyFilter), - } - - let embedder_name = split.next(); - - let mut fragment_tokens = None; - if split.peek().map(|t| t.value()) == Some("fragments") { - let token = split.next().expect("it was peeked before"); - let name = split.next().ok_or_else(|| MissingFragmentName(token.clone()))?; - - fragment_tokens = Some((token, name)); - } - - let mut remaining_tokens = split.collect::>(); - - let mut user_provided_token = None; - if let Some(position) = remaining_tokens.iter().position(|t| t.value() == "userProvided") { - user_provided_token = Some(remaining_tokens.remove(position)); - } - - let mut document_template_token = None; - if let Some(position) = - remaining_tokens.iter().position(|t| t.value() == "documentTemplate") - { - document_template_token = Some(remaining_tokens.remove(position)); - } - - let mut regenerate_token = None; - if let Some(position) = remaining_tokens.iter().position(|t| t.value() == "regenerate") { - regenerate_token = Some(remaining_tokens.remove(position)); - } - - if !remaining_tokens.is_empty() { - return Err(LeftoverToken(remaining_tokens.remove(0))); - } - - let inner = - match (fragment_tokens, user_provided_token, document_template_token, regenerate_token) - { - (Some((_token, name)), None, None, None) => VectorFilterInner::Fragment(name), - (None, Some(_), None, None) => VectorFilterInner::UserProvided, - (None, None, Some(_), None) => VectorFilterInner::DocumentTemplate, - (None, None, None, Some(_)) => VectorFilterInner::Regenerate, - (None, None, None, None) => VectorFilterInner::None, - (a, b, c, d) => { - let a = a.map(|(token, _)| token); - let present = [a, b, c, d].into_iter().flatten().collect(); - return Err(ExclusiveOptions(present)); - } - }; - - Ok(Self { inner, embedder: embedder_name }) - } - - pub(super) fn evaluate( - self, - rtxn: &heed::RoTxn<'_>, - index: &Index, - universe: Option<&RoaringBitmap>, - ) -> crate::Result { - let index_embedding_configs = index.embedding_configs(); - let embedding_configs = index_embedding_configs.embedding_configs(rtxn)?; - - let embedders = match self.embedder { - Some(embedder) => vec![embedder], - None => { - embedding_configs.iter().map(|config| Token::from(config.name.as_str())).collect() - } - }; - - let mut docids = embedders - .iter() - .map(|e| self.inner.evaluate(rtxn, index, e, &embedding_configs)) - .union()?; - - if let Some(universe) = universe { - docids &= universe; - } - - Ok(docids) - } + Ok(docids) } -impl VectorFilterInner<'_> { - fn evaluate( - &self, - rtxn: &heed::RoTxn<'_>, - index: &Index, - embedder: &Token<'_>, - embedding_configs: &[IndexEmbeddingConfig], - ) -> crate::Result { - let embedder_name = embedder.value(); - let available_embedders = - || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); +fn evaluate_inner( + rtxn: &heed::RoTxn<'_>, + index: &Index, + embedder: &Token<'_>, + embedding_configs: &[IndexEmbeddingConfig], + filter: &VectorFilter<'_>, +) -> crate::Result { + let embedder_name = embedder.value(); + let available_embedders = + || embedding_configs.iter().map(|c| c.name.clone()).collect::>(); - let embedding_config = embedding_configs - .iter() - .find(|config| config.name == embedder_name) - .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; + let embedding_config = embedding_configs + .iter() + .find(|config| config.name == embedder_name) + .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; - let embedder_info = index - .embedding_configs() - .embedder_info(rtxn, embedder_name)? - .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; + let embedder_info = index + .embedding_configs() + .embedder_info(rtxn, embedder_name)? + .ok_or_else(|| EmbedderDoesNotExist { embedder, available: available_embedders() })?; - let arroy_wrapper = ArroyWrapper::new( - index.vector_arroy, - embedder_info.embedder_id, - embedding_config.config.quantized(), - ); + let arroy_wrapper = ArroyWrapper::new( + index.vector_arroy, + embedder_info.embedder_id, + embedding_config.config.quantized(), + ); - let docids = match self { - VectorFilterInner::Fragment(fragment) => { - let fragment_name = fragment.value(); - let fragment_config = embedding_config - .fragments - .as_slice() - .iter() - .find(|fragment| fragment.name == fragment_name) - .ok_or_else(|| FragmentDoesNotExist { - embedder, - fragment, - available: embedding_config - .fragments - .as_slice() - .iter() - .map(|f| f.name.clone()) - .collect(), - })?; + let docids = match filter { + VectorFilter::Fragment(fragment) => { + let fragment_name = fragment.value(); + let fragment_config = embedding_config + .fragments + .as_slice() + .iter() + .find(|fragment| fragment.name == fragment_name) + .ok_or_else(|| FragmentDoesNotExist { + embedder, + fragment, + available: embedding_config + .fragments + .as_slice() + .iter() + .map(|f| f.name.clone()) + .collect(), + })?; - arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + } + VectorFilter::DocumentTemplate => { + if !embedding_config.fragments.as_slice().is_empty() { + return Ok(RoaringBitmap::new()); } - VectorFilterInner::DocumentTemplate => { - if !embedding_config.fragments.as_slice().is_empty() { - return Ok(RoaringBitmap::new()); - } - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents - user_provided_docsids.clone() - } - VectorFilterInner::UserProvided => { - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - user_provided_docsids.clone() - } - VectorFilterInner::Regenerate => { - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); - stats.documents - skip_regenerate - } - VectorFilterInner::None => { - let mut stats = ArroyStats::default(); - arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents - } - }; + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents - user_provided_docsids.clone() + } + VectorFilter::UserProvided => { + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + user_provided_docsids.clone() + } + VectorFilter::Regenerate => { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + let skip_regenerate = embedder_info.embedding_status.skip_regenerate_docids(); + stats.documents - skip_regenerate + } + VectorFilter::None => { + let mut stats = ArroyStats::default(); + arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; + stats.documents + } + }; - Ok(docids) - } + Ok(docids) } From 8f1b697b911c41a86ba91a1f8119690dd8ecadae Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 14:57:06 +0200 Subject: [PATCH 187/312] Format --- crates/filter-parser/src/condition.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index af0767706..1e8deef64 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -135,7 +135,8 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> // From this point, we are certain this is a vector filter, so our errors must be final. // We could use nom's `cut`` but it's better to be explicit about the errors - let (input, embedder_name) = parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidEmbedder)?; + let (input, embedder_name) = + parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidEmbedder)?; let (input, filter) = alt(( map( From ad068286856e5360952863d26066e6cac1c63e86 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 15:24:42 +0200 Subject: [PATCH 188/312] Add tests on parser --- crates/filter-parser/src/lib.rs | 83 ++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index b5697f914..1e342d8d2 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -594,15 +594,18 @@ impl std::fmt::Display for FilterCondition<'_> { FilterCondition::VectorExists { fid: _, embedder, filter: inner } => { write!(f, "_vectors")?; if let Some(embedder) = embedder { - write!(f, ".{embedder:?}")?; + write!(f, ".{:?}", embedder.value())?; } match inner { - VectorFilter::Fragment(fragment) => write!(f, ".fragments.{fragment:?}"), - VectorFilter::DocumentTemplate => write!(f, ".documentTemplate"), - VectorFilter::UserProvided => write!(f, ".userProvided"), - VectorFilter::Regenerate => write!(f, ".regenerate"), - VectorFilter::None => Ok(()), + VectorFilter::Fragment(fragment) => { + write!(f, ".fragments.{:?}", fragment.value())? + } + VectorFilter::DocumentTemplate => write!(f, ".documentTemplate")?, + VectorFilter::UserProvided => write!(f, ".userProvided")?, + VectorFilter::Regenerate => write!(f, ".regenerate")?, + VectorFilter::None => (), } + write!(f, " EXISTS") } FilterCondition::GeoLowerThan { point, radius } => { write!(f, "_geoRadius({}, {}, {})", point[0], point[1], radius) @@ -677,6 +680,9 @@ pub mod tests { insta::assert_snapshot!(p(r"title = 'foo\\\\\\\\'"), @r#"{title} = {foo\\\\}"#); // but it also works with other sequences insta::assert_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}"); + + insta::assert_snapshot!(p(r#"_vectors." valid.name ".fragments."also.. valid! " EXISTS"#), @r#"_vectors." valid.name ".fragments."also.. valid! " EXISTS"#); + insta::assert_snapshot!(p("_vectors.\"\n\t\r\\\"\" EXISTS"), @r#"_vectors."\n\t\r\"" EXISTS"#); } #[test] @@ -739,6 +745,18 @@ pub mod tests { insta::assert_snapshot!(p("NOT subscribers IS NOT EMPTY"), @"{subscribers} IS EMPTY"); insta::assert_snapshot!(p("subscribers IS NOT EMPTY"), @"NOT ({subscribers} IS EMPTY)"); + // Test _vectors EXISTS + _vectors NOT EXITS + insta::assert_snapshot!(p("_vectors EXISTS"), @"_vectors EXISTS"); + insta::assert_snapshot!(p("_vectors.embedderName EXISTS"), @r#"_vectors."embedderName" EXISTS"#); + insta::assert_snapshot!(p("_vectors.embedderName.documentTemplate EXISTS"), @r#"_vectors."embedderName".documentTemplate EXISTS"#); + insta::assert_snapshot!(p("_vectors.embedderName.regenerate EXISTS"), @r#"_vectors."embedderName".regenerate EXISTS"#); + insta::assert_snapshot!(p("_vectors.embedderName.regenerate EXISTS"), @r#"_vectors."embedderName".regenerate EXISTS"#); + insta::assert_snapshot!(p("_vectors.embedderName.fragments.fragmentName EXISTS"), @r#"_vectors."embedderName".fragments."fragmentName" EXISTS"#); + insta::assert_snapshot!(p(" _vectors.embedderName.fragments.fragmentName EXISTS"), @r#"_vectors."embedderName".fragments."fragmentName" EXISTS"#); + insta::assert_snapshot!(p("NOT _vectors EXISTS"), @"NOT (_vectors EXISTS)"); + insta::assert_snapshot!(p(" NOT _vectors EXISTS"), @"NOT (_vectors EXISTS)"); + insta::assert_snapshot!(p(" _vectors NOT EXISTS"), @"NOT (_vectors EXISTS)"); + // Test EXISTS + NOT EXITS insta::assert_snapshot!(p("subscribers EXISTS"), @"{subscribers} EXISTS"); insta::assert_snapshot!(p("NOT subscribers EXISTS"), @"NOT ({subscribers} EXISTS)"); @@ -993,6 +1011,59 @@ pub mod tests { "### ); + insta::assert_snapshot!(p(r#"_vectors _vectors EXISTS"#), @r" + Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors _vectors EXISTS`. + 1:25 _vectors _vectors EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors. embedderName EXISTS"#), @r" + The vector filter's embedder is invalid. + 10:30 _vectors. embedderName EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors .embedderName EXISTS"#), @r" + Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors .embedderName EXISTS`. + 1:30 _vectors .embedderName EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName. EXISTS"#), @r" + The vector filter has leftover tokens. + 22:30 _vectors.embedderName. EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors."embedderName EXISTS"#), @r#" + The vector filter's embedder is invalid. + 30:30 _vectors."embedderName EXISTS + "#); + insta::assert_snapshot!(p(r#"_vectors."embedderNam"e EXISTS"#), @r#" + The vector filter has leftover tokens. + 23:31 _vectors."embedderNam"e EXISTS + "#); + insta::assert_snapshot!(p(r#"_vectors.embedderName.documentTemplate. EXISTS"#), @r" + The vector filter has leftover tokens. + 39:47 _vectors.embedderName.documentTemplate. EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments EXISTS"#), @r" + The vector filter is missing a fragment name. + 32:39 _vectors.embedderName.fragments EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments. EXISTS"#), @r" + The vector filter's fragment is invalid. + 33:40 _vectors.embedderName.fragments. EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments.test test EXISTS"#), @r" + Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors.embedderName.fragments.test test EXISTS`. + 1:49 _vectors.embedderName.fragments.test test EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments. test EXISTS"#), @r" + The vector filter's fragment is invalid. + 33:45 _vectors.embedderName.fragments. test EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName .fragments. test EXISTS"#), @r" + Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors.embedderName .fragments. test EXISTS`. + 1:46 _vectors.embedderName .fragments. test EXISTS + "); + insta::assert_snapshot!(p(r#"_vectors.embedderName .fragments.test EXISTS"#), @r" + Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors.embedderName .fragments.test EXISTS`. + 1:45 _vectors.embedderName .fragments.test EXISTS + "); + insta::assert_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###" Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes. 5:7 NOT OR EXISTS AND EXISTS NOT EXISTS From a92e36ab836626bfbbcc2bdade5d029f0a458ba2 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 15:28:17 +0200 Subject: [PATCH 189/312] Small improvements --- crates/filter-parser/src/condition.rs | 2 +- crates/filter-parser/src/error.rs | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index 1e8deef64..4d156c269 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -133,7 +133,7 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> let (input, _) = char('.')(input)?; // From this point, we are certain this is a vector filter, so our errors must be final. - // We could use nom's `cut`` but it's better to be explicit about the errors + // We could use nom's `cut` but it's better to be explicit about the errors let (input, embedder_name) = parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidEmbedder)?; diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index cf2419b01..bbf2c8d17 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -78,6 +78,10 @@ pub enum ErrorKind<'a> { GeoBoundingBox, MisusedGeoRadius, MisusedGeoBoundingBox, + VectorFilterLeftover, + VectorFilterInvalidEmbedder, + VectorFilterMissingFragment, + VectorFilterInvalidFragment, InvalidPrimary, InvalidEscapedNumber, ExpectedEof, @@ -93,11 +97,6 @@ pub enum ErrorKind<'a> { InternalError(error::ErrorKind), DepthLimitReached, External(String), - - VectorFilterLeftover, - VectorFilterInvalidEmbedder, - VectorFilterMissingFragment, - VectorFilterInvalidFragment, } impl<'a> Error<'a> { From dbb670a9eeabed8c4d1a465fb8a0223296988086 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 15:28:58 +0200 Subject: [PATCH 190/312] Remove old split function --- crates/filter-parser/src/lib.rs | 109 -------------------------------- 1 file changed, 109 deletions(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 1e342d8d2..608f73290 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -124,16 +124,6 @@ impl<'a> Token<'a> { Err(Error::new_from_kind(self.span, ErrorKind::NonFiniteFloat)) } } - - /// Split the token by a delimiter and return an iterator of tokens. - /// Each token in the iterator will have its own span that corresponds to a slice of the original token's span. - pub fn split(&self, delimiter: &'a str) -> impl Iterator> + '_ { - let original_addr = self.value().as_ptr() as usize; - self.value().split(delimiter).map(move |part| { - let offset = part.as_ptr() as usize - original_addr; - Token::new(self.span.slice(offset..offset + part.len()), Some(part.to_string())) - }) - } } impl<'a> From> for Token<'a> { @@ -1161,103 +1151,4 @@ pub mod tests { let token: Token = s.into(); assert_eq!(token.value(), s); } - - #[test] - fn split() { - let s = "test string that should not be parsed\n newline"; - let token: Token = s.into(); - let parts: Vec<_> = token.split(" ").collect(); - insta::assert_snapshot!(format!("{parts:#?}"), @r#" - [ - Token { - span: LocatedSpan { - offset: 0, - line: 1, - fragment: "test", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "test", - ), - }, - Token { - span: LocatedSpan { - offset: 5, - line: 1, - fragment: "string", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "string", - ), - }, - Token { - span: LocatedSpan { - offset: 12, - line: 1, - fragment: "that", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "that", - ), - }, - Token { - span: LocatedSpan { - offset: 17, - line: 1, - fragment: "should", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "should", - ), - }, - Token { - span: LocatedSpan { - offset: 24, - line: 1, - fragment: "not", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "not", - ), - }, - Token { - span: LocatedSpan { - offset: 28, - line: 1, - fragment: "be", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "be", - ), - }, - Token { - span: LocatedSpan { - offset: 31, - line: 1, - fragment: "parsed\n", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "parsed\n", - ), - }, - Token { - span: LocatedSpan { - offset: 39, - line: 2, - fragment: "newline", - extra: "test string that should not be parsed\n newline", - }, - value: Some( - "newline", - ), - }, - ] - "#); - } } From 4264abda23d9b1723a0a0e89d94076bbc10a1214 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 15:30:36 +0200 Subject: [PATCH 191/312] Remove debugs --- crates/milli/src/search/facet/filter.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index 4e67814d3..1ddfe96c7 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -230,7 +230,7 @@ impl<'a> Filter<'a> { } pub fn use_vector_filter(&self) -> Option<&Token> { - dbg!(self.condition.use_vector_filter()) + self.condition.use_vector_filter() } } @@ -241,7 +241,7 @@ impl<'a> Filter<'a> { let filterable_attributes_rules = index.filterable_attributes_rules(rtxn)?; for fid in self.condition.fids(MAX_FILTER_DEPTH) { - let attribute = dbg!(fid.value()); + let attribute = fid.value(); if matching_features(attribute, &filterable_attributes_rules) .is_some_and(|(_, features)| features.is_filterable()) || attribute == RESERVED_VECTORS_FIELD_NAME From 13d38d59bfacfe7dbefe3de003de3bddee44df4c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 15:44:11 +0200 Subject: [PATCH 192/312] Remove useless import --- crates/filter-parser/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 608f73290..ae11ccf55 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -60,7 +60,7 @@ use nom::combinator::{cut, eof, map, opt}; use nom::multi::{many0, separated_list1}; use nom::number::complete::recognize_float; use nom::sequence::{delimited, preceded, terminated, tuple}; -use nom::{Finish, Slice}; +use nom::Finish; use nom_locate::LocatedSpan; pub(crate) use value::parse_value; use value::word_exact; From 26da478b5b232fee93ccc02837aa1235d561c864 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 24 Jul 2025 17:27:49 +0200 Subject: [PATCH 193/312] Add query vector to response --- .../src/routes/indexes/search_analytics.rs | 1 + .../src/search/federated/perform.rs | 2 + crates/meilisearch/src/search/mod.rs | 9 + crates/meilisearch/tests/search/hybrid.rs | 201 +++++++++++++++++- crates/milli/src/search/hybrid.rs | 12 +- crates/milli/src/search/mod.rs | 2 + crates/milli/src/search/similar.rs | 1 + crates/milli/src/test_index.rs | 1 + 8 files changed, 223 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/search_analytics.rs b/crates/meilisearch/src/routes/indexes/search_analytics.rs index 07f79eba7..2a8e78059 100644 --- a/crates/meilisearch/src/routes/indexes/search_analytics.rs +++ b/crates/meilisearch/src/routes/indexes/search_analytics.rs @@ -224,6 +224,7 @@ impl SearchAggregator { let SearchResult { hits: _, query: _, + query_vector: _, processing_time_ms, hits_info: _, semantic_hit_count: _, diff --git a/crates/meilisearch/src/search/federated/perform.rs b/crates/meilisearch/src/search/federated/perform.rs index 5ad64d63c..b747d4da4 100644 --- a/crates/meilisearch/src/search/federated/perform.rs +++ b/crates/meilisearch/src/search/federated/perform.rs @@ -13,6 +13,7 @@ use meilisearch_types::error::ResponseError; use meilisearch_types::features::{Network, Remote}; use meilisearch_types::milli::order_by_map::OrderByMap; use meilisearch_types::milli::score_details::{ScoreDetails, WeightedScoreValue}; +use meilisearch_types::milli::vector::Embedding; use meilisearch_types::milli::{self, DocumentId, OrderBy, TimeBudget, DEFAULT_VALUES_PER_FACET}; use roaring::RoaringBitmap; use tokio::task::JoinHandle; @@ -838,6 +839,7 @@ impl SearchByIndex { document_scores, degraded: query_degraded, used_negative_operator: query_used_negative_operator, + query_vector, } = result; candidates |= query_candidates; diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 1c987a70c..1329a6f72 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -841,6 +841,8 @@ pub struct SearchHit { pub struct SearchResult { pub hits: Vec, pub query: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub query_vector: Option>, pub processing_time_ms: u128, #[serde(flatten)] pub hits_info: HitsInfo, @@ -865,6 +867,7 @@ impl fmt::Debug for SearchResult { let SearchResult { hits, query, + query_vector, processing_time_ms, hits_info, facet_distribution, @@ -879,6 +882,9 @@ impl fmt::Debug for SearchResult { debug.field("processing_time_ms", &processing_time_ms); debug.field("hits", &format!("[{} hits returned]", hits.len())); debug.field("query", &query); + if query_vector.is_some() { + debug.field("query_vector", &"[...]"); + } debug.field("hits_info", &hits_info); if *used_negative_operator { debug.field("used_negative_operator", used_negative_operator); @@ -1131,6 +1137,7 @@ pub fn perform_search( document_scores, degraded, used_negative_operator, + query_vector, }, semantic_hit_count, ) = search_from_kind(index_uid, search_kind, search)?; @@ -1221,6 +1228,7 @@ pub fn perform_search( hits: documents, hits_info, query: q.unwrap_or_default(), + query_vector, processing_time_ms: before_search.elapsed().as_millis(), facet_distribution, facet_stats, @@ -1730,6 +1738,7 @@ pub fn perform_similar( document_scores, degraded: _, used_negative_operator: _, + query_vector: _, } = similar.execute().map_err(|err| match err { milli::Error::UserError(milli::UserError::InvalidFilter(_)) => { ResponseError::from_msg(err.to_string(), Code::InvalidSimilarFilter) diff --git a/crates/meilisearch/tests/search/hybrid.rs b/crates/meilisearch/tests/search/hybrid.rs index d95e6fb64..b2970f233 100644 --- a/crates/meilisearch/tests/search/hybrid.rs +++ b/crates/meilisearch/tests/search/hybrid.rs @@ -148,7 +148,70 @@ async fn simple_search() { ) .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}}},{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}}}]"###); + snapshot!(response, @r#" + { + "hits": [ + { + "title": "Captain Planet", + "desc": "He's not part of the Marvel Cinematic Universe", + "id": "2", + "_vectors": { + "default": { + "embeddings": [ + [ + 1.0, + 2.0 + ] + ], + "regenerate": false + } + } + }, + { + "title": "Captain Marvel", + "desc": "a Shazam ersatz", + "id": "3", + "_vectors": { + "default": { + "embeddings": [ + [ + 2.0, + 3.0 + ] + ], + "regenerate": false + } + } + }, + { + "title": "Shazam!", + "desc": "a Captain Marvel ersatz", + "id": "1", + "_vectors": { + "default": { + "embeddings": [ + [ + 1.0, + 3.0 + ] + ], + "regenerate": false + } + } + } + ], + "query": "Captain", + "queryVector": [ + 1.0, + 1.0 + ], + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "semanticHitCount": 0 + } + "#); snapshot!(response["semanticHitCount"], @"0"); let (response, code) = index @@ -157,7 +220,73 @@ async fn simple_search() { ) .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":0.9848484848484848},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":0.9472135901451112}]"###); + snapshot!(response, @r#" + { + "hits": [ + { + "title": "Captain Marvel", + "desc": "a Shazam ersatz", + "id": "3", + "_vectors": { + "default": { + "embeddings": [ + [ + 2.0, + 3.0 + ] + ], + "regenerate": false + } + }, + "_rankingScore": 0.990290343761444 + }, + { + "title": "Captain Planet", + "desc": "He's not part of the Marvel Cinematic Universe", + "id": "2", + "_vectors": { + "default": { + "embeddings": [ + [ + 1.0, + 2.0 + ] + ], + "regenerate": false + } + }, + "_rankingScore": 0.9848484848484848 + }, + { + "title": "Shazam!", + "desc": "a Captain Marvel ersatz", + "id": "1", + "_vectors": { + "default": { + "embeddings": [ + [ + 1.0, + 3.0 + ] + ], + "regenerate": false + } + }, + "_rankingScore": 0.9472135901451112 + } + ], + "query": "Captain", + "queryVector": [ + 1.0, + 1.0 + ], + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "semanticHitCount": 2 + } + "#); snapshot!(response["semanticHitCount"], @"2"); let (response, code) = index @@ -166,7 +295,73 @@ async fn simple_search() { ) .await; snapshot!(code, @"200 OK"); - snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":{"embeddings":[[2.0,3.0]],"regenerate":false}},"_rankingScore":0.990290343761444},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":{"embeddings":[[1.0,2.0]],"regenerate":false}},"_rankingScore":0.974341630935669},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":{"embeddings":[[1.0,3.0]],"regenerate":false}},"_rankingScore":0.9472135901451112}]"###); + snapshot!(response, @r#" + { + "hits": [ + { + "title": "Captain Marvel", + "desc": "a Shazam ersatz", + "id": "3", + "_vectors": { + "default": { + "embeddings": [ + [ + 2.0, + 3.0 + ] + ], + "regenerate": false + } + }, + "_rankingScore": 0.990290343761444 + }, + { + "title": "Captain Planet", + "desc": "He's not part of the Marvel Cinematic Universe", + "id": "2", + "_vectors": { + "default": { + "embeddings": [ + [ + 1.0, + 2.0 + ] + ], + "regenerate": false + } + }, + "_rankingScore": 0.974341630935669 + }, + { + "title": "Shazam!", + "desc": "a Captain Marvel ersatz", + "id": "1", + "_vectors": { + "default": { + "embeddings": [ + [ + 1.0, + 3.0 + ] + ], + "regenerate": false + } + }, + "_rankingScore": 0.9472135901451112 + } + ], + "query": "Captain", + "queryVector": [ + 1.0, + 1.0 + ], + "processingTimeMs": "[duration]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 3, + "semanticHitCount": 3 + } + "#); snapshot!(response["semanticHitCount"], @"3"); } diff --git a/crates/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs index c906e1eb7..e5b4e6787 100644 --- a/crates/milli/src/search/hybrid.rs +++ b/crates/milli/src/search/hybrid.rs @@ -7,7 +7,7 @@ use roaring::RoaringBitmap; use crate::score_details::{ScoreDetails, ScoreValue, ScoringStrategy}; use crate::search::new::{distinct_fid, distinct_single_docid}; use crate::search::SemanticSearch; -use crate::vector::SearchQuery; +use crate::vector::{Embedding, SearchQuery}; use crate::{Index, MatchingWords, Result, Search, SearchResult}; struct ScoreWithRatioResult { @@ -16,6 +16,7 @@ struct ScoreWithRatioResult { document_scores: Vec<(u32, ScoreWithRatio)>, degraded: bool, used_negative_operator: bool, + query_vector: Option, } type ScoreWithRatio = (Vec, f32); @@ -85,6 +86,7 @@ impl ScoreWithRatioResult { document_scores, degraded: results.degraded, used_negative_operator: results.used_negative_operator, + query_vector: results.query_vector, } } @@ -186,6 +188,7 @@ impl ScoreWithRatioResult { degraded: vector_results.degraded | keyword_results.degraded, used_negative_operator: vector_results.used_negative_operator | keyword_results.used_negative_operator, + query_vector: vector_results.query_vector, }, semantic_hit_count, )) @@ -264,7 +267,7 @@ impl Search<'_> { }; search.semantic = Some(SemanticSearch { - vector: Some(vector_query), + vector: Some(vector_query.clone()), embedder_name, embedder, quantized, @@ -277,7 +280,7 @@ impl Search<'_> { let keyword_results = ScoreWithRatioResult::new(keyword_results, 1.0 - semantic_ratio); let vector_results = ScoreWithRatioResult::new(vector_results, semantic_ratio); - let (merge_results, semantic_hit_count) = ScoreWithRatioResult::merge( + let (mut merge_results, semantic_hit_count) = ScoreWithRatioResult::merge( vector_results, keyword_results, self.offset, @@ -286,6 +289,7 @@ impl Search<'_> { search.index, search.rtxn, )?; + merge_results.query_vector = Some(vector_query); assert!(merge_results.documents_ids.len() <= self.limit); Ok((merge_results, Some(semantic_hit_count))) } @@ -321,6 +325,7 @@ fn return_keyword_results( mut document_scores, degraded, used_negative_operator, + query_vector, }: SearchResult, ) -> (SearchResult, Option) { let (documents_ids, document_scores) = if offset >= documents_ids.len() || @@ -347,6 +352,7 @@ fn return_keyword_results( document_scores, degraded, used_negative_operator, + query_vector, }, Some(0), ) diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index 97d542524..cd0d5bc9b 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -295,6 +295,7 @@ impl<'a> Search<'a> { documents_ids, degraded, used_negative_operator, + query_vector: None, }) } } @@ -353,6 +354,7 @@ pub struct SearchResult { pub document_scores: Vec>, pub degraded: bool, pub used_negative_operator: bool, + pub query_vector: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/crates/milli/src/search/similar.rs b/crates/milli/src/search/similar.rs index 903b5fcf9..2235f6436 100644 --- a/crates/milli/src/search/similar.rs +++ b/crates/milli/src/search/similar.rs @@ -130,6 +130,7 @@ impl<'a> Similar<'a> { document_scores, degraded: false, used_negative_operator: false, + query_vector: None, }) } } diff --git a/crates/milli/src/test_index.rs b/crates/milli/src/test_index.rs index 6bb6b1345..d174319d0 100644 --- a/crates/milli/src/test_index.rs +++ b/crates/milli/src/test_index.rs @@ -1097,6 +1097,7 @@ fn bug_3021_fourth() { mut documents_ids, degraded: _, used_negative_operator: _, + query_vector: _, } = search.execute().unwrap(); let primary_key_id = index.fields_ids_map(&rtxn).unwrap().id("primary_key").unwrap(); documents_ids.sort_unstable(); From a7fe2abca4f6425967c55c6789940ad1dd303f04 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 25 Jul 2025 11:45:51 +0200 Subject: [PATCH 194/312] Implement for multi-search --- .../src/search/federated/perform.rs | 37 +++- .../meilisearch/src/search/federated/types.rs | 9 + crates/meilisearch/src/search/mod.rs | 2 +- .../meilisearch/tests/search/multi/proxy.rs | 208 +++++++++++++++++- crates/milli/src/search/hybrid.rs | 18 +- crates/milli/src/search/mod.rs | 70 ++++-- 6 files changed, 318 insertions(+), 26 deletions(-) diff --git a/crates/meilisearch/src/search/federated/perform.rs b/crates/meilisearch/src/search/federated/perform.rs index b747d4da4..4c66e5f68 100644 --- a/crates/meilisearch/src/search/federated/perform.rs +++ b/crates/meilisearch/src/search/federated/perform.rs @@ -47,6 +47,7 @@ pub async fn perform_federated_search( let deadline = before_search + std::time::Duration::from_secs(9); let required_hit_count = federation.limit + federation.offset; + let retrieve_vectors = queries.iter().any(|q| q.retrieve_vectors); let network = index_scheduler.network(); @@ -92,6 +93,7 @@ pub async fn perform_federated_search( federation, mut semantic_hit_count, mut results_by_index, + mut query_vectors, previous_query_data: _, facet_order, } = search_by_index; @@ -123,7 +125,26 @@ pub async fn perform_federated_search( .map(|hit| hit.hit()) .collect(); - // 3.3. merge facets + // 3.3. merge query vectors + let query_vectors = if retrieve_vectors { + for remote_results in remote_results.iter_mut() { + if let Some(remote_vectors) = remote_results.query_vectors.take() { + for (key, value) in remote_vectors.into_iter() { + debug_assert!( + !query_vectors.contains_key(&key), + "Query vector for query {key} already exists" + ); + query_vectors.insert(key, value); + } + } + } + + Some(query_vectors) + } else { + None + }; + + // 3.4. merge facets let (facet_distribution, facet_stats, facets_by_index) = facet_order.merge(federation.merge_facets, remote_results, facets); @@ -141,6 +162,7 @@ pub async fn perform_federated_search( offset: federation.offset, estimated_total_hits, }, + query_vectors, semantic_hit_count, degraded, used_negative_operator, @@ -409,6 +431,7 @@ fn merge_metadata( hits: _, processing_time_ms, hits_info, + query_vectors: _, semantic_hit_count: _, facet_distribution: _, facet_stats: _, @@ -658,6 +681,7 @@ struct SearchByIndex { // Then when merging, we'll update its value if there is any semantic hit semantic_hit_count: Option, results_by_index: Vec, + query_vectors: BTreeMap, previous_query_data: Option<(RankingRules, usize, String)>, // remember the order and name of first index for each facet when merging with index settings // to detect if the order is inconsistent for a facet. @@ -675,6 +699,7 @@ impl SearchByIndex { federation, semantic_hit_count: None, results_by_index: Vec::with_capacity(index_count), + query_vectors: BTreeMap::new(), previous_query_data: None, } } @@ -842,6 +867,16 @@ impl SearchByIndex { query_vector, } = result; + if query.retrieve_vectors { + if let Some(query_vector) = query_vector { + debug_assert!( + !self.query_vectors.contains_key(&query_index), + "Query vector for query {query_index} already exists" + ); + self.query_vectors.insert(query_index, query_vector); + } + } + candidates |= query_candidates; degraded |= query_degraded; used_negative_operator |= query_used_negative_operator; diff --git a/crates/meilisearch/src/search/federated/types.rs b/crates/meilisearch/src/search/federated/types.rs index 3cf28c815..9c96fe768 100644 --- a/crates/meilisearch/src/search/federated/types.rs +++ b/crates/meilisearch/src/search/federated/types.rs @@ -18,6 +18,7 @@ use serde::{Deserialize, Serialize}; use utoipa::ToSchema; use super::super::{ComputedFacets, FacetStats, HitsInfo, SearchHit, SearchQueryWithIndex}; +use crate::milli::vector::Embedding; pub const DEFAULT_FEDERATED_WEIGHT: f64 = 1.0; @@ -117,6 +118,9 @@ pub struct FederatedSearchResult { #[serde(flatten)] pub hits_info: HitsInfo, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub query_vectors: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] pub semantic_hit_count: Option, @@ -144,6 +148,7 @@ impl fmt::Debug for FederatedSearchResult { hits, processing_time_ms, hits_info, + query_vectors, semantic_hit_count, degraded, used_negative_operator, @@ -158,6 +163,10 @@ impl fmt::Debug for FederatedSearchResult { debug.field("processing_time_ms", &processing_time_ms); debug.field("hits", &format!("[{} hits returned]", hits.len())); debug.field("hits_info", &hits_info); + if let Some(query_vectors) = query_vectors { + let known = query_vectors.len(); + debug.field("query_vectors", &format!("[{known} known vectors]")); + } if *used_negative_operator { debug.field("used_negative_operator", used_negative_operator); } diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index 1329a6f72..e08a13a7b 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -1020,7 +1020,7 @@ pub fn prepare_search<'t>( .map_err(milli::Error::from)? } }; - search.semantic( + search.semantic_auto_embedded( embedder_name.clone(), embedder.clone(), *quantized, diff --git a/crates/meilisearch/tests/search/multi/proxy.rs b/crates/meilisearch/tests/search/multi/proxy.rs index 311f69d9e..8f2741202 100644 --- a/crates/meilisearch/tests/search/multi/proxy.rs +++ b/crates/meilisearch/tests/search/multi/proxy.rs @@ -2,8 +2,9 @@ use std::sync::Arc; use actix_http::StatusCode; use meili_snap::{json_string, snapshot}; -use wiremock::matchers::AnyMatcher; -use wiremock::{Mock, MockServer, ResponseTemplate}; +use wiremock::matchers::method; +use wiremock::matchers::{path, AnyMatcher}; +use wiremock::{Mock, MockServer, Request, ResponseTemplate}; use crate::common::{Server, Value, SCORE_DOCUMENTS}; use crate::json; @@ -415,6 +416,209 @@ async fn remote_sharding() { "###); } +#[actix_rt::test] +async fn remote_sharding_retrieve_vectors() { + let ms0 = Server::new().await; + let ms1 = Server::new().await; + let ms2 = Server::new().await; + let index0 = ms0.index("test"); + let index1 = ms1.index("test"); + let index2 = ms2.index("test"); + + // enable feature + + let (response, code) = ms0.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms1.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + let (response, code) = ms2.set_features(json!({"network": true})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response["network"]), @"true"); + + // set self + + let (response, code) = ms0.set_network(json!({"self": "ms0"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms0", + "remotes": {} + } + "###); + let (response, code) = ms1.set_network(json!({"self": "ms1"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms1", + "remotes": {} + } + "###); + let (response, code) = ms2.set_network(json!({"self": "ms2"})).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response), @r###" + { + "self": "ms2", + "remotes": {} + } + "###); + + // setup embedders + + let mock_server = MockServer::start().await; + Mock::given(method("POST")) + .and(path("/")) + .respond_with(move |req: &Request| { + println!("Received request: {:?}", req); + let text = req.body_json::().unwrap().to_lowercase(); + let patterns = [ + ("batman", [1.0, 0.0, 0.0]), + ("dark", [0.0, 0.1, 0.0]), + ("knight", [0.1, 0.1, 0.0]), + ("returns", [0.0, 0.0, 0.2]), + ("part", [0.05, 0.1, 0.0]), + ("1", [0.3, 0.05, 0.0]), + ("2", [0.2, 0.05, 0.0]), + ]; + let mut embedding = vec![0.; 3]; + for (pattern, vector) in patterns { + if text.contains(pattern) { + for (i, v) in vector.iter().enumerate() { + embedding[i] += v; + } + } + } + ResponseTemplate::new(200).set_body_json(json!({ "data": embedding })) + }) + .mount(&mock_server) + .await; + let url = mock_server.uri(); + + for (server, index) in [(&ms0, &index0), (&ms1, &index1), (&ms2, &index2)] { + let (response, code) = index + .update_settings(json!({ + "embedders": { + "rest": { + "source": "rest", + "url": url, + "dimensions": 3, + "request": "{{text}}", + "response": { "data": "{{embedding}}" }, + "documentTemplate": "{{doc.name}}", + }, + }, + })) + .await; + snapshot!(code, @"202 Accepted"); + server.wait_task(response.uid()).await.succeeded(); + } + + // wrap servers + let ms0 = Arc::new(ms0); + let ms1 = Arc::new(ms1); + let ms2 = Arc::new(ms2); + + let rms0 = LocalMeili::new(ms0.clone()).await; + let rms1 = LocalMeili::new(ms1.clone()).await; + let rms2 = LocalMeili::new(ms2.clone()).await; + + // set network + let network = json!({"remotes": { + "ms0": { + "url": rms0.url() + }, + "ms1": { + "url": rms1.url() + }, + "ms2": { + "url": rms2.url() + } + }}); + + let (_response, status_code) = ms0.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + let (_response, status_code) = ms1.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + let (_response, status_code) = ms2.set_network(network.clone()).await; + snapshot!(status_code, @"200 OK"); + + // perform multi-search + let query = "badman returns"; + let request = json!({ + "federation": {}, + "queries": [ + { + "q": query, + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": query, + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": query, + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms2" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r#" + { + "hits": [], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0, + "queryVectors": { + "0": [ + 0.0, + 0.0, + 0.2 + ], + "1": [ + 0.0, + 0.0, + 0.2 + ], + "2": [ + 0.0, + 0.0, + 0.2 + ] + }, + "semanticHitCount": 0, + "remoteErrors": {} + } + "#); +} + #[actix_rt::test] async fn error_unregistered_remote() { let ms0 = Server::new().await; diff --git a/crates/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs index e5b4e6787..547d66409 100644 --- a/crates/milli/src/search/hybrid.rs +++ b/crates/milli/src/search/hybrid.rs @@ -230,7 +230,14 @@ impl Search<'_> { } // no embedder, no semantic search - let Some(SemanticSearch { vector, embedder_name, embedder, quantized, media }) = semantic + let Some(SemanticSearch { + vector, + mut auto_embedded, + embedder_name, + embedder, + quantized, + media, + }) = semantic else { return Ok(return_keyword_results(self.limit, self.offset, keyword_results)); }; @@ -253,7 +260,10 @@ impl Search<'_> { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); match embedder.embed_search(query, Some(deadline)) { - Ok(embedding) => embedding, + Ok(embedding) => { + auto_embedded = true; + embedding + } Err(error) => { tracing::error!(error=%error, "Embedding failed"); return Ok(return_keyword_results( @@ -268,6 +278,7 @@ impl Search<'_> { search.semantic = Some(SemanticSearch { vector: Some(vector_query.clone()), + auto_embedded, embedder_name, embedder, quantized, @@ -280,7 +291,7 @@ impl Search<'_> { let keyword_results = ScoreWithRatioResult::new(keyword_results, 1.0 - semantic_ratio); let vector_results = ScoreWithRatioResult::new(vector_results, semantic_ratio); - let (mut merge_results, semantic_hit_count) = ScoreWithRatioResult::merge( + let (merge_results, semantic_hit_count) = ScoreWithRatioResult::merge( vector_results, keyword_results, self.offset, @@ -289,7 +300,6 @@ impl Search<'_> { search.index, search.rtxn, )?; - merge_results.query_vector = Some(vector_query); assert!(merge_results.documents_ids.len() <= self.limit); Ok((merge_results, Some(semantic_hit_count))) } diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index cd0d5bc9b..de0514b0b 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -32,6 +32,7 @@ pub mod similar; #[derive(Debug, Clone)] pub struct SemanticSearch { vector: Option>, + auto_embedded: bool, media: Option, embedder_name: String, embedder: Arc, @@ -97,7 +98,33 @@ impl<'a> Search<'a> { vector: Option, media: Option, ) -> &mut Search<'a> { - self.semantic = Some(SemanticSearch { embedder_name, embedder, quantized, vector, media }); + self.semantic = Some(SemanticSearch { + embedder_name, + auto_embedded: false, + embedder, + quantized, + vector, + media, + }); + self + } + + pub fn semantic_auto_embedded( + &mut self, + embedder_name: String, + embedder: Arc, + quantized: bool, + vector: Option, + media: Option, + ) -> &mut Search<'a> { + self.semantic = Some(SemanticSearch { + embedder_name, + auto_embedded: true, + embedder, + quantized, + vector, + media, + }); self } @@ -225,6 +252,7 @@ impl<'a> Search<'a> { } let universe = filtered_universe(ctx.index, ctx.txn, &self.filter)?; + let mut query_vector = None; let PartialSearchResult { located_query_terms, candidates, @@ -235,26 +263,32 @@ impl<'a> Search<'a> { } = match self.semantic.as_ref() { Some(SemanticSearch { vector: Some(vector), + auto_embedded, embedder_name, embedder, quantized, media: _, - }) => execute_vector_search( - &mut ctx, - vector, - self.scoring_strategy, - universe, - &self.sort_criteria, - &self.distinct, - self.geo_param, - self.offset, - self.limit, - embedder_name, - embedder, - *quantized, - self.time_budget.clone(), - self.ranking_score_threshold, - )?, + }) => { + if *auto_embedded { + query_vector = Some(vector.clone()); + } + execute_vector_search( + &mut ctx, + vector, + self.scoring_strategy, + universe, + &self.sort_criteria, + &self.distinct, + self.geo_param, + self.offset, + self.limit, + embedder_name, + embedder, + *quantized, + self.time_budget.clone(), + self.ranking_score_threshold, + )? + } _ => execute_search( &mut ctx, self.query.as_deref(), @@ -295,7 +329,7 @@ impl<'a> Search<'a> { documents_ids, degraded, used_negative_operator, - query_vector: None, + query_vector, }) } } From d243504296eba4e3eb0003c325b4162c8ba44502 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 25 Jul 2025 11:58:34 +0200 Subject: [PATCH 195/312] Improve test --- .../meilisearch/tests/search/multi/proxy.rs | 314 +++++++++++++++++- 1 file changed, 304 insertions(+), 10 deletions(-) diff --git a/crates/meilisearch/tests/search/multi/proxy.rs b/crates/meilisearch/tests/search/multi/proxy.rs index 8f2741202..920d88234 100644 --- a/crates/meilisearch/tests/search/multi/proxy.rs +++ b/crates/meilisearch/tests/search/multi/proxy.rs @@ -543,13 +543,13 @@ async fn remote_sharding_retrieve_vectors() { let (_response, status_code) = ms2.set_network(network.clone()).await; snapshot!(status_code, @"200 OK"); - // perform multi-search - let query = "badman returns"; + // multi vector search: one query per remote + let request = json!({ "federation": {}, "queries": [ { - "q": query, + "q": "batman", "indexUid": "test", "hybrid": { "semanticRatio": 1.0, @@ -561,7 +561,7 @@ async fn remote_sharding_retrieve_vectors() { } }, { - "q": query, + "q": "dark knight", "indexUid": "test", "hybrid": { "semanticRatio": 1.0, @@ -573,7 +573,7 @@ async fn remote_sharding_retrieve_vectors() { } }, { - "q": query, + "q": "returns", "indexUid": "test", "hybrid": { "semanticRatio": 1.0, @@ -598,14 +598,14 @@ async fn remote_sharding_retrieve_vectors() { "estimatedTotalHits": 0, "queryVectors": { "0": [ + 1.0, 0.0, - 0.0, - 0.2 + 0.0 ], "1": [ - 0.0, - 0.0, - 0.2 + 0.1, + 0.2, + 0.0 ], "2": [ 0.0, @@ -617,6 +617,300 @@ async fn remote_sharding_retrieve_vectors() { "remoteErrors": {} } "#); + + // multi vector search: two local queries, one remote + + let request = json!({ + "federation": {}, + "queries": [ + { + "q": "batman", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": "dark knight", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": "returns", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms2" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r#" + { + "hits": [], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0, + "queryVectors": { + "0": [ + 1.0, + 0.0, + 0.0 + ], + "1": [ + 0.1, + 0.2, + 0.0 + ], + "2": [ + 0.0, + 0.0, + 0.2 + ] + }, + "semanticHitCount": 0, + "remoteErrors": {} + } + "#); + + // multi vector search: two queries on the same remote + + let request = json!({ + "federation": {}, + "queries": [ + { + "q": "batman", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": "dark knight", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": "returns", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r#" + { + "hits": [], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0, + "queryVectors": { + "0": [ + 1.0, + 0.0, + 0.0 + ], + "1": [ + 0.1, + 0.2, + 0.0 + ], + "2": [ + 0.0, + 0.0, + 0.2 + ] + }, + "semanticHitCount": 0, + "remoteErrors": {} + } + "#); + + // multi search: two vector, one keyword + + let request = json!({ + "federation": {}, + "queries": [ + { + "q": "batman", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms0" + } + }, + { + "q": "dark knight", + "indexUid": "test", + "hybrid": { + "semanticRatio": 0.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": "returns", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r#" + { + "hits": [], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0, + "queryVectors": { + "0": [ + 1.0, + 0.0, + 0.0 + ], + "2": [ + 0.0, + 0.0, + 0.2 + ] + }, + "semanticHitCount": 0, + "remoteErrors": {} + } + "#); + + // multi vector search: no local queries, all remote + + let request = json!({ + "federation": {}, + "queries": [ + { + "q": "batman", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": "dark knight", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + { + "q": "returns", + "indexUid": "test", + "hybrid": { + "semanticRatio": 1.0, + "embedder": "rest" + }, + "retrieveVectors": true, + "federationOptions": { + "remote": "ms1" + } + }, + ] + }); + + let (response, _status_code) = ms0.multi_search(request.clone()).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(response, { ".processingTimeMs" => "[time]" }), @r#" + { + "hits": [], + "processingTimeMs": "[time]", + "limit": 20, + "offset": 0, + "estimatedTotalHits": 0, + "queryVectors": { + "0": [ + 1.0, + 0.0, + 0.0 + ], + "1": [ + 0.1, + 0.2, + 0.0 + ], + "2": [ + 0.0, + 0.0, + 0.2 + ] + }, + "remoteErrors": {} + } + "#); } #[actix_rt::test] From a439f57d7076682e449f1102cf275429a6a54b62 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 25 Jul 2025 13:41:31 +0200 Subject: [PATCH 196/312] Update tests --- crates/meilisearch/tests/search/hybrid.rs | 4 ---- crates/meilisearch/tests/search/multi/mod.rs | 15 +++++++++++++-- .../tests/vector/binary_quantized.rs | 9 +++++++-- crates/meilisearch/tests/vector/mod.rs | 18 ++++++++++++++---- 4 files changed, 34 insertions(+), 12 deletions(-) diff --git a/crates/meilisearch/tests/search/hybrid.rs b/crates/meilisearch/tests/search/hybrid.rs index b2970f233..fce94dd7f 100644 --- a/crates/meilisearch/tests/search/hybrid.rs +++ b/crates/meilisearch/tests/search/hybrid.rs @@ -201,10 +201,6 @@ async fn simple_search() { } ], "query": "Captain", - "queryVector": [ - 1.0, - 1.0 - ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, diff --git a/crates/meilisearch/tests/search/multi/mod.rs b/crates/meilisearch/tests/search/multi/mod.rs index b9eed56da..3b0d9380e 100644 --- a/crates/meilisearch/tests/search/multi/mod.rs +++ b/crates/meilisearch/tests/search/multi/mod.rs @@ -3703,7 +3703,7 @@ async fn federation_vector_two_indexes() { ]})) .await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(response, { ".processingTimeMs" => "[duration]", ".**._rankingScore" => "[score]" }), @r###" + snapshot!(json_string!(response, { ".processingTimeMs" => "[duration]", ".**._rankingScore" => "[score]" }), @r#" { "hits": [ { @@ -3911,9 +3911,20 @@ async fn federation_vector_two_indexes() { "limit": 20, "offset": 0, "estimatedTotalHits": 8, + "queryVectors": { + "0": [ + 1.0, + 0.0, + 0.5 + ], + "1": [ + 0.8, + 0.6 + ] + }, "semanticHitCount": 6 } - "###); + "#); // hybrid search, distinct embedder let (response, code) = server diff --git a/crates/meilisearch/tests/vector/binary_quantized.rs b/crates/meilisearch/tests/vector/binary_quantized.rs index 6fcfa3563..e0fa9a37c 100644 --- a/crates/meilisearch/tests/vector/binary_quantized.rs +++ b/crates/meilisearch/tests/vector/binary_quantized.rs @@ -323,15 +323,20 @@ async fn binary_quantize_clear_documents() { // Make sure the arroy DB has been cleared let (documents, _code) = index.search_post(json!({ "hybrid": { "embedder": "manual" }, "vector": [1, 1, 1] })).await; - snapshot!(documents, @r###" + snapshot!(documents, @r#" { "hits": [], "query": "", + "queryVector": [ + 1.0, + 1.0, + 1.0 + ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, "estimatedTotalHits": 0, "semanticHitCount": 0 } - "###); + "#); } diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index ca2ecc998..5d056e365 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -685,17 +685,22 @@ async fn clear_documents() { // Make sure the arroy DB has been cleared let (documents, _code) = index.search_post(json!({ "vector": [1, 1, 1], "hybrid": {"embedder": "manual"} })).await; - snapshot!(documents, @r###" + snapshot!(documents, @r#" { "hits": [], "query": "", + "queryVector": [ + 1.0, + 1.0, + 1.0 + ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, "estimatedTotalHits": 0, "semanticHitCount": 0 } - "###); + "#); } #[actix_rt::test] @@ -739,7 +744,7 @@ async fn add_remove_one_vector_4588() { json!({"vector": [1, 1, 1], "hybrid": {"semanticRatio": 1.0, "embedder": "manual"} }), ) .await; - snapshot!(documents, @r###" + snapshot!(documents, @r#" { "hits": [ { @@ -748,13 +753,18 @@ async fn add_remove_one_vector_4588() { } ], "query": "", + "queryVector": [ + 1.0, + 1.0, + 1.0 + ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, "estimatedTotalHits": 1, "semanticHitCount": 1 } - "###); + "#); let (documents, _code) = index .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() }) From 10567b150cc95aed2cc102c85b71f03aca03785c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 25 Jul 2025 14:25:35 +0200 Subject: [PATCH 197/312] Continue updating tests --- crates/meilisearch/tests/search/hybrid.rs | 8 -------- crates/meilisearch/tests/search/multi/mod.rs | 15 +++++++++++++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/crates/meilisearch/tests/search/hybrid.rs b/crates/meilisearch/tests/search/hybrid.rs index fce94dd7f..172242e47 100644 --- a/crates/meilisearch/tests/search/hybrid.rs +++ b/crates/meilisearch/tests/search/hybrid.rs @@ -272,10 +272,6 @@ async fn simple_search() { } ], "query": "Captain", - "queryVector": [ - 1.0, - 1.0 - ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, @@ -347,10 +343,6 @@ async fn simple_search() { } ], "query": "Captain", - "queryVector": [ - 1.0, - 1.0 - ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, diff --git a/crates/meilisearch/tests/search/multi/mod.rs b/crates/meilisearch/tests/search/multi/mod.rs index 3b0d9380e..16ee3906e 100644 --- a/crates/meilisearch/tests/search/multi/mod.rs +++ b/crates/meilisearch/tests/search/multi/mod.rs @@ -3934,7 +3934,7 @@ async fn federation_vector_two_indexes() { ]})) .await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(response, { ".processingTimeMs" => "[duration]", ".**._rankingScore" => "[score]" }), @r###" + snapshot!(json_string!(response, { ".processingTimeMs" => "[duration]", ".**._rankingScore" => "[score]" }), @r#" { "hits": [ { @@ -4150,9 +4150,20 @@ async fn federation_vector_two_indexes() { "limit": 20, "offset": 0, "estimatedTotalHits": 8, + "queryVectors": { + "0": [ + 1.0, + 0.0, + 0.5 + ], + "1": [ + -1.0, + 0.6 + ] + }, "semanticHitCount": 8 } - "###); + "#); } #[actix_rt::test] From f6bc6854f8080af25453f0722016716c961ca36e Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 28 Jul 2025 11:10:55 +0200 Subject: [PATCH 198/312] Fix key action inconsistencies --- crates/meilisearch-types/src/keys.rs | 75 ++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 3 deletions(-) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index e210f8df3..aec3199a3 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -233,9 +233,6 @@ pub enum Action { #[serde(rename = "*")] #[deserr(rename = "*")] All = 0, - #[serde(rename = "*.get")] - #[deserr(rename = "*.get")] - AllGet, #[serde(rename = "search")] #[deserr(rename = "search")] Search, @@ -365,6 +362,9 @@ pub enum Action { #[serde(rename = "chatsSettings.update")] #[deserr(rename = "chatsSettings.update")] ChatsSettingsUpdate, + #[serde(rename = "*.get")] + #[deserr(rename = "*.get")] + AllGet, } impl Action { @@ -403,6 +403,7 @@ impl Action { METRICS_GET => Some(Self::MetricsGet), DUMPS_ALL => Some(Self::DumpsAll), DUMPS_CREATE => Some(Self::DumpsCreate), + SNAPSHOTS_ALL => Some(Self::SnapshotsAll), SNAPSHOTS_CREATE => Some(Self::SnapshotsCreate), VERSION => Some(Self::Version), KEYS_CREATE => Some(Self::KeysAdd), @@ -411,8 +412,10 @@ impl Action { KEYS_DELETE => Some(Self::KeysDelete), EXPERIMENTAL_FEATURES_GET => Some(Self::ExperimentalFeaturesGet), EXPERIMENTAL_FEATURES_UPDATE => Some(Self::ExperimentalFeaturesUpdate), + EXPORT => Some(Self::Export), NETWORK_GET => Some(Self::NetworkGet), NETWORK_UPDATE => Some(Self::NetworkUpdate), + ALL_GET => Some(Self::AllGet), _otherwise => None, } } @@ -497,6 +500,7 @@ pub mod actions { pub const METRICS_GET: u8 = MetricsGet.repr(); pub const DUMPS_ALL: u8 = DumpsAll.repr(); pub const DUMPS_CREATE: u8 = DumpsCreate.repr(); + pub const SNAPSHOTS_ALL: u8 = SnapshotsAll.repr(); pub const SNAPSHOTS_CREATE: u8 = SnapshotsCreate.repr(); pub const VERSION: u8 = Version.repr(); pub const KEYS_CREATE: u8 = KeysAdd.repr(); @@ -519,3 +523,68 @@ pub mod actions { pub const CHATS_SETTINGS_GET: u8 = ChatsSettingsGet.repr(); pub const CHATS_SETTINGS_UPDATE: u8 = ChatsSettingsUpdate.repr(); } + +#[cfg(test)] +pub(crate) mod test { + use super::actions::*; + use super::Action::*; + use super::*; + + #[test] + fn test_action_repr_and_constants() { + assert!(All.repr() == 0 && ALL == 0); + assert!(Search.repr() == 1 && SEARCH == 1); + assert!(DocumentsAll.repr() == 2 && DOCUMENTS_ALL == 2); + assert!(DocumentsAdd.repr() == 3 && DOCUMENTS_ADD == 3); + assert!(DocumentsGet.repr() == 4 && DOCUMENTS_GET == 4); + assert!(DocumentsDelete.repr() == 5 && DOCUMENTS_DELETE == 5); + assert!(IndexesAll.repr() == 6 && INDEXES_ALL == 6); + assert!(IndexesAdd.repr() == 7 && INDEXES_CREATE == 7); + assert!(IndexesGet.repr() == 8 && INDEXES_GET == 8); + assert!(IndexesUpdate.repr() == 9 && INDEXES_UPDATE == 9); + assert!(IndexesDelete.repr() == 10 && INDEXES_DELETE == 10); + assert!(IndexesSwap.repr() == 11 && INDEXES_SWAP == 11); + assert!(TasksAll.repr() == 12 && TASKS_ALL == 12); + assert!(TasksCancel.repr() == 13 && TASKS_CANCEL == 13); + assert!(TasksDelete.repr() == 14 && TASKS_DELETE == 14); + assert!(TasksGet.repr() == 15 && TASKS_GET == 15); + assert!(SettingsAll.repr() == 16 && SETTINGS_ALL == 16); + assert!(SettingsGet.repr() == 17 && SETTINGS_GET == 17); + assert!(SettingsUpdate.repr() == 18 && SETTINGS_UPDATE == 18); + assert!(StatsAll.repr() == 19 && STATS_ALL == 19); + assert!(StatsGet.repr() == 20 && STATS_GET == 20); + assert!(MetricsAll.repr() == 21 && METRICS_ALL == 21); + assert!(MetricsGet.repr() == 22 && METRICS_GET == 22); + assert!(DumpsAll.repr() == 23 && DUMPS_ALL == 23); + assert!(DumpsCreate.repr() == 24 && DUMPS_CREATE == 24); + assert!(SnapshotsAll.repr() == 25 && SNAPSHOTS_ALL == 25); + assert!(SnapshotsCreate.repr() == 26 && SNAPSHOTS_CREATE == 26); + assert!(Version.repr() == 27 && VERSION == 27); + assert!(KeysAdd.repr() == 28 && KEYS_CREATE == 28); + assert!(KeysGet.repr() == 29 && KEYS_GET == 29); + assert!(KeysUpdate.repr() == 30 && KEYS_UPDATE == 30); + assert!(KeysDelete.repr() == 31 && KEYS_DELETE == 31); + assert!(ExperimentalFeaturesGet.repr() == 32 && EXPERIMENTAL_FEATURES_GET == 32); + assert!(ExperimentalFeaturesUpdate.repr() == 33 && EXPERIMENTAL_FEATURES_UPDATE == 33); + assert!(Export.repr() == 34 && EXPORT == 34); + assert!(NetworkGet.repr() == 35 && NETWORK_GET == 35); + assert!(NetworkUpdate.repr() == 36 && NETWORK_UPDATE == 36); + assert!(ChatCompletions.repr() == 37 && CHAT_COMPLETIONS == 37); + assert!(ChatsAll.repr() == 38 && CHATS_ALL == 38); + assert!(ChatsGet.repr() == 39 && CHATS_GET == 39); + assert!(ChatsDelete.repr() == 40 && CHATS_DELETE == 40); + assert!(ChatsSettingsAll.repr() == 41 && CHATS_SETTINGS_ALL == 41); + assert!(ChatsSettingsGet.repr() == 42 && CHATS_SETTINGS_GET == 42); + assert!(ChatsSettingsUpdate.repr() == 43 && CHATS_SETTINGS_UPDATE == 43); + assert!(AllGet.repr() == 44 && ALL_GET == 44); + } + + #[test] + fn test_from_repr() { + for action in enum_iterator::all::() { + let repr = action.repr(); + let action_from_repr = Action::from_repr(repr); + assert_eq!(Some(action), action_from_repr, "Failed for action: {:?}", action); + } + } +} From d90c76d3cc24f9822ae091174aa27499176e646f Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 28 Jul 2025 11:35:15 +0200 Subject: [PATCH 199/312] Update tests --- crates/meilisearch/tests/auth/api_keys.rs | 2 +- crates/meilisearch/tests/auth/errors.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs index 0b8a3d2c5..6dc3f429b 100644 --- a/crates/meilisearch/tests/auth/api_keys.rs +++ b/crates/meilisearch/tests/auth/api_keys.rs @@ -421,7 +421,7 @@ async fn error_add_api_key_invalid_parameters_actions() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r#" { - "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `*.get`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", + "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" diff --git a/crates/meilisearch/tests/auth/errors.rs b/crates/meilisearch/tests/auth/errors.rs index e8d935fde..b16ccb2f5 100644 --- a/crates/meilisearch/tests/auth/errors.rs +++ b/crates/meilisearch/tests/auth/errors.rs @@ -93,7 +93,7 @@ async fn create_api_key_bad_actions() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r#" { - "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `*.get`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`", + "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" From 478f374b9d42465ec5ce1bfc31fa7e692f0b259b Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 28 Jul 2025 16:23:26 +0200 Subject: [PATCH 200/312] Add benchmark --- crates/benchmarks/Cargo.toml | 4 ++ .../benchmarks/benches/filter_starts_with.rs | 72 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 crates/benchmarks/benches/filter_starts_with.rs diff --git a/crates/benchmarks/Cargo.toml b/crates/benchmarks/Cargo.toml index 9dccc444b..25b44436d 100644 --- a/crates/benchmarks/Cargo.toml +++ b/crates/benchmarks/Cargo.toml @@ -51,3 +51,7 @@ harness = false [[bench]] name = "indexing" harness = false + +[[bench]] +name = "filter_starts_with" +harness = false diff --git a/crates/benchmarks/benches/filter_starts_with.rs b/crates/benchmarks/benches/filter_starts_with.rs new file mode 100644 index 000000000..b1d0b502f --- /dev/null +++ b/crates/benchmarks/benches/filter_starts_with.rs @@ -0,0 +1,72 @@ +mod datasets_paths; +mod utils; + +use criterion::{criterion_group, criterion_main}; +use milli::update::Settings; +use milli::FilterableAttributesRule; +use utils::Conf; + +#[cfg(not(windows))] +#[global_allocator] +static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; + +fn base_conf(builder: &mut Settings) { + let displayed_fields = + ["geonameid", "name"] + .iter() + .map(|s| s.to_string()) + .collect(); + builder.set_displayed_fields(displayed_fields); + + let filterable_fields = ["name"] + .iter() + .map(|s| FilterableAttributesRule::Field(s.to_string())) + .collect(); + builder.set_filterable_fields(filterable_fields); +} + +#[rustfmt::skip] +const BASE_CONF: Conf = Conf { + dataset: datasets_paths::SMOL_ALL_COUNTRIES, + dataset_format: "jsonl", + queries: &[ + "", + ], + configure: base_conf, + primary_key: Some("geonameid"), + ..Conf::BASE +}; + +fn filter_starts_with(c: &mut criterion::Criterion) { + #[rustfmt::skip] + let confs = &[ + utils::Conf { + group_name: "1 letter", + filter: Some("name STARTS WITH e"), + ..BASE_CONF + }, + + utils::Conf { + group_name: "2 letters", + filter: Some("name STARTS WITH es"), + ..BASE_CONF + }, + + utils::Conf { + group_name: "3 letters", + filter: Some("name STARTS WITH est"), + ..BASE_CONF + }, + + utils::Conf { + group_name: "6 letters", + filter: Some("name STARTS WITH estoni"), + ..BASE_CONF + } + ]; + + utils::run_benches(c, confs); +} + +criterion_group!(benches, filter_starts_with); +criterion_main!(benches); From e8a818f53d6387194f645df2dea87acdb6327d28 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 28 Jul 2025 16:24:04 +0200 Subject: [PATCH 201/312] Optimize the filter --- crates/milli/src/search/facet/filter.rs | 37 +++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index c3eba8031..54c7535dd 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -18,6 +18,7 @@ use crate::heed_codec::facet::{ FacetGroupKey, FacetGroupKeyCodec, FacetGroupValue, FacetGroupValueCodec, }; use crate::index::db_name::FACET_ID_STRING_DOCIDS; +use crate::search::facet::facet_range_search::find_docids_of_facet_within_bounds; use crate::{ distance_between_two_points, lat_lng_to_xyz, FieldId, FieldsIdsMap, FilterableAttributesFeatures, FilterableAttributesRule, Index, InternalError, Result, @@ -416,7 +417,43 @@ impl<'a> Filter<'a> { return Ok(docids); } Condition::StartsWith { keyword: _, word } => { + // There are two algorithms: + // + // - The first one looks directly at level 0 of the facet group database. + // This pessimistic approach is more efficient when the value is unique. + // + // - The second one is recursive over levels. + // This is more efficient when the prefix is common among many values. + let value = crate::normalize_facet(word.value()); + + if value.len() <= 6 { + // 6 is abitrary, but it works well in practice + let mut value2 = value.as_bytes().to_owned(); + if let Some(last) = value2.last_mut() { + if *last != 255 { + *last += 1; + if let Ok(value2) = String::from_utf8(value2) { + // The idea here is that "STARTS WITH baba" is the same as "baba <= value < babb". + // We just increase the last letter to find the upper bound. + // The result could be invalid utf8, so it can fallback. + let mut docids = RoaringBitmap::new(); + find_docids_of_facet_within_bounds( + rtxn, + strings_db, + field_id, + &Included(&value), + &Excluded(&value2), + universe, + &mut docids, + )?; + + return Ok(docids); + } + } + } + } + let base = FacetGroupKey { field_id, level: 0, left_bound: value.as_str() }; let docids = strings_db .prefix_iter(rtxn, &base)? From 691a9ae4b18c00459ca2512429a364fc2b8722cf Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 28 Jul 2025 16:24:11 +0200 Subject: [PATCH 202/312] Format --- crates/benchmarks/benches/filter_starts_with.rs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/crates/benchmarks/benches/filter_starts_with.rs b/crates/benchmarks/benches/filter_starts_with.rs index b1d0b502f..a7682cbf8 100644 --- a/crates/benchmarks/benches/filter_starts_with.rs +++ b/crates/benchmarks/benches/filter_starts_with.rs @@ -11,17 +11,11 @@ use utils::Conf; static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc; fn base_conf(builder: &mut Settings) { - let displayed_fields = - ["geonameid", "name"] - .iter() - .map(|s| s.to_string()) - .collect(); + let displayed_fields = ["geonameid", "name"].iter().map(|s| s.to_string()).collect(); builder.set_displayed_fields(displayed_fields); - let filterable_fields = ["name"] - .iter() - .map(|s| FilterableAttributesRule::Field(s.to_string())) - .collect(); + let filterable_fields = + ["name"].iter().map(|s| FilterableAttributesRule::Field(s.to_string())).collect(); builder.set_filterable_fields(filterable_fields); } From 224892e692fb31df55dc25b275d9412d247314a2 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 28 Jul 2025 16:28:06 +0200 Subject: [PATCH 203/312] Enable new algorithm every time --- crates/milli/src/search/facet/filter.rs | 41 ++++++++++++------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index 54c7535dd..907b12f1a 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -427,29 +427,26 @@ impl<'a> Filter<'a> { let value = crate::normalize_facet(word.value()); - if value.len() <= 6 { - // 6 is abitrary, but it works well in practice - let mut value2 = value.as_bytes().to_owned(); - if let Some(last) = value2.last_mut() { - if *last != 255 { - *last += 1; - if let Ok(value2) = String::from_utf8(value2) { - // The idea here is that "STARTS WITH baba" is the same as "baba <= value < babb". - // We just increase the last letter to find the upper bound. - // The result could be invalid utf8, so it can fallback. - let mut docids = RoaringBitmap::new(); - find_docids_of_facet_within_bounds( - rtxn, - strings_db, - field_id, - &Included(&value), - &Excluded(&value2), - universe, - &mut docids, - )?; + let mut value2 = value.as_bytes().to_owned(); + if let Some(last) = value2.last_mut() { + if *last != 255 { + *last += 1; + if let Ok(value2) = String::from_utf8(value2) { + // The idea here is that "STARTS WITH baba" is the same as "baba <= value < babb". + // We just increase the last letter to find the upper bound. + // The result could be invalid utf8, so it can fallback. + let mut docids = RoaringBitmap::new(); + find_docids_of_facet_within_bounds( + rtxn, + strings_db, + field_id, + &Included(&value), + &Excluded(&value2), + universe, + &mut docids, + )?; - return Ok(docids); - } + return Ok(docids); } } } From 48a5f4db2d48b2b8fb98abf9e74acf90d349e9c0 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 28 Jul 2025 16:42:33 +0200 Subject: [PATCH 204/312] Improve comment --- crates/milli/src/search/facet/filter.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index 907b12f1a..955e75753 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -419,11 +419,12 @@ impl<'a> Filter<'a> { Condition::StartsWith { keyword: _, word } => { // There are two algorithms: // - // - The first one looks directly at level 0 of the facet group database. - // This pessimistic approach is more efficient when the value is unique. - // - // - The second one is recursive over levels. + // - The first one is recursive over levels. // This is more efficient when the prefix is common among many values. + // + // - The second one looks directly at level 0 of the facet group database. + // This pessimistic approach is more efficient when the value is unique. + // It's used as a fallback. let value = crate::normalize_facet(word.value()); From 6c3dd83ae528a8b82614880c0206cd7886a187c7 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 09:03:48 +0200 Subject: [PATCH 205/312] Fix old test --- crates/milli/src/test_index.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/crates/milli/src/test_index.rs b/crates/milli/src/test_index.rs index 0ec348301..a6ed9ad91 100644 --- a/crates/milli/src/test_index.rs +++ b/crates/milli/src/test_index.rs @@ -1340,11 +1340,10 @@ fn vectors_are_never_indexed_as_searchable_or_filterable() { assert!(results.candidates.is_empty()); let mut search = index.search(&rtxn); - let results = search - .filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()) - .execute() - .unwrap_err(); - assert!(matches!(results, Error::UserError(UserError::InvalidFilter(_)))); + let results = + dbg!(search.filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()).execute()) + .unwrap(); + assert!(results.candidates.is_empty()); index .update_settings(|settings| { @@ -1375,6 +1374,6 @@ fn vectors_are_never_indexed_as_searchable_or_filterable() { let results = search .filter(Filter::from_str("_vectors.doggo = 6789").unwrap().unwrap()) .execute() - .unwrap_err(); - assert!(matches!(results, Error::UserError(UserError::InvalidFilter(_)))); + .unwrap(); + assert!(results.candidates.is_empty()); } From 66b6e47494c0d6809c0c78fd7f794dcd673afc62 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 10:52:21 +0200 Subject: [PATCH 206/312] Remove warning --- crates/milli/src/test_index.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/crates/milli/src/test_index.rs b/crates/milli/src/test_index.rs index a6ed9ad91..12ac4e158 100644 --- a/crates/milli/src/test_index.rs +++ b/crates/milli/src/test_index.rs @@ -19,9 +19,7 @@ use crate::update::{ }; use crate::vector::settings::{EmbedderSource, EmbeddingSettings}; use crate::vector::RuntimeEmbedders; -use crate::{ - db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult, UserError, -}; +use crate::{db_snap, obkv_to_json, Filter, FilterableAttributesRule, Index, Search, SearchResult}; pub(crate) struct TempIndex { pub inner: Index, From 3580b3a4ef5370c5168954378e2aa4df9464b6af Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 10:56:54 +0200 Subject: [PATCH 207/312] Remove userProvided from fragments --- crates/meilisearch/tests/search/filters.rs | 5 +---- crates/milli/src/search/facet/filter_vector.rs | 5 ++++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 67f9ebb71..209261f70 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -930,9 +930,6 @@ async fn vector_filter_specific_fragment() { { "name": "kefir" }, - { - "name": "echo" - }, { "name": "intel" }, @@ -944,7 +941,7 @@ async fn vector_filter_specific_fragment() { "processingTimeMs": "[duration]", "limit": 20, "offset": 0, - "estimatedTotalHits": 4 + "estimatedTotalHits": 3 } "#); } diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 2ddd801ed..2cde3aaa7 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -119,7 +119,10 @@ fn evaluate_inner( .collect(), })?; - arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| bitmap.clone())? + let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| { + bitmap.clone() - user_provided_docsids + })? } VectorFilter::DocumentTemplate => { if !embedding_config.fragments.as_slice().is_empty() { From 223df5a43346ea4bdd2c63218c9e0985d21a9f30 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 11:02:59 +0200 Subject: [PATCH 208/312] Remove incorrect break --- crates/milli/src/vector/mod.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/crates/milli/src/vector/mod.rs b/crates/milli/src/vector/mod.rs index f64223e41..088d98b72 100644 --- a/crates/milli/src/vector/mod.rs +++ b/crates/milli/src/vector/mod.rs @@ -556,9 +556,6 @@ impl ArroyWrapper { for reader in self.readers(rtxn, self.quantized_db()) { let reader = reader?; let documents = reader.item_ids(); - if documents.is_empty() { - break; - } stats.documents |= documents; stats.number_of_embeddings += documents.len(); } @@ -566,9 +563,6 @@ impl ArroyWrapper { for reader in self.readers(rtxn, self.angular_db()) { let reader = reader?; let documents = reader.item_ids(); - if documents.is_empty() { - break; - } stats.documents |= documents; stats.number_of_embeddings += documents.len(); } From 93864009cc68a5af622ef0ce2f6d7f96ec85f332 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 11:04:08 +0200 Subject: [PATCH 209/312] Rename variable with typo Co-Authored-By: Louis Dureuil --- crates/milli/src/search/facet/filter_vector.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 2cde3aaa7..625bd5dde 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -119,9 +119,9 @@ fn evaluate_inner( .collect(), })?; - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + let user_provided_docids = embedder_info.embedding_status.user_provided_docids(); arroy_wrapper.items_in_store(rtxn, fragment_config.id, |bitmap| { - bitmap.clone() - user_provided_docsids + bitmap.clone() - user_provided_docids })? } VectorFilter::DocumentTemplate => { @@ -129,14 +129,14 @@ fn evaluate_inner( return Ok(RoaringBitmap::new()); } - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); + let user_provided_docids = embedder_info.embedding_status.user_provided_docids(); let mut stats = ArroyStats::default(); arroy_wrapper.aggregate_stats(rtxn, &mut stats)?; - stats.documents - user_provided_docsids.clone() + stats.documents - user_provided_docids.clone() } VectorFilter::UserProvided => { - let user_provided_docsids = embedder_info.embedding_status.user_provided_docids(); - user_provided_docsids.clone() + let user_provided_docids = embedder_info.embedding_status.user_provided_docids(); + user_provided_docids.clone() } VectorFilter::Regenerate => { let mut stats = ArroyStats::default(); From 60acdf8574e9e0d2e7e26eb553dd5410554f8b9c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 11:05:16 +0200 Subject: [PATCH 210/312] Fix grammar Co-Authored-By: Louis Dureuil --- crates/meilisearch/tests/search/filters.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 209261f70..76f252a6d 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -788,7 +788,7 @@ async fn vector_filter_missing_fragment() { } #[actix_rt::test] -async fn vector_filter_non_existant_embedder() { +async fn vector_filter_nonexistent_embedder() { let index = shared_index_for_fragments().await; let (value, _code) = index From 5e867f7ce068c2ca8a94cb9e22945a70dc9f6667 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 16:47:20 +0200 Subject: [PATCH 211/312] Add webhooks api key action --- crates/meilisearch-types/src/keys.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index aec3199a3..2eddb9547 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -365,6 +365,12 @@ pub enum Action { #[serde(rename = "*.get")] #[deserr(rename = "*.get")] AllGet, + #[serde(rename = "webhooks.get")] + #[deserr(rename = "webhooks.get")] + WebhooksGet, + #[serde(rename = "webhooks.update")] + #[deserr(rename = "webhooks.update")] + WebhooksUpdate, } impl Action { @@ -416,6 +422,8 @@ impl Action { NETWORK_GET => Some(Self::NetworkGet), NETWORK_UPDATE => Some(Self::NetworkUpdate), ALL_GET => Some(Self::AllGet), + WEBHOOKS_GET => Some(Self::WebhooksGet), + WEBHOOKS_UPDATE => Some(Self::WebhooksUpdate), _otherwise => None, } } @@ -463,6 +471,8 @@ impl Action { ChatsDelete => false, ChatsSettingsGet => true, ChatsSettingsUpdate => false, + WebhooksGet => true, + WebhooksUpdate => false, } } @@ -522,6 +532,9 @@ pub mod actions { pub const CHATS_SETTINGS_ALL: u8 = ChatsSettingsAll.repr(); pub const CHATS_SETTINGS_GET: u8 = ChatsSettingsGet.repr(); pub const CHATS_SETTINGS_UPDATE: u8 = ChatsSettingsUpdate.repr(); + + pub const WEBHOOKS_GET: u8 = WebhooksGet.repr(); + pub const WEBHOOKS_UPDATE: u8 = WebhooksUpdate.repr(); } #[cfg(test)] @@ -577,6 +590,8 @@ pub(crate) mod test { assert!(ChatsSettingsGet.repr() == 42 && CHATS_SETTINGS_GET == 42); assert!(ChatsSettingsUpdate.repr() == 43 && CHATS_SETTINGS_UPDATE == 43); assert!(AllGet.repr() == 44 && ALL_GET == 44); + assert!(WebhooksGet.repr() == 45 && WEBHOOKS_GET == 45); + assert!(WebhooksUpdate.repr() == 46 && WEBHOOKS_UPDATE == 46); } #[test] From 5567653c96cfefe1779e95c54bde730bd3758f7a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 16:47:28 +0200 Subject: [PATCH 212/312] Fix network documentation --- crates/meilisearch/src/routes/network.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/network.rs b/crates/meilisearch/src/routes/network.rs index 7e58df113..6ee68ea33 100644 --- a/crates/meilisearch/src/routes/network.rs +++ b/crates/meilisearch/src/routes/network.rs @@ -51,7 +51,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { get, path = "", tag = "Network", - security(("Bearer" = ["network.get", "network.*", "*"])), + security(("Bearer" = ["network.get", "*"])), responses( (status = OK, description = "Known nodes are returned", body = Network, content_type = "application/json", example = json!( { From cc37eb870f7cb3835a49bae2785fadaff4689166 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 12:01:40 +0200 Subject: [PATCH 213/312] Initial implementation --- crates/index-scheduler/src/features.rs | 1 + crates/index-scheduler/src/insta_snapshot.rs | 20 +- crates/index-scheduler/src/lib.rs | 190 +++++++++------ crates/meilisearch-types/src/error.rs | 6 +- crates/meilisearch-types/src/lib.rs | 1 + crates/meilisearch-types/src/webhooks.rs | 18 ++ crates/meilisearch/src/routes/mod.rs | 5 +- crates/meilisearch/src/routes/network.rs | 2 +- crates/meilisearch/src/routes/webhooks.rs | 239 +++++++++++++++++++ 9 files changed, 409 insertions(+), 73 deletions(-) create mode 100644 crates/meilisearch-types/src/webhooks.rs create mode 100644 crates/meilisearch/src/routes/webhooks.rs diff --git a/crates/index-scheduler/src/features.rs b/crates/index-scheduler/src/features.rs index b52a659a6..dacac1a9c 100644 --- a/crates/index-scheduler/src/features.rs +++ b/crates/index-scheduler/src/features.rs @@ -182,6 +182,7 @@ impl FeatureData { ..persisted_features })); + // Once this is stabilized, network should be stored along with webhooks in index-scheduler's persisted database let network_db = runtime_features_db.remap_data_type::>(); let network: Network = network_db.get(wtxn, db_keys::NETWORK)?.unwrap_or_default(); diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index 32ce131b5..21626fb2e 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -26,11 +26,11 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { version, queue, scheduler, + persisted, index_mapper, features: _, - webhook_url: _, - webhook_authorization_header: _, + cached_webhooks: _, test_breakpoint_sdr: _, planned_failures: _, run_loop_iteration: _, @@ -62,6 +62,10 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { } snap.push_str("\n----------------------------------------------------------------------\n"); + snap.push_str("### Persisted:\n"); + snap.push_str(&snapshot_persisted_db(&rtxn, persisted)); + snap.push_str("----------------------------------------------------------------------\n"); + snap.push_str("### All Tasks:\n"); snap.push_str(&snapshot_all_tasks(&rtxn, queue.tasks.all_tasks)); snap.push_str("----------------------------------------------------------------------\n"); @@ -200,6 +204,16 @@ pub fn snapshot_date_db(rtxn: &RoTxn, db: Database) -> String { + let mut snap = String::new(); + let iter = db.iter(rtxn).unwrap(); + for next in iter { + let (key, value) = next.unwrap(); + snap.push_str(&format!("{key}: {value}\n")); + } + snap +} + pub fn snapshot_task(task: &Task) -> String { let mut snap = String::new(); let Task { @@ -311,6 +325,7 @@ pub fn snapshot_status( } snap } + pub fn snapshot_kind(rtxn: &RoTxn, db: Database, RoaringBitmapCodec>) -> String { let mut snap = String::new(); let iter = db.iter(rtxn).unwrap(); @@ -331,6 +346,7 @@ pub fn snapshot_index_tasks(rtxn: &RoTxn, db: Database) } snap } + pub fn snapshot_canceled_by(rtxn: &RoTxn, db: Database) -> String { let mut snap = String::new(); let iter = db.iter(rtxn).unwrap(); diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 46566b9ba..9e1d8d1a8 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -65,6 +65,7 @@ use meilisearch_types::milli::vector::{ use meilisearch_types::milli::{self, Index}; use meilisearch_types::task_view::TaskView; use meilisearch_types::tasks::{KindWithContent, Task}; +use meilisearch_types::webhooks::{Webhook, Webhooks}; use milli::vector::db::IndexEmbeddingConfig; use processing::ProcessingTasks; pub use queue::Query; @@ -80,7 +81,15 @@ use crate::utils::clamp_to_page_size; pub(crate) type BEI128 = I128; const TASK_SCHEDULER_SIZE_THRESHOLD_PERCENT_INT: u64 = 40; -const CHAT_SETTINGS_DB_NAME: &str = "chat-settings"; + +mod db_name { + pub const CHAT_SETTINGS: &str = "chat-settings"; + pub const PERSISTED: &str = "persisted"; +} + +mod db_keys { + pub const WEBHOOKS: &str = "webhooks"; +} #[derive(Debug)] pub struct IndexSchedulerOptions { @@ -171,10 +180,11 @@ pub struct IndexScheduler { /// Whether we should use the old document indexer or the new one. pub(crate) experimental_no_edition_2024_for_dumps: bool, - /// The webhook url we should send tasks to after processing every batches. - pub(crate) webhook_url: Option, - /// The Authorization header to send to the webhook URL. - pub(crate) webhook_authorization_header: Option, + /// A database to store single-keyed data that is persisted across restarts. + persisted: Database, + + /// Webhook + cached_webhooks: Arc>, /// A map to retrieve the runtime representation of an embedder depending on its configuration. /// @@ -214,8 +224,8 @@ impl IndexScheduler { index_mapper: self.index_mapper.clone(), cleanup_enabled: self.cleanup_enabled, experimental_no_edition_2024_for_dumps: self.experimental_no_edition_2024_for_dumps, - webhook_url: self.webhook_url.clone(), - webhook_authorization_header: self.webhook_authorization_header.clone(), + persisted: self.persisted, + cached_webhooks: self.cached_webhooks.clone(), embedders: self.embedders.clone(), #[cfg(test)] test_breakpoint_sdr: self.test_breakpoint_sdr.clone(), @@ -284,10 +294,16 @@ impl IndexScheduler { let version = versioning::Versioning::new(&env, from_db_version)?; let mut wtxn = env.write_txn()?; + let features = features::FeatureData::new(&env, &mut wtxn, options.instance_features)?; let queue = Queue::new(&env, &mut wtxn, &options)?; let index_mapper = IndexMapper::new(&env, &mut wtxn, &options, budget)?; - let chat_settings = env.create_database(&mut wtxn, Some(CHAT_SETTINGS_DB_NAME))?; + let chat_settings = env.create_database(&mut wtxn, Some(db_name::CHAT_SETTINGS))?; + + let persisted = env.create_database(&mut wtxn, Some(db_name::PERSISTED))?; + let webhooks_db = persisted.remap_data_type::>(); + let webhooks = webhooks_db.get(&wtxn, db_keys::WEBHOOKS)?.unwrap_or_default(); + wtxn.commit()?; // allow unreachable_code to get rids of the warning in the case of a test build. @@ -303,8 +319,9 @@ impl IndexScheduler { experimental_no_edition_2024_for_dumps: options .indexer_config .experimental_no_edition_2024_for_dumps, - webhook_url: options.webhook_url, - webhook_authorization_header: options.webhook_authorization_header, + persisted, + cached_webhooks: Arc::new(RwLock::new(webhooks)), + embedders: Default::default(), #[cfg(test)] @@ -754,80 +771,103 @@ impl IndexScheduler { /// Once the tasks changes have been committed we must send all the tasks that were updated to our webhook if there is one. fn notify_webhook(&self, updated: &RoaringBitmap) -> Result<()> { - if let Some(ref url) = self.webhook_url { - struct TaskReader<'a, 'b> { - rtxn: &'a RoTxn<'a>, - index_scheduler: &'a IndexScheduler, - tasks: &'b mut roaring::bitmap::Iter<'b>, - buffer: Vec, - written: usize, - } + let webhooks = self.cached_webhooks.read().unwrap_or_else(|poisoned| poisoned.into_inner()); + if webhooks.webhooks.is_empty() { + return Ok(()); + } + let webhooks = Webhooks::clone(&*webhooks); - impl Read for TaskReader<'_, '_> { - fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result { - if self.buffer.is_empty() { - match self.tasks.next() { - None => return Ok(0), - Some(task_id) => { - let task = self - .index_scheduler - .queue - .tasks - .get_task(self.rtxn, task_id) - .map_err(|err| io::Error::new(io::ErrorKind::Other, err))? - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::Other, - Error::CorruptedTaskQueue, - ) - })?; + struct TaskReader<'a, 'b> { + rtxn: &'a RoTxn<'a>, + index_scheduler: &'a IndexScheduler, + tasks: &'b mut roaring::bitmap::Iter<'b>, + buffer: Vec, + written: usize, + } - serde_json::to_writer( - &mut self.buffer, - &TaskView::from_task(&task), - )?; - self.buffer.push(b'\n'); - } + impl Read for TaskReader<'_, '_> { + fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result { + if self.buffer.is_empty() { + match self.tasks.next() { + None => return Ok(0), + Some(task_id) => { + let task = self + .index_scheduler + .queue + .tasks + .get_task(self.rtxn, task_id) + .map_err(|err| io::Error::new(io::ErrorKind::Other, err))? + .ok_or_else(|| { + io::Error::new(io::ErrorKind::Other, Error::CorruptedTaskQueue) + })?; + + serde_json::to_writer(&mut self.buffer, &TaskView::from_task(&task))?; + self.buffer.push(b'\n'); } } + } - let mut to_write = &self.buffer[self.written..]; - let wrote = io::copy(&mut to_write, &mut buf)?; - self.written += wrote as usize; + let mut to_write = &self.buffer[self.written..]; + let wrote = io::copy(&mut to_write, &mut buf)?; + self.written += wrote as usize; - // we wrote everything and must refresh our buffer on the next call - if self.written == self.buffer.len() { - self.written = 0; - self.buffer.clear(); - } + // we wrote everything and must refresh our buffer on the next call + if self.written == self.buffer.len() { + self.written = 0; + self.buffer.clear(); + } - Ok(wrote as usize) + Ok(wrote as usize) + } + } + + let rtxn = self.env.read_txn()?; + + let task_reader = TaskReader { + rtxn: &rtxn, + index_scheduler: self, + tasks: &mut updated.into_iter(), + buffer: Vec::with_capacity(800), // on average a task is around ~600 bytes + written: 0, + }; + + enum EitherRead { + Other(T), + Data(Vec), + } + + impl Read for &mut EitherRead { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + EitherRead::Other(reader) => reader.read(buf), + EitherRead::Data(data) => data.as_slice().read(buf), } } + } - let rtxn = self.env.read_txn()?; + let mut reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); - let task_reader = TaskReader { - rtxn: &rtxn, - index_scheduler: self, - tasks: &mut updated.into_iter(), - buffer: Vec::with_capacity(50), // on average a task is around ~100 bytes - written: 0, - }; + // When there is more than one webhook, cache the data in memory + let mut reader = match webhooks.webhooks.len() { + 1 => EitherRead::Other(reader), + _ => { + let mut data = Vec::new(); + reader.read_to_end(&mut data)?; + EitherRead::Data(data) + } + }; - // let reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); - let reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); - let request = ureq::post(url) + for (name, Webhook { url, headers }) in webhooks.webhooks.iter() { + let mut request = ureq::post(url) .timeout(Duration::from_secs(30)) .set("Content-Encoding", "gzip") .set("Content-Type", "application/x-ndjson"); - let request = match &self.webhook_authorization_header { - Some(header) => request.set("Authorization", header), - None => request, - }; + for (header_name, header_value) in headers.iter() { + request = request.set(header_name, header_value); + } - if let Err(e) = request.send(reader) { - tracing::error!("While sending data to the webhook: {e}"); + if let Err(e) = request.send(&mut reader) { + tracing::error!("While sending data to the webhook {name}: {e}"); } } @@ -862,6 +902,20 @@ impl IndexScheduler { self.features.network() } + pub fn put_webhooks(&self, webhooks: Webhooks) -> Result<()> { + let mut wtxn = self.env.write_txn()?; + let webhooks_db = self.persisted.remap_data_type::>(); + webhooks_db.put(&mut wtxn, db_keys::WEBHOOKS, &webhooks)?; + wtxn.commit()?; + *self.cached_webhooks.write().unwrap() = webhooks; + Ok(()) + } + + pub fn webhooks(&self) -> Webhooks { + let webhooks = self.cached_webhooks.read().unwrap_or_else(|poisoned| poisoned.into_inner()); + Webhooks::clone(&*webhooks) + } + pub fn embedders( &self, index_uid: String, diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 458034c00..92425d386 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -418,7 +418,11 @@ InvalidChatCompletionSearchDescriptionPrompt , InvalidRequest , BAD_REQU InvalidChatCompletionSearchQueryParamPrompt , InvalidRequest , BAD_REQUEST ; InvalidChatCompletionSearchFilterParamPrompt , InvalidRequest , BAD_REQUEST ; InvalidChatCompletionSearchIndexUidParamPrompt , InvalidRequest , BAD_REQUEST ; -InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQUEST +InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQUEST ; +// Webhooks +InvalidWebhooks , InvalidRequest , BAD_REQUEST ; +InvalidWebhooksUrl , InvalidRequest , BAD_REQUEST ; +InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST } impl ErrorCode for JoinError { diff --git a/crates/meilisearch-types/src/lib.rs b/crates/meilisearch-types/src/lib.rs index fe69da526..9857bfb29 100644 --- a/crates/meilisearch-types/src/lib.rs +++ b/crates/meilisearch-types/src/lib.rs @@ -15,6 +15,7 @@ pub mod star_or; pub mod task_view; pub mod tasks; pub mod versioning; +pub mod webhooks; pub use milli::{heed, Index}; use uuid::Uuid; pub use versioning::VERSION_FILE_NAME; diff --git a/crates/meilisearch-types/src/webhooks.rs b/crates/meilisearch-types/src/webhooks.rs new file mode 100644 index 000000000..c30d32bc6 --- /dev/null +++ b/crates/meilisearch-types/src/webhooks.rs @@ -0,0 +1,18 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Webhook { + pub url: String, + #[serde(default)] + pub headers: BTreeMap, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Webhooks { + #[serde(default)] + pub webhooks: BTreeMap, +} diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 260d973a1..4ae72b0bd 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -70,6 +70,7 @@ mod swap_indexes; pub mod tasks; #[cfg(test)] mod tasks_test; +mod webhooks; #[derive(OpenApi)] #[openapi( @@ -89,6 +90,7 @@ mod tasks_test; (path = "/experimental-features", api = features::ExperimentalFeaturesApi), (path = "/export", api = export::ExportApi), (path = "/network", api = network::NetworkApi), + (path = "/webhooks", api = webhooks::WebhooksApi), ), paths(get_health, get_version, get_stats), tags( @@ -120,7 +122,8 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::scope("/experimental-features").configure(features::configure)) .service(web::scope("/network").configure(network::configure)) .service(web::scope("/export").configure(export::configure)) - .service(web::scope("/chats").configure(chats::configure)); + .service(web::scope("/chats").configure(chats::configure)) + .service(web::scope("/webhooks").configure(webhooks::configure)); #[cfg(feature = "swagger")] { diff --git a/crates/meilisearch/src/routes/network.rs b/crates/meilisearch/src/routes/network.rs index 6ee68ea33..4afa32c09 100644 --- a/crates/meilisearch/src/routes/network.rs +++ b/crates/meilisearch/src/routes/network.rs @@ -168,7 +168,7 @@ impl Aggregate for PatchNetworkAnalytics { path = "", tag = "Network", request_body = Network, - security(("Bearer" = ["network.update", "network.*", "*"])), + security(("Bearer" = ["network.update", "*"])), responses( (status = OK, description = "New network state is returned", body = Network, content_type = "application/json", example = json!( { diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs new file mode 100644 index 000000000..d05c16672 --- /dev/null +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -0,0 +1,239 @@ +use std::collections::BTreeMap; + +use actix_web::web::{self, Data}; +use actix_web::{HttpRequest, HttpResponse}; +use deserr::actix_web::AwebJson; +use deserr::Deserr; +use index_scheduler::IndexScheduler; +use meilisearch_types::deserr::DeserrJsonError; +use meilisearch_types::error::deserr_codes::{ + InvalidWebhooks, InvalidWebhooksHeaders, InvalidWebhooksUrl, +}; +use meilisearch_types::error::{ErrorCode, ResponseError}; +use meilisearch_types::keys::actions; +use meilisearch_types::milli::update::Setting; +use meilisearch_types::webhooks::{Webhook, Webhooks}; +use serde::Serialize; +use tracing::debug; +use utoipa::{OpenApi, ToSchema}; + +use crate::analytics::{Aggregate, Analytics}; +use crate::extractors::authentication::policies::ActionPolicy; +use crate::extractors::authentication::GuardedData; +use crate::extractors::sequential_extractor::SeqHandler; + +#[derive(OpenApi)] +#[openapi( + paths(get_webhooks, patch_webhooks), + tags(( + name = "Webhooks", + description = "The `/webhooks` route allows you to register endpoints to be called once tasks are processed.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/webhooks"), + )), +)] +pub struct WebhooksApi; + +pub fn configure(cfg: &mut web::ServiceConfig) { + cfg.service( + web::resource("") + .route(web::get().to(get_webhooks)) + .route(web::patch().to(SeqHandler(patch_webhooks))), + ); +} + +#[utoipa::path( + get, + path = "", + tag = "Webhooks", + security(("Bearer" = ["webhooks.get", "*.get", "*"])), + responses( + (status = OK, description = "Webhooks are returned", body = WebhooksSettings, content_type = "application/json", example = json!({ + "webhooks": { + "name": { + "url": "http://example.com/webhook", + }, + "anotherName": { + "url": "https://your.site/on-tasks-completed", + "headers": { + "Authorization": "Bearer a-secret-token" + } + } + } + })), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] +async fn get_webhooks( + index_scheduler: GuardedData, Data>, +) -> Result { + let webhooks = index_scheduler.webhooks(); + debug!(returns = ?webhooks, "Get webhooks"); + Ok(HttpResponse::Ok().json(webhooks)) +} + +#[derive(Debug, Deserr, ToSchema)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] +struct WebhookSettings { + #[schema(value_type = Option)] + #[deserr(default, error = DeserrJsonError)] + #[serde(default)] + url: Setting, + #[schema(value_type = Option>, example = json!({"Authorization":"Bearer a-secret-token"}))] + #[deserr(default, error = DeserrJsonError)] + #[serde(default)] + headers: Setting>>, +} + +#[derive(Debug, Deserr, ToSchema)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] +struct WebhooksSettings { + #[schema(value_type = Option>)] + #[deserr(default, error = DeserrJsonError)] + #[serde(default)] + webhooks: Setting>>, +} + +#[derive(Serialize)] +pub struct PatchWebhooksAnalytics; + +impl Aggregate for PatchWebhooksAnalytics { + fn event_name(&self) -> &'static str { + "Webhooks Updated" + } + + fn aggregate(self: Box, _new: Box) -> Box { + self + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(*self).unwrap_or_default() + } +} + +#[derive(Debug, thiserror::Error)] +enum WebhooksError { + #[error("The URL for the webhook `{0}` is missing.")] + MissingUrl(String), +} + +impl ErrorCode for WebhooksError { + fn error_code(&self) -> meilisearch_types::error::Code { + match self { + WebhooksError::MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhooksUrl, + } + } +} + +#[utoipa::path( + patch, + path = "", + tag = "Webhooks", + request_body = WebhooksSettings, + security(("Bearer" = ["webhooks.update", "*"])), + responses( + (status = 200, description = "Returns the updated webhooks", body = WebhooksSettings, content_type = "application/json", example = json!({ + "webhooks": { + "name": { + "url": "http://example.com/webhook", + }, + "anotherName": { + "url": "https://your.site/on-tasks-completed", + "headers": { + "Authorization": "Bearer a-secret-token" + } + } + } + })), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!({ + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + })), + ) +)] +async fn patch_webhooks( + index_scheduler: GuardedData, Data>, + new_webhooks: AwebJson, + req: HttpRequest, + analytics: Data, +) -> Result { + let WebhooksSettings { webhooks: new_webhooks } = new_webhooks.0; + let Webhooks { mut webhooks } = index_scheduler.webhooks(); + debug!(parameters = ?new_webhooks, "Patch webhooks"); + + fn merge_webhook( + name: &str, + old_webhook: Option, + new_webhook: WebhookSettings, + ) -> Result { + let (old_url, mut headers) = + old_webhook.map(|w| (Some(w.url), w.headers)).unwrap_or((None, BTreeMap::new())); + + let url = match new_webhook.url { + Setting::Set(url) => url, + Setting::NotSet => old_url.ok_or_else(|| WebhooksError::MissingUrl(name.to_owned()))?, + Setting::Reset => return Err(WebhooksError::MissingUrl(name.to_owned())), + }; + + let headers = match new_webhook.headers { + Setting::Set(new_headers) => { + for (name, value) in new_headers { + match value { + Setting::Set(value) => { + headers.insert(name, value); + } + Setting::NotSet => continue, + Setting::Reset => { + headers.remove(&name); + continue; + } + } + } + headers + } + Setting::NotSet => headers, + Setting::Reset => BTreeMap::new(), + }; + + Ok(Webhook { url, headers }) + } + + match new_webhooks { + Setting::Set(new_webhooks) => { + for (name, new_webhook) in new_webhooks { + match new_webhook { + Setting::Set(new_webhook) => { + let old_webhook = webhooks.remove(&name); + let webhook = merge_webhook(&name, old_webhook, new_webhook)?; + webhooks.insert(name.clone(), webhook); + } + Setting::Reset => { + webhooks.remove(&name); + } + Setting::NotSet => (), + } + } + } + Setting::Reset => webhooks.clear(), + Setting::NotSet => (), + }; + + analytics.publish(PatchWebhooksAnalytics, &req); + + let webhooks = Webhooks { webhooks }; + index_scheduler.put_webhooks(webhooks.clone())?; + debug!(returns = ?webhooks, "Patch webhooks"); + Ok(HttpResponse::Ok().json(webhooks)) +} From 466e1a7aac704a4d93355aef76902cb08581160a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 12:25:59 +0200 Subject: [PATCH 214/312] Support legacy cli arguments --- crates/index-scheduler/src/lib.rs | 4 ---- crates/index-scheduler/src/test_utils.rs | 2 -- crates/meilisearch-types/src/webhooks.rs | 2 +- crates/meilisearch/src/lib.rs | 28 ++++++++++++++++++++++-- crates/meilisearch/src/option.rs | 2 ++ 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 9e1d8d1a8..dfe8138f3 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -107,10 +107,6 @@ pub struct IndexSchedulerOptions { pub snapshots_path: PathBuf, /// The path to the folder containing the dumps. pub dumps_path: PathBuf, - /// The URL on which we must send the tasks statuses - pub webhook_url: Option, - /// The value we will send into the Authorization HTTP header on the webhook URL - pub webhook_authorization_header: Option, /// The maximum size, in bytes, of the task index. pub task_db_size: usize, /// The size, in bytes, with which a meilisearch index is opened the first time of each meilisearch index. diff --git a/crates/index-scheduler/src/test_utils.rs b/crates/index-scheduler/src/test_utils.rs index bfed7f53a..b7d69b5b3 100644 --- a/crates/index-scheduler/src/test_utils.rs +++ b/crates/index-scheduler/src/test_utils.rs @@ -98,8 +98,6 @@ impl IndexScheduler { indexes_path: tempdir.path().join("indexes"), snapshots_path: tempdir.path().join("snapshots"), dumps_path: tempdir.path().join("dumps"), - webhook_url: None, - webhook_authorization_header: None, task_db_size: 1000 * 1000 * 10, // 10 MB, we don't use MiB on purpose. index_base_map_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. enable_mdb_writemap: false, diff --git a/crates/meilisearch-types/src/webhooks.rs b/crates/meilisearch-types/src/webhooks.rs index c30d32bc6..9d371bd5f 100644 --- a/crates/meilisearch-types/src/webhooks.rs +++ b/crates/meilisearch-types/src/webhooks.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] #[serde(rename_all = "camelCase")] pub struct Webhook { pub url: String, diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 0fb93b65a..24741d22d 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -13,6 +13,7 @@ pub mod routes; pub mod search; pub mod search_queue; +use std::collections::BTreeMap; use std::fs::File; use std::io::{BufReader, BufWriter}; use std::path::Path; @@ -48,6 +49,7 @@ use meilisearch_types::tasks::KindWithContent; use meilisearch_types::versioning::{ create_current_version_file, get_version, VersionFileError, VERSION_MINOR, VERSION_PATCH, }; +use meilisearch_types::webhooks::Webhook; use meilisearch_types::{compression, heed, milli, VERSION_FILE_NAME}; pub use option::Opt; use option::ScheduleSnapshot; @@ -223,8 +225,6 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< indexes_path: opt.db_path.join("indexes"), snapshots_path: opt.snapshot_dir.clone(), dumps_path: opt.dump_dir.clone(), - webhook_url: opt.task_webhook_url.as_ref().map(|url| url.to_string()), - webhook_authorization_header: opt.task_webhook_authorization_header.clone(), task_db_size: opt.max_task_db_size.as_u64() as usize, index_base_map_size: opt.max_index_size.as_u64() as usize, enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage, @@ -327,6 +327,30 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< .unwrap(); } + // We set the webhook url + let cli_webhook = opt.task_webhook_url.as_ref().map(|u| Webhook { + url: u.to_string(), + headers: { + let mut headers = BTreeMap::new(); + if let Some(value) = &opt.task_webhook_authorization_header { + headers.insert(String::from("Authorization"), value.to_string()); + } + headers + }, + }); + let mut webhooks = index_scheduler.webhooks(); + if webhooks.webhooks.get("_cli") != cli_webhook.as_ref() { + match cli_webhook { + Some(webhook) => { + webhooks.webhooks.insert("_cli".to_string(), webhook); + } + None => { + webhooks.webhooks.remove("_cli"); + } + } + index_scheduler.put_webhooks(webhooks)?; + } + Ok((index_scheduler, auth_controller)) } diff --git a/crates/meilisearch/src/option.rs b/crates/meilisearch/src/option.rs index dd77a1222..e27fa08cd 100644 --- a/crates/meilisearch/src/option.rs +++ b/crates/meilisearch/src/option.rs @@ -206,11 +206,13 @@ pub struct Opt { pub env: String, /// Called whenever a task finishes so a third party can be notified. + /// See also the dedicated API `/webhooks`. #[clap(long, env = MEILI_TASK_WEBHOOK_URL)] pub task_webhook_url: Option, /// The Authorization header to send on the webhook URL whenever /// a task finishes so a third party can be notified. + /// See also the dedicated API `/webhooks`. #[clap(long, env = MEILI_TASK_WEBHOOK_AUTHORIZATION_HEADER)] pub task_webhook_authorization_header: Option, From 93f8b31eecc91615505a927372213ff0693871e5 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 12:52:01 +0200 Subject: [PATCH 215/312] Fix tests --- crates/index-scheduler/src/insta_snapshot.rs | 9 ++++++--- crates/index-scheduler/src/lib.rs | 1 + crates/meilisearch/tests/auth/api_keys.rs | 2 +- crates/meilisearch/tests/auth/errors.rs | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index 21626fb2e..f3431dd33 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -62,9 +62,12 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { } snap.push_str("\n----------------------------------------------------------------------\n"); - snap.push_str("### Persisted:\n"); - snap.push_str(&snapshot_persisted_db(&rtxn, persisted)); - snap.push_str("----------------------------------------------------------------------\n"); + let persisted_db_snapshot = snapshot_persisted_db(&rtxn, persisted); + if !persisted_db_snapshot.is_empty() { + snap.push_str("### Persisted:\n"); + snap.push_str(&persisted_db_snapshot); + snap.push_str("----------------------------------------------------------------------\n"); + } snap.push_str("### All Tasks:\n"); snap.push_str(&snapshot_all_tasks(&rtxn, queue.tasks.all_tasks)); diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index dfe8138f3..ce8791a63 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -240,6 +240,7 @@ impl IndexScheduler { + IndexMapper::nb_db() + features::FeatureData::nb_db() + 1 // chat-prompts + + 1 // persisted } /// Create an index scheduler and start its run loop. diff --git a/crates/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs index 6dc3f429b..f16789add 100644 --- a/crates/meilisearch/tests/auth/api_keys.rs +++ b/crates/meilisearch/tests/auth/api_keys.rs @@ -421,7 +421,7 @@ async fn error_add_api_key_invalid_parameters_actions() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r#" { - "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`", + "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`, `webhooks.get`, `webhooks.update`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" diff --git a/crates/meilisearch/tests/auth/errors.rs b/crates/meilisearch/tests/auth/errors.rs index b16ccb2f5..6d3369144 100644 --- a/crates/meilisearch/tests/auth/errors.rs +++ b/crates/meilisearch/tests/auth/errors.rs @@ -93,7 +93,7 @@ async fn create_api_key_bad_actions() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r#" { - "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`", + "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`, `webhooks.get`, `webhooks.update`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" From 064d9d5ff81be5d342a3c44fc7fbea48b41acfd3 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:06:37 +0200 Subject: [PATCH 216/312] Add dump support --- crates/dump/src/reader/compat/v5_to_v6.rs | 4 +++ crates/dump/src/reader/mod.rs | 7 +++++ crates/dump/src/reader/v6/mod.rs | 25 +++++++++++---- crates/dump/src/writer.rs | 11 +++++++ crates/index-scheduler/src/processing.rs | 1 + .../src/scheduler/process_dump_creation.rs | 5 +++ crates/meilisearch-types/src/webhooks.rs | 2 +- crates/meilisearch/src/lib.rs | 31 +++++++++++-------- 8 files changed, 66 insertions(+), 20 deletions(-) diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index f173bb6bd..3a0c8ef0d 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -202,6 +202,10 @@ impl CompatV5ToV6 { pub fn network(&self) -> Result> { Ok(None) } + + pub fn webhooks(&self) -> Option<&v6::Webhooks> { + None + } } pub enum CompatIndexV5ToV6 { diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index c894c255f..328a01f60 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -138,6 +138,13 @@ impl DumpReader { DumpReader::Compat(compat) => compat.network(), } } + + pub fn webhooks(&self) -> Option<&v6::Webhooks> { + match self { + DumpReader::Current(current) => current.webhooks(), + DumpReader::Compat(compat) => compat.webhooks(), + } + } } impl From for DumpReader { diff --git a/crates/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs index 08d4700e5..d8ce430f9 100644 --- a/crates/dump/src/reader/v6/mod.rs +++ b/crates/dump/src/reader/v6/mod.rs @@ -25,6 +25,7 @@ pub type Key = meilisearch_types::keys::Key; pub type ChatCompletionSettings = meilisearch_types::features::ChatCompletionSettings; pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures; pub type Network = meilisearch_types::features::Network; +pub type Webhooks = meilisearch_types::webhooks::Webhooks; // ===== Other types to clarify the code of the compat module // everything related to the tasks @@ -59,6 +60,7 @@ pub struct V6Reader { keys: BufReader, features: Option, network: Option, + webhooks: Option, } impl V6Reader { @@ -93,8 +95,8 @@ impl V6Reader { Err(e) => return Err(e.into()), }; - let network_file = match fs::read(dump.path().join("network.json")) { - Ok(network_file) => Some(network_file), + let network = match fs::read(dump.path().join("network.json")) { + Ok(network_file) => Some(serde_json::from_reader(&*network_file)?), Err(error) => match error.kind() { // Allows the file to be missing, this will only result in all experimental features disabled. ErrorKind::NotFound => { @@ -104,10 +106,16 @@ impl V6Reader { _ => return Err(error.into()), }, }; - let network = if let Some(network_file) = network_file { - Some(serde_json::from_reader(&*network_file)?) - } else { - None + + let webhooks = match fs::read(dump.path().join("webhooks.json")) { + Ok(webhooks_file) => Some(serde_json::from_reader(&*webhooks_file)?), + Err(error) => match error.kind() { + ErrorKind::NotFound => { + debug!("`webhooks.json` not found in dump"); + None + } + _ => return Err(error.into()), + }, }; Ok(V6Reader { @@ -119,6 +127,7 @@ impl V6Reader { features, network, dump, + webhooks, }) } @@ -229,6 +238,10 @@ impl V6Reader { pub fn network(&self) -> Option<&Network> { self.network.as_ref() } + + pub fn webhooks(&self) -> Option<&Webhooks> { + self.webhooks.as_ref() + } } pub struct UpdateFile { diff --git a/crates/dump/src/writer.rs b/crates/dump/src/writer.rs index 9f828595a..84a76e483 100644 --- a/crates/dump/src/writer.rs +++ b/crates/dump/src/writer.rs @@ -8,6 +8,7 @@ use meilisearch_types::batches::Batch; use meilisearch_types::features::{ChatCompletionSettings, Network, RuntimeTogglableFeatures}; use meilisearch_types::keys::Key; use meilisearch_types::settings::{Checked, Settings}; +use meilisearch_types::webhooks::Webhooks; use serde_json::{Map, Value}; use tempfile::TempDir; use time::OffsetDateTime; @@ -74,6 +75,16 @@ impl DumpWriter { Ok(std::fs::write(self.dir.path().join("network.json"), serde_json::to_string(&network)?)?) } + pub fn create_webhooks(&self, webhooks: Webhooks) -> Result<()> { + if webhooks == Webhooks::default() { + return Ok(()); + } + Ok(std::fs::write( + self.dir.path().join("webhooks.json"), + serde_json::to_string(&webhooks)?, + )?) + } + pub fn persist_to(self, mut writer: impl Write) -> Result<()> { let gz_encoder = GzEncoder::new(&mut writer, Compression::default()); let mut tar_encoder = tar::Builder::new(gz_encoder); diff --git a/crates/index-scheduler/src/processing.rs b/crates/index-scheduler/src/processing.rs index fdd8e42ef..3da81f143 100644 --- a/crates/index-scheduler/src/processing.rs +++ b/crates/index-scheduler/src/processing.rs @@ -108,6 +108,7 @@ make_enum_progress! { DumpTheBatches, DumpTheIndexes, DumpTheExperimentalFeatures, + DumpTheWebhooks, CompressTheDump, } } diff --git a/crates/index-scheduler/src/scheduler/process_dump_creation.rs b/crates/index-scheduler/src/scheduler/process_dump_creation.rs index b14f23d0b..8f47cbd0c 100644 --- a/crates/index-scheduler/src/scheduler/process_dump_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_dump_creation.rs @@ -270,6 +270,11 @@ impl IndexScheduler { let network = self.network(); dump.create_network(network)?; + // 7. Dump the webhooks + progress.update_progress(DumpCreationProgress::DumpTheWebhooks); + let webhooks = self.webhooks(); + dump.create_webhooks(webhooks)?; + let dump_uid = started_at.format(format_description!( "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]" )).unwrap(); diff --git a/crates/meilisearch-types/src/webhooks.rs b/crates/meilisearch-types/src/webhooks.rs index 9d371bd5f..8849182ac 100644 --- a/crates/meilisearch-types/src/webhooks.rs +++ b/crates/meilisearch-types/src/webhooks.rs @@ -10,7 +10,7 @@ pub struct Webhook { pub headers: BTreeMap, } -#[derive(Debug, Serialize, Deserialize, Default, Clone)] +#[derive(Debug, Serialize, Deserialize, Default, Clone, PartialEq)] #[serde(rename_all = "camelCase")] pub struct Webhooks { #[serde(default)] diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 24741d22d..fcc71f04d 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -515,7 +515,12 @@ fn import_dump( let _ = std::fs::write(db_path.join("instance-uid"), instance_uid.to_string().as_bytes()); }; - // 2. Import the `Key`s. + // 2. Import the webhooks + if let Some(webhooks) = dump_reader.webhooks() { + index_scheduler.put_webhooks(webhooks.clone())?; + } + + // 3. Import the `Key`s. let mut keys = Vec::new(); auth.raw_delete_all_keys()?; for key in dump_reader.keys()? { @@ -524,20 +529,20 @@ fn import_dump( keys.push(key); } - // 3. Import the `ChatCompletionSettings`s. + // 4. Import the `ChatCompletionSettings`s. for result in dump_reader.chat_completions_settings()? { let (name, settings) = result?; index_scheduler.put_chat_settings(&name, &settings)?; } - // 4. Import the runtime features and network + // 5. Import the runtime features and network let features = dump_reader.features()?.unwrap_or_default(); index_scheduler.put_runtime_features(features)?; let network = dump_reader.network()?.cloned().unwrap_or_default(); index_scheduler.put_network(network)?; - // 4.1 Use all cpus to process dump if `max_indexing_threads` not configured + // 5.1 Use all cpus to process dump if `max_indexing_threads` not configured let backup_config; let base_config = index_scheduler.indexer_config(); @@ -554,7 +559,7 @@ fn import_dump( // /!\ The tasks must be imported AFTER importing the indexes or else the scheduler might // try to process tasks while we're trying to import the indexes. - // 5. Import the indexes. + // 6. Import the indexes. for index_reader in dump_reader.indexes()? { let mut index_reader = index_reader?; let metadata = index_reader.metadata(); @@ -567,12 +572,12 @@ fn import_dump( let mut wtxn = index.write_txn()?; let mut builder = milli::update::Settings::new(&mut wtxn, &index, indexer_config); - // 5.1 Import the primary key if there is one. + // 6.1 Import the primary key if there is one. if let Some(ref primary_key) = metadata.primary_key { builder.set_primary_key(primary_key.to_string()); } - // 5.2 Import the settings. + // 6.2 Import the settings. tracing::info!("Importing the settings."); let settings = index_reader.settings()?; apply_settings_to_builder(&settings, &mut builder); @@ -584,8 +589,8 @@ fn import_dump( let rtxn = index.read_txn()?; if index_scheduler.no_edition_2024_for_dumps() { - // 5.3 Import the documents. - // 5.3.1 We need to recreate the grenad+obkv format accepted by the index. + // 6.3 Import the documents. + // 6.3.1 We need to recreate the grenad+obkv format accepted by the index. tracing::info!("Importing the documents."); let file = tempfile::tempfile()?; let mut builder = DocumentsBatchBuilder::new(BufWriter::new(file)); @@ -596,7 +601,7 @@ fn import_dump( // This flush the content of the batch builder. let file = builder.into_inner()?.into_inner()?; - // 5.3.2 We feed it to the milli index. + // 6.3.2 We feed it to the milli index. let reader = BufReader::new(file); let reader = DocumentsBatchReader::from_reader(reader)?; @@ -675,15 +680,15 @@ fn import_dump( index_scheduler.refresh_index_stats(&uid)?; } - // 6. Import the queue + // 7. Import the queue let mut index_scheduler_dump = index_scheduler.register_dumped_task()?; - // 6.1. Import the batches + // 7.1. Import the batches for ret in dump_reader.batches()? { let batch = ret?; index_scheduler_dump.register_dumped_batch(batch)?; } - // 6.2. Import the tasks + // 7.2. Import the tasks for ret in dump_reader.tasks()? { let (task, file) = ret?; index_scheduler_dump.register_dumped_task(task, file)?; From dc7af47371274f2367d739b8f5b0c857034782bd Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:18:43 +0200 Subject: [PATCH 217/312] Add new errors --- crates/meilisearch/src/routes/webhooks.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index d05c16672..99306fa1e 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -125,12 +125,20 @@ impl Aggregate for PatchWebhooksAnalytics { enum WebhooksError { #[error("The URL for the webhook `{0}` is missing.")] MissingUrl(String), + #[error("Defining too many webhooks would crush the server. Please limit the number of webhooks to 20. You may use a third-party proxy server to dispatch events to more than 20 endpoints.")] + TooManyWebhooks, + #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] + TooManyHeaders(String), } impl ErrorCode for WebhooksError { fn error_code(&self) -> meilisearch_types::error::Code { match self { WebhooksError::MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhooksUrl, + WebhooksError::TooManyWebhooks => meilisearch_types::error::Code::InvalidWebhooks, + WebhooksError::TooManyHeaders(_) => { + meilisearch_types::error::Code::InvalidWebhooksHeaders + } } } } @@ -217,6 +225,9 @@ async fn patch_webhooks( Setting::Set(new_webhook) => { let old_webhook = webhooks.remove(&name); let webhook = merge_webhook(&name, old_webhook, new_webhook)?; + if webhook.headers.len() > 200 { + return Err(WebhooksError::TooManyHeaders(name).into()); + } webhooks.insert(name.clone(), webhook); } Setting::Reset => { @@ -230,6 +241,10 @@ async fn patch_webhooks( Setting::NotSet => (), }; + if webhooks.len() > 20 { + return Err(WebhooksError::TooManyWebhooks.into()); + } + analytics.publish(PatchWebhooksAnalytics, &req); let webhooks = Webhooks { webhooks }; From 3e77c1d8c84d0af236ee344b4bb64d7aff46d418 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:23:06 +0200 Subject: [PATCH 218/312] Add reserved webhook --- crates/meilisearch-types/src/error.rs | 3 ++- crates/meilisearch/src/routes/webhooks.rs | 14 +++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 92425d386..56590e79d 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -422,7 +422,8 @@ InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQU // Webhooks InvalidWebhooks , InvalidRequest , BAD_REQUEST ; InvalidWebhooksUrl , InvalidRequest , BAD_REQUEST ; -InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST +InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST ; +ReservedWebhook , InvalidRequest , BAD_REQUEST } impl ErrorCode for JoinError { diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 99306fa1e..631dd822d 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -129,6 +129,8 @@ enum WebhooksError { TooManyWebhooks, #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] TooManyHeaders(String), + #[error("Cannot edit webhook `{0}`. Webhooks prefixed with an underscore are special and may not be modified using the API.")] + ReservedWebhook(String), } impl ErrorCode for WebhooksError { @@ -139,6 +141,7 @@ impl ErrorCode for WebhooksError { WebhooksError::TooManyHeaders(_) => { meilisearch_types::error::Code::InvalidWebhooksHeaders } + WebhooksError::ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, } } } @@ -186,6 +189,10 @@ async fn patch_webhooks( old_webhook: Option, new_webhook: WebhookSettings, ) -> Result { + if name.starts_with('_') { + return Err(WebhooksError::ReservedWebhook(name.to_owned())); + } + let (old_url, mut headers) = old_webhook.map(|w| (Some(w.url), w.headers)).unwrap_or((None, BTreeMap::new())); @@ -215,6 +222,10 @@ async fn patch_webhooks( Setting::Reset => BTreeMap::new(), }; + if headers.len() > 200 { + return Err(WebhooksError::TooManyHeaders(name.to_owned())); + } + Ok(Webhook { url, headers }) } @@ -225,9 +236,6 @@ async fn patch_webhooks( Setting::Set(new_webhook) => { let old_webhook = webhooks.remove(&name); let webhook = merge_webhook(&name, old_webhook, new_webhook)?; - if webhook.headers.len() > 200 { - return Err(WebhooksError::TooManyHeaders(name).into()); - } webhooks.insert(name.clone(), webhook); } Setting::Reset => { From b565ec1497d8978ca31064fa075fbb01d9248b8b Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:44:42 +0200 Subject: [PATCH 219/312] Test cli behavior --- crates/meilisearch/tests/common/server.rs | 8 ++++++++ crates/meilisearch/tests/tasks/webhook.rs | 18 +++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index ad0678122..1dfe2e593 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -182,6 +182,10 @@ impl Server { self.service.patch("/network", value).await } + pub async fn set_webhooks(&self, value: Value) -> (Value, StatusCode) { + self.service.patch("/webhooks", value).await + } + pub async fn get_metrics(&self) -> (Value, StatusCode) { self.service.get("/metrics").await } @@ -447,6 +451,10 @@ impl Server { pub async fn get_network(&self) -> (Value, StatusCode) { self.service.get("/network").await } + + pub async fn get_webhooks(&self) -> (Value, StatusCode) { + self.service.get("/webhooks").await + } } pub fn default_settings(dir: impl AsRef) -> Opt { diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index b18002eb7..984cfc23e 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -68,12 +68,13 @@ async fn create_webhook_server() -> WebhookHandle { } #[actix_web::test] -async fn test_basic_webhook() { +async fn test_cli_webhook() { let WebhookHandle { server_handle, url, mut receiver } = create_webhook_server().await; let db_path = tempfile::tempdir().unwrap(); let server = Server::new_with_options(Opt { task_webhook_url: Some(Url::parse(&url).unwrap()), + task_webhook_authorization_header: Some(String::from("Bearer a-secret-token")), ..default_settings(db_path.path()) }) .await @@ -125,5 +126,20 @@ async fn test_basic_webhook() { assert!(nb_tasks == 5, "We should have received the 5 tasks but only received {nb_tasks}"); + let (webhooks, code) = server.get_webhooks().await; + snapshot!(code, @"200 OK"); + snapshot!(webhooks, @r#" + { + "webhooks": { + "_cli": { + "url": "http://127.0.0.1:51503/", + "headers": { + "Authorization": "Bearer a-secret-token" + } + } + } + } + "#); + server_handle.abort(); } From e88480c7c4d3f37d98fa80004ac0515048ead489 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:44:51 +0200 Subject: [PATCH 220/312] Fix reserved name check --- crates/meilisearch/src/routes/webhooks.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 631dd822d..9cf57c585 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -189,10 +189,6 @@ async fn patch_webhooks( old_webhook: Option, new_webhook: WebhookSettings, ) -> Result { - if name.starts_with('_') { - return Err(WebhooksError::ReservedWebhook(name.to_owned())); - } - let (old_url, mut headers) = old_webhook.map(|w| (Some(w.url), w.headers)).unwrap_or((None, BTreeMap::new())); @@ -232,6 +228,10 @@ async fn patch_webhooks( match new_webhooks { Setting::Set(new_webhooks) => { for (name, new_webhook) in new_webhooks { + if name.starts_with('_') { + return Err(WebhooksError::ReservedWebhook(name).into()); + } + match new_webhook { Setting::Set(new_webhook) => { let old_webhook = webhooks.remove(&name); From c70ae91d3442ab81fbbbf50f03756e135b8f20a4 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:52:24 +0200 Subject: [PATCH 221/312] Add test for reserved webhooks --- crates/meilisearch/src/routes/webhooks.rs | 2 +- crates/meilisearch/tests/tasks/webhook.rs | 35 +++++++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 9cf57c585..adca710a0 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -129,7 +129,7 @@ enum WebhooksError { TooManyWebhooks, #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] TooManyHeaders(String), - #[error("Cannot edit webhook `{0}`. Webhooks prefixed with an underscore are special and may not be modified using the API.")] + #[error("Cannot edit webhook `{0}`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.")] ReservedWebhook(String), } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 984cfc23e..5b77394f8 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -8,7 +8,7 @@ use actix_http::body::MessageBody; use actix_web::dev::{ServiceFactory, ServiceResponse}; use actix_web::web::{Bytes, Data}; use actix_web::{post, App, HttpRequest, HttpResponse, HttpServer}; -use meili_snap::snapshot; +use meili_snap::{json_string, snapshot}; use meilisearch::Opt; use tokio::sync::mpsc; use url::Url; @@ -128,11 +128,11 @@ async fn test_cli_webhook() { let (webhooks, code) = server.get_webhooks().await; snapshot!(code, @"200 OK"); - snapshot!(webhooks, @r#" + snapshot!(json_string!(webhooks, { ".webhooks._cli.url" => "[ignored]" }), @r#" { "webhooks": { "_cli": { - "url": "http://127.0.0.1:51503/", + "url": "[ignored]", "headers": { "Authorization": "Bearer a-secret-token" } @@ -143,3 +143,32 @@ async fn test_cli_webhook() { server_handle.abort(); } + +#[actix_web::test] +async fn reserved_names() { + let server = Server::new().await; + + let (value, code) = server + .set_webhooks(json!({ "webhooks": { "_cli": { "url": "http://localhost:8080" } } })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Cannot edit webhook `_cli`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", + "code": "reserved_webhook", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#reserved_webhook" + } + "#); + + let (value, code) = server.set_webhooks(json!({ "webhooks": { "_cli": null } })).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Cannot edit webhook `_cli`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", + "code": "reserved_webhook", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#reserved_webhook" + } + "#); +} From a75b327b376f1f568d624be1eec2fe9733edd004 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:59:19 +0200 Subject: [PATCH 222/312] Add test for webhooks over limits --- crates/meilisearch/tests/tasks/webhook.rs | 55 +++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 5b77394f8..c271446d8 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -172,3 +172,58 @@ async fn reserved_names() { } "#); } + +#[actix_web::test] +async fn over_limits() { + let server = Server::new().await; + + // Too many webhooks + for i in 0..20 { + let (_value, code) = server + .set_webhooks(json!({ "webhooks": { format!("webhook_{i}"): { "url": "http://localhost:8080" } } })) + .await; + snapshot!(code, @"200 OK"); + } + let (value, code) = server + .set_webhooks(json!({ "webhooks": { "webhook_21": { "url": "http://localhost:8080" } } })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Defining too many webhooks would crush the server. Please limit the number of webhooks to 20. You may use a third-party proxy server to dispatch events to more than 20 endpoints.", + "code": "invalid_webhooks", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks" + } + "#); + + // Reset webhooks + let (value, code) = server.set_webhooks(json!({ "webhooks": null })).await; + snapshot!(code, @"200 OK"); + snapshot!(value, @r#" + { + "webhooks": {} + } + "#); + + // Test too many headers + for i in 0..200 { + let header_name = format!("header_{i}"); + let (_value, code) = server + .set_webhooks(json!({ "webhooks": { "webhook": { "url": "http://localhost:8080", "headers": { header_name: "value" } } } })) + .await; + snapshot!(code, @"200 OK"); + } + let (value, code) = server + .set_webhooks(json!({ "webhooks": { "webhook": { "url": "http://localhost:8080", "headers": { "header_201": "value" } } } })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Too many headers for the webhook `webhook`. Please limit the number of headers to 200.", + "code": "invalid_webhooks_headers", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" + } + "#); +} From fc4c5d2718731442517059f3eceff8dcacce42c2 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 16:16:12 +0200 Subject: [PATCH 223/312] Add dump test --- crates/dump/src/reader/mod.rs | 39 ++++++++++++++++++ .../dump/tests/assets/v6-with-webhooks.dump | Bin 0 -> 1693 bytes 2 files changed, 39 insertions(+) create mode 100644 crates/dump/tests/assets/v6-with-webhooks.dump diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index 328a01f60..a34365905 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -372,6 +372,7 @@ pub(crate) mod test { assert_eq!(dump.features().unwrap().unwrap(), RuntimeTogglableFeatures::default()); assert_eq!(dump.network().unwrap(), None); + assert_eq!(dump.webhooks(), None); } #[test] @@ -442,6 +443,44 @@ pub(crate) mod test { insta::assert_snapshot!(network.remotes.get("ms-2").as_ref().unwrap().search_api_key.as_ref().unwrap(), @"foo"); } + #[test] + fn import_dump_v6_webhooks() { + let dump = File::open("tests/assets/v6-with-webhooks.dump").unwrap(); + let dump = DumpReader::open(dump).unwrap(); + + // top level infos + insta::assert_snapshot!(dump.date().unwrap(), @"2025-07-30 14:06:57.240882 +00:00:00"); + insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @r" + Some( + cb887dcc-34b3-48d1-addd-9815ae721a81, + ) + "); + + // webhooks + + let webhooks = dump.webhooks().unwrap(); + insta::assert_json_snapshot!(webhooks, @r#" + { + "webhooks": { + "exampleName": { + "url": "https://example.com/hook", + "headers": { + "authorization": "TOKEN" + } + }, + "otherName": { + "url": "https://example.com/authorization-less", + "headers": {} + }, + "third": { + "url": "https://third.com", + "headers": {} + } + } + } + "#); + } + #[test] fn import_dump_v5() { let dump = File::open("tests/assets/v5.dump").unwrap(); diff --git a/crates/dump/tests/assets/v6-with-webhooks.dump b/crates/dump/tests/assets/v6-with-webhooks.dump new file mode 100644 index 0000000000000000000000000000000000000000..89b1f61be5a0cb74449d0f372aa388fb7f5ac1f1 GIT binary patch literal 1693 zcmV;O24eXiiwFP!00000|Ls~|lbbdW_j5i4p)=`SGSkUS(n&9QXr_}0 zB+m3OxFF_o_1wqm7w9MJN?`lWcMeW!J4vJ8gE3+yV3*ZOzg_oabZ~ec@j^=B=y_q# z9X+3P#||SzJ>nA|6A!8O5QpA~j*ft~k*v0?(SODAMV7~J1bNBD2hofUiBGpb@&bg7 z{f~fz$#nV^+Mj})vH#)G+yCUxR{$4k^6Yuzc^%(4_yV2XrI6b3dh! zfVe&y?$IwCpa;d#PCdkjxv2A*DB+v2GO$=%>;5DP-6$IFeRcUn~*AKT@P1#9( zKda9}J)#@n2j$gRUR!U61@8^K&4gU*a&PPXZQp(m`d41V%CeGH zvi!t&*N>IgRs{yC^;oWb^Jrx7C6Jf!sg|jD2VuZ%;b*je48(s}9+J)Y@6!OA_F}929qIp%dQmk48&*fQ_w(2Wse!70vmT9-z?yvXDgXURYNKvnv zOChhS`f_!CA*9+UtVX(f6SZusnp}#fvqW$%>h%+;<@f1o^}TMNwS~-Q6=y}?bTcye zMD#}5v&Y}XVEo7arvIbB3yuFD0kBaQMP^URwkXDSEoPO}`}7;*OWy@x_Gg?)cE|}} z$p4Z5fd7Yq@&6;BBjnydxb66}Hvv!vJXtp&7YGthHGh7%+H9O9w>jD$J|q7_{J`Y@jsZo3A1Lj!!chJP(|h&@0?0S>{}BN5zi84r zpZEEb*9}nnQjyvXFwDUHW_2~wR66`Ar6@Usf|AWt7&sFdgDsf#YbW{Sm!GZY;u@~& zl#G2rc-?oNC%%@!EM~DPEf`)l>nE zqIqxEI%Tc;^MiF(X&HF-sv8DUN6ebHNs*@?QRfpr%gcM(WIPEnEU1$T>N|vD;jlP` z#9T%=4?Qo5aduamkN2iXiv1n+s3o_m5xy$88Hd~vD0HUPWwErb`ee=3v??HF1FO-5 zHDaqG!3vE^-|eVQ$Z2rrrAk{otk@_^upjuyNiY+j*fek==6en$ltf<4z?nA{>jISl zO?vKBfA^$k$n^MlH%0fNAJeo>TC$RVf+L@*E}QxxVH~3*2%Ruuu|q@_IShG$lai48 zC?uF==&pV?`I_;b9h>?X)7|vci<UvmG4@jd<*zALl;I|df^I;{>#C;deib_*w}2RPsq=2p$rE^L^qr&XQ*wJgLqfBF3B3!Mws zdEVh6?+oA+kfpzK12q!N{58Knif&e8Y;-Uj)-ZGp!|1U1occcuN3;JsJhc5!x00G(F71bTsQmusZYA(Q_3!=*krn5ppAMOrVPb}f875|!nDG_G4E4xEF`_Hj nzy$6$5F>iYIE)5kL|+9Gg%}&c-N3-W;H$^K8)Yw;07L))oiSHZ literal 0 HcmV?d00001 From 41262b008b8fb684626c0bb7fedb9d5d165e2833 Mon Sep 17 00:00:00 2001 From: nicolasvienot Date: Wed, 30 Jul 2025 17:55:02 +0200 Subject: [PATCH 224/312] feat(chat): update metrics name --- crates/meilisearch/src/metrics.rs | 20 +++++++++---------- .../src/routes/chats/chat_completions.rs | 12 +++++------ 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs index d52e04cc6..027fc9aa5 100644 --- a/crates/meilisearch/src/metrics.rs +++ b/crates/meilisearch/src/metrics.rs @@ -15,30 +15,30 @@ lazy_static! { "Meilisearch number of degraded search requests" )) .expect("Can't create a metric"); - pub static ref MEILISEARCH_CHAT_SEARCH_REQUESTS: IntCounterVec = register_int_counter_vec!( + pub static ref MEILISEARCH_CHAT_SEARCHES_TOTAL: IntCounterVec = register_int_counter_vec!( opts!( - "meilisearch_chat_search_requests", - "Meilisearch number of search requests performed by the chat route itself" + "meilisearch_chat_searches_total", + "Total number of searches performed by the chat route" ), &["type"] ) .expect("Can't create a metric"); - pub static ref MEILISEARCH_CHAT_PROMPT_TOKENS_USAGE: IntCounterVec = register_int_counter_vec!( - opts!("meilisearch_chat_prompt_tokens_usage", "Meilisearch Chat Prompt Tokens Usage"), + pub static ref MEILISEARCH_CHAT_PROMPT_TOKENS_TOTAL: IntCounterVec = register_int_counter_vec!( + opts!("meilisearch_chat_prompt_tokens_total", "Total number of prompt tokens consumed"), &["workspace", "model"] ) .expect("Can't create a metric"); - pub static ref MEILISEARCH_CHAT_COMPLETION_TOKENS_USAGE: IntCounterVec = + pub static ref MEILISEARCH_CHAT_COMPLETION_TOKENS_TOTAL: IntCounterVec = register_int_counter_vec!( opts!( - "meilisearch_chat_completion_tokens_usage", - "Meilisearch Chat Completion Tokens Usage" + "meilisearch_chat_completion_tokens_total", + "Total number of completion tokens consumed" ), &["workspace", "model"] ) .expect("Can't create a metric"); - pub static ref MEILISEARCH_CHAT_TOTAL_TOKENS_USAGE: IntCounterVec = register_int_counter_vec!( - opts!("meilisearch_chat_total_tokens_usage", "Meilisearch Chat Total Tokens Usage"), + pub static ref MEILISEARCH_CHAT_TOKENS_TOTAL: IntCounterVec = register_int_counter_vec!( + opts!("meilisearch_chat_tokens_total", "Total number of tokens consumed (prompt + completion)"), &["workspace", "model"] ) .expect("Can't create a metric"); diff --git a/crates/meilisearch/src/routes/chats/chat_completions.rs b/crates/meilisearch/src/routes/chats/chat_completions.rs index b636678f5..f2c17a696 100644 --- a/crates/meilisearch/src/routes/chats/chat_completions.rs +++ b/crates/meilisearch/src/routes/chats/chat_completions.rs @@ -50,8 +50,8 @@ use crate::error::MeilisearchHttpError; use crate::extractors::authentication::policies::ActionPolicy; use crate::extractors::authentication::{extract_token_from_request, GuardedData, Policy as _}; use crate::metrics::{ - MEILISEARCH_CHAT_COMPLETION_TOKENS_USAGE, MEILISEARCH_CHAT_PROMPT_TOKENS_USAGE, - MEILISEARCH_CHAT_SEARCH_REQUESTS, MEILISEARCH_CHAT_TOTAL_TOKENS_USAGE, + MEILISEARCH_CHAT_COMPLETION_TOKENS_TOTAL, MEILISEARCH_CHAT_PROMPT_TOKENS_TOTAL, + MEILISEARCH_CHAT_SEARCHES_TOTAL, MEILISEARCH_CHAT_TOKENS_TOTAL, MEILISEARCH_DEGRADED_SEARCH_REQUESTS, }; use crate::routes::chats::utils::SseEventSender; @@ -319,7 +319,7 @@ async fn process_search_request( }; let mut documents = Vec::new(); if let Ok((ref rtxn, ref search_result)) = output { - MEILISEARCH_CHAT_SEARCH_REQUESTS.with_label_values(&["internal"]).inc(); + MEILISEARCH_CHAT_SEARCHES_TOTAL.with_label_values(&["internal"]).inc(); if search_result.degraded { MEILISEARCH_DEGRADED_SEARCH_REQUESTS.inc(); } @@ -596,13 +596,13 @@ async fn run_conversation( match result { Ok(resp) => { if let Some(usage) = resp.usage.as_ref() { - MEILISEARCH_CHAT_PROMPT_TOKENS_USAGE + MEILISEARCH_CHAT_PROMPT_TOKENS_TOTAL .with_label_values(&[workspace_uid, &chat_completion.model]) .inc_by(usage.prompt_tokens as u64); - MEILISEARCH_CHAT_COMPLETION_TOKENS_USAGE + MEILISEARCH_CHAT_COMPLETION_TOKENS_TOTAL .with_label_values(&[workspace_uid, &chat_completion.model]) .inc_by(usage.completion_tokens as u64); - MEILISEARCH_CHAT_TOTAL_TOKENS_USAGE + MEILISEARCH_CHAT_TOKENS_TOTAL .with_label_values(&[workspace_uid, &chat_completion.model]) .inc_by(usage.total_tokens as u64); } From 941da56ee3a860db5b94149ab09440f548dd2f60 Mon Sep 17 00:00:00 2001 From: nicolasvienot Date: Thu, 31 Jul 2025 06:49:53 +0200 Subject: [PATCH 225/312] fix linter --- crates/meilisearch/src/metrics.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/src/metrics.rs b/crates/meilisearch/src/metrics.rs index 027fc9aa5..607bc91eb 100644 --- a/crates/meilisearch/src/metrics.rs +++ b/crates/meilisearch/src/metrics.rs @@ -38,7 +38,10 @@ lazy_static! { ) .expect("Can't create a metric"); pub static ref MEILISEARCH_CHAT_TOKENS_TOTAL: IntCounterVec = register_int_counter_vec!( - opts!("meilisearch_chat_tokens_total", "Total number of tokens consumed (prompt + completion)"), + opts!( + "meilisearch_chat_tokens_total", + "Total number of tokens consumed (prompt + completion)" + ), &["workspace", "model"] ) .expect("Can't create a metric"); From f67043801b81b7266635e367f3833b6c7d34509c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 09:35:16 +0200 Subject: [PATCH 226/312] Add a test for concurrent cli and dump --- crates/dump/src/reader/mod.rs | 8 ++- .../dump/tests/assets/v6-with-webhooks.dump | Bin 1693 -> 1913 bytes crates/meilisearch/tests/tasks/webhook.rs | 46 +++++++++++++++++- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index a34365905..85e5df432 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -449,7 +449,7 @@ pub(crate) mod test { let dump = DumpReader::open(dump).unwrap(); // top level infos - insta::assert_snapshot!(dump.date().unwrap(), @"2025-07-30 14:06:57.240882 +00:00:00"); + insta::assert_snapshot!(dump.date().unwrap(), @"2025-07-31 7:28:28.091553 +00:00:00"); insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @r" Some( cb887dcc-34b3-48d1-addd-9815ae721a81, @@ -462,6 +462,12 @@ pub(crate) mod test { insta::assert_json_snapshot!(webhooks, @r#" { "webhooks": { + "_cli": { + "url": "https://defined-in-dump.com/", + "headers": { + "Authorization": "Bearer defined in dump" + } + }, "exampleName": { "url": "https://example.com/hook", "headers": { diff --git a/crates/dump/tests/assets/v6-with-webhooks.dump b/crates/dump/tests/assets/v6-with-webhooks.dump index 89b1f61be5a0cb74449d0f372aa388fb7f5ac1f1..83989f25c4cc882dafeea5c60d917d594867f019 100644 GIT binary patch literal 1913 zcmV-<2Zs0`iwFP!00000|Lt4bj@vd9_Vxc1`gK`YS{`-qMS(Qi0!4weXt%v+QKYm; z$7*EBl_+esi+!xVK%cBbO19VDb>vNKH);4`v9hQkDa~kR$eB@iF*-RAVkqVujtIq3 zcSMAB#||UR5hD@ijN0~?Aw1%vGoWp>s6B7=zF0lT>iCf$FQs~;n$aoobopZx!F4D9 z(_<=hHva(S&*R9+{|xBmfAL4FS{44@RAn(bHb9&xVr-xM8H3+D@~4R7(OH=IiOc`i zZ*%ES(Abwsi@a$3$*nKj#qvkf>=Q&diYW>(L!8qn=KkH-2bO6|s4Fds7t<@zyoPh! zU1{y5rp+>?q=Hc}bkR3UwI2IiP^>rjB=fTAnR zCWucs4I|JbPJV{LW_g)6^S6y%)ezkQv-3JTv%*8~fnib*|@LECu z6_Z^*QQmkJ2&lGWh5p_DdP9T{FhmiL2~9t6L&QRko??h-$ll8k5t1BkQSQCeaG!i} zs2Kw4lq}xG2+?S9xDARzGEBZPKqMw1i{8TkAslk{Bm+bvuz7rMm&a@n3vsm122mEq z$ssm~aRkZhyV^$<0@Jp;f(ZE|tEn>euV@S4x}p`Ff=7 zNz9U|YI3cF!K|%K? zRw2dF@$e4$Uk0%zm;XHjO!WCgY2P&l>_4IWf&4LvsI&iP0N8)kWOcsm(-uEA5LwqI zn%qEG7Xq%cx?0#UHvA~HDkZprQY?%cxKuL%TQKj3j`G=eUwhxE8@O&WIlfF&G8Ksm zlt@ItaGC~bmhnK&NF)gtl#2w2Mb_`?e(}30{a^c3jIxcNnR5tJG^2_Knc^g1oT@;i8RWxeOv)Ic zX-a3?-2CiljO3IY$VW@LmyPgN!DJFrH^9(^WtS(iy6O`p*Ym1?bP23Rr=n4N83|Ho zOxEl`c0wXS2H%=o!Q+fgW>ZQcf`cg)3K*M3K_UnWIO8lqsQ_i(Wo!tc3}~{p)BHRV zpCQvr=wXcRN#Dk4n=2Hhe1alBH(hr5!{Ri>(*(n)&M~PN%*Y^%t+u|YWZ*! z+ib?z>G08q$!zh!=(Kp3`d^BP+y9*bR{j^ABYio`i&K<>zk~cq%w7HWInc@fl-1uu z{vY=G5A(l-N5uL6GhkEyZHuC8v1PXoUsXe{&Q}$c_X;YzYRZSjl(&9Y)cA{wF5HhU zj9ql$7}15Xi!Pigx-fRpg@d9CcIHhTr72oHpsddBT<#USU)nn1&X)GWJofp`-FEry zK=uFeKSa1+s_%S;^BK-(IG^Es#)s)MR$odD_>7&;G*0lz$l7TT=QNzta8AQH4d*mI zAg5t&dB2mmU*(rDMDSiGu~iR<5{mabi7is_Jk(`r9UL4S&WV2jsh^k*08Rh^55LRg literal 1693 zcmV;O24eXiiwFP!00000|Ls~|lbbdW_j5i4p)=`SGSkUS(n&9QXr_}0 zB+m3OxFF_o_1wqm7w9MJN?`lWcMeW!J4vJ8gE3+yV3*ZOzg_oabZ~ec@j^=B=y_q# z9X+3P#||SzJ>nA|6A!8O5QpA~j*ft~k*v0?(SODAMV7~J1bNBD2hofUiBGpb@&bg7 z{f~fz$#nV^+Mj})vH#)G+yCUxR{$4k^6Yuzc^%(4_yV2XrI6b3dh! zfVe&y?$IwCpa;d#PCdkjxv2A*DB+v2GO$=%>;5DP-6$IFeRcUn~*AKT@P1#9( zKda9}J)#@n2j$gRUR!U61@8^K&4gU*a&PPXZQp(m`d41V%CeGH zvi!t&*N>IgRs{yC^;oWb^Jrx7C6Jf!sg|jD2VuZ%;b*je48(s}9+J)Y@6!OA_F}929qIp%dQmk48&*fQ_w(2Wse!70vmT9-z?yvXDgXURYNKvnv zOChhS`f_!CA*9+UtVX(f6SZusnp}#fvqW$%>h%+;<@f1o^}TMNwS~-Q6=y}?bTcye zMD#}5v&Y}XVEo7arvIbB3yuFD0kBaQMP^URwkXDSEoPO}`}7;*OWy@x_Gg?)cE|}} z$p4Z5fd7Yq@&6;BBjnydxb66}Hvv!vJXtp&7YGthHGh7%+H9O9w>jD$J|q7_{J`Y@jsZo3A1Lj!!chJP(|h&@0?0S>{}BN5zi84r zpZEEb*9}nnQjyvXFwDUHW_2~wR66`Ar6@Usf|AWt7&sFdgDsf#YbW{Sm!GZY;u@~& zl#G2rc-?oNC%%@!EM~DPEf`)l>nE zqIqxEI%Tc;^MiF(X&HF-sv8DUN6ebHNs*@?QRfpr%gcM(WIPEnEU1$T>N|vD;jlP` z#9T%=4?Qo5aduamkN2iXiv1n+s3o_m5xy$88Hd~vD0HUPWwErb`ee=3v??HF1FO-5 zHDaqG!3vE^-|eVQ$Z2rrrAk{otk@_^upjuyNiY+j*fek==6en$ltf<4z?nA{>jISl zO?vKBfA^$k$n^MlH%0fNAJeo>TC$RVf+L@*E}QxxVH~3*2%Ruuu|q@_IShG$lai48 zC?uF==&pV?`I_;b9h>?X)7|vci<UvmG4@jd<*zALl;I|df^I;{>#C;deib_*w}2RPsq=2p$rE^L^qr&XQ*wJgLqfBF3B3!Mws zdEVh6?+oA+kfpzK12q!N{58Knif&e8Y;-Uj)-ZGp!|1U1occcuN3;JsJhc5!x00G(F71bTsQmusZYA(Q_3!=*krn5ppAMOrVPb}f875|!nDG_G4E4xEF`_Hj nzy$6$5F>iYIE)5kL|+9Gg%}&c-N3-W;H$^K8)Yw;07L))oiSHZ diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index c271446d8..12a8228fa 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -2,6 +2,7 @@ //! post requests. The webhook handle starts a server and forwards all the //! received requests into a channel for you to handle. +use std::path::PathBuf; use std::sync::Arc; use actix_http::body::MessageBody; @@ -68,7 +69,7 @@ async fn create_webhook_server() -> WebhookHandle { } #[actix_web::test] -async fn test_cli_webhook() { +async fn cli_only() { let WebhookHandle { server_handle, url, mut receiver } = create_webhook_server().await; let db_path = tempfile::tempdir().unwrap(); @@ -144,6 +145,49 @@ async fn test_cli_webhook() { server_handle.abort(); } + +#[actix_web::test] +async fn cli_with_dumps() { + let db_path = tempfile::tempdir().unwrap(); + let server = Server::new_with_options(Opt { + task_webhook_url: Some(Url::parse("http://defined-in-test-cli.com").unwrap()), + task_webhook_authorization_header: Some(String::from("Bearer a-secret-token-defined-in-test-cli")), + import_dump: Some(PathBuf::from("../dump/tests/assets/v6-with-webhooks.dump")), + ..default_settings(db_path.path()) + }) + .await + .unwrap(); + + let (webhooks, code) = server.get_webhooks().await; + snapshot!(code, @"200 OK"); + snapshot!(webhooks, @r#" + { + "webhooks": { + "_cli": { + "url": "http://defined-in-test-cli.com/", + "headers": { + "Authorization": "Bearer a-secret-token-defined-in-test-cli" + } + }, + "exampleName": { + "url": "https://example.com/hook", + "headers": { + "authorization": "TOKEN" + } + }, + "otherName": { + "url": "https://example.com/authorization-less", + "headers": {} + }, + "third": { + "url": "https://third.com", + "headers": {} + } + } + } + "#); +} + #[actix_web::test] async fn reserved_names() { let server = Server::new().await; From 446fce6c1679df3a94112804fe0b79285d1f53bd Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 10:01:25 +0200 Subject: [PATCH 227/312] Extract logic from route --- crates/meilisearch/src/routes/webhooks.rs | 25 ++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index adca710a0..710873d55 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -180,10 +180,17 @@ async fn patch_webhooks( req: HttpRequest, analytics: Data, ) -> Result { - let WebhooksSettings { webhooks: new_webhooks } = new_webhooks.0; - let Webhooks { mut webhooks } = index_scheduler.webhooks(); - debug!(parameters = ?new_webhooks, "Patch webhooks"); + let webhooks = patch_webhooks_inner(&index_scheduler, new_webhooks.0)?; + analytics.publish(PatchWebhooksAnalytics, &req); + + Ok(HttpResponse::Ok().json(webhooks)) +} + +fn patch_webhooks_inner( + index_scheduler: &GuardedData, Data>, + new_webhooks: WebhooksSettings, +) -> Result { fn merge_webhook( name: &str, old_webhook: Option, @@ -225,7 +232,11 @@ async fn patch_webhooks( Ok(Webhook { url, headers }) } - match new_webhooks { + debug!(parameters = ?new_webhooks, "Patch webhooks"); + + let Webhooks { mut webhooks } = index_scheduler.webhooks(); + + match new_webhooks.webhooks { Setting::Set(new_webhooks) => { for (name, new_webhook) in new_webhooks { if name.starts_with('_') { @@ -253,10 +264,10 @@ async fn patch_webhooks( return Err(WebhooksError::TooManyWebhooks.into()); } - analytics.publish(PatchWebhooksAnalytics, &req); - let webhooks = Webhooks { webhooks }; index_scheduler.put_webhooks(webhooks.clone())?; + debug!(returns = ?webhooks, "Patch webhooks"); - Ok(HttpResponse::Ok().json(webhooks)) + + Ok(webhooks) } From 7c2c17129f303aeaebea792069cf9d06b4311192 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 10:59:06 +0200 Subject: [PATCH 228/312] Add get webhook route --- crates/meilisearch/src/routes/webhooks.rs | 65 ++++++++++++++++++++--- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 710873d55..6157b8efa 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use actix_web::web::{self, Data}; +use actix_web::web::{self, Data, Path}; use actix_web::{HttpRequest, HttpResponse}; use deserr::actix_web::AwebJson; use deserr::Deserr; @@ -24,7 +24,7 @@ use crate::extractors::sequential_extractor::SeqHandler; #[derive(OpenApi)] #[openapi( - paths(get_webhooks, patch_webhooks), + paths(get_webhooks, patch_webhooks, get_webhook), tags(( name = "Webhooks", description = "The `/webhooks` route allows you to register endpoints to be called once tasks are processed.", @@ -37,7 +37,11 @@ pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") .route(web::get().to(get_webhooks)) - .route(web::patch().to(SeqHandler(patch_webhooks))), + .route(web::patch().to(SeqHandler(patch_webhooks))) + ) + .service( + web::resource("/{name}") + .route(web::get().to(get_webhook)) ); } @@ -104,16 +108,29 @@ struct WebhooksSettings { webhooks: Setting>>, } -#[derive(Serialize)] -pub struct PatchWebhooksAnalytics; +#[derive(Serialize, Default)] +pub struct PatchWebhooksAnalytics { + patch_webhooks_count: usize, +} + +impl PatchWebhooksAnalytics { + pub fn patch_webhooks() -> Self { + PatchWebhooksAnalytics { + patch_webhooks_count: 1, + ..Self::default() + } + } +} impl Aggregate for PatchWebhooksAnalytics { fn event_name(&self) -> &'static str { "Webhooks Updated" } - fn aggregate(self: Box, _new: Box) -> Box { - self + fn aggregate(self: Box, new: Box) -> Box { + Box::new(PatchWebhooksAnalytics { + patch_webhooks_count: self.patch_webhooks_count + new.patch_webhooks_count, + }) } fn into_event(self: Box) -> serde_json::Value { @@ -131,6 +148,8 @@ enum WebhooksError { TooManyHeaders(String), #[error("Cannot edit webhook `{0}`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.")] ReservedWebhook(String), + #[error("Webhook `{0}` not found.")] + WebhookNotFound(String), } impl ErrorCode for WebhooksError { @@ -142,6 +161,7 @@ impl ErrorCode for WebhooksError { meilisearch_types::error::Code::InvalidWebhooksHeaders } WebhooksError::ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, + WebhooksError::WebhookNotFound(_) => meilisearch_types::error::Code::InvalidWebhooks, } } } @@ -182,7 +202,7 @@ async fn patch_webhooks( ) -> Result { let webhooks = patch_webhooks_inner(&index_scheduler, new_webhooks.0)?; - analytics.publish(PatchWebhooksAnalytics, &req); + analytics.publish(PatchWebhooksAnalytics::patch_webhooks(), &req); Ok(HttpResponse::Ok().json(webhooks)) } @@ -271,3 +291,32 @@ fn patch_webhooks_inner( Ok(webhooks) } + +#[utoipa::path( + get, + path = "/{name}", + tag = "Webhooks", + security(("Bearer" = ["webhooks.get", "*.get", "*"])), + responses( + (status = 200, description = "Webhook found", body = WebhookSettings, content_type = "application/json"), + (status = 404, description = "Webhook not found", body = ResponseError, content_type = "application/json"), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json"), + ), + params( + ("name" = String, Path, description = "The name of the webhook") + ) +)] +async fn get_webhook( + index_scheduler: GuardedData, Data>, + name: Path, +) -> Result { + let webhook_name = name.into_inner(); + let webhooks = index_scheduler.webhooks(); + + if let Some(webhook) = webhooks.webhooks.get(&webhook_name) { + debug!(returns = ?webhook, "Get webhook {}", webhook_name); + Ok(HttpResponse::Ok().json(webhook)) + } else { + Err(WebhooksError::WebhookNotFound(webhook_name).into()) + } +} From 53397e28fce1a844fb8021ddfe30cc06f07afe54 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 11:19:46 +0200 Subject: [PATCH 229/312] Replace name by uuid --- crates/meilisearch-types/src/webhooks.rs | 3 +- crates/meilisearch/src/lib.rs | 7 +- crates/meilisearch/src/routes/webhooks.rs | 183 +++++++++++++--------- crates/meilisearch/tests/tasks/webhook.rs | 5 +- 4 files changed, 117 insertions(+), 81 deletions(-) diff --git a/crates/meilisearch-types/src/webhooks.rs b/crates/meilisearch-types/src/webhooks.rs index 8849182ac..0f0741d69 100644 --- a/crates/meilisearch-types/src/webhooks.rs +++ b/crates/meilisearch-types/src/webhooks.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; +use uuid::Uuid; #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] #[serde(rename_all = "camelCase")] @@ -14,5 +15,5 @@ pub struct Webhook { #[serde(rename_all = "camelCase")] pub struct Webhooks { #[serde(default)] - pub webhooks: BTreeMap, + pub webhooks: BTreeMap, } diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index fcc71f04d..613268936 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -56,6 +56,7 @@ use option::ScheduleSnapshot; use search_queue::SearchQueue; use tracing::{error, info_span}; use tracing_subscriber::filter::Targets; +use uuid::Uuid; use crate::error::MeilisearchHttpError; @@ -339,13 +340,13 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< }, }); let mut webhooks = index_scheduler.webhooks(); - if webhooks.webhooks.get("_cli") != cli_webhook.as_ref() { + if webhooks.webhooks.get(&Uuid::nil()) != cli_webhook.as_ref() { match cli_webhook { Some(webhook) => { - webhooks.webhooks.insert("_cli".to_string(), webhook); + webhooks.webhooks.insert(Uuid::nil(), webhook); } None => { - webhooks.webhooks.remove("_cli"); + webhooks.webhooks.remove(&Uuid::nil()); } } index_scheduler.put_webhooks(webhooks)?; diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 6157b8efa..a78c36b0c 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -16,6 +16,7 @@ use meilisearch_types::webhooks::{Webhook, Webhooks}; use serde::Serialize; use tracing::debug; use utoipa::{OpenApi, ToSchema}; +use uuid::Uuid; use crate::analytics::{Aggregate, Analytics}; use crate::extractors::authentication::policies::ActionPolicy; @@ -37,49 +38,9 @@ pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") .route(web::get().to(get_webhooks)) - .route(web::patch().to(SeqHandler(patch_webhooks))) + .route(web::patch().to(SeqHandler(patch_webhooks))), ) - .service( - web::resource("/{name}") - .route(web::get().to(get_webhook)) - ); -} - -#[utoipa::path( - get, - path = "", - tag = "Webhooks", - security(("Bearer" = ["webhooks.get", "*.get", "*"])), - responses( - (status = OK, description = "Webhooks are returned", body = WebhooksSettings, content_type = "application/json", example = json!({ - "webhooks": { - "name": { - "url": "http://example.com/webhook", - }, - "anotherName": { - "url": "https://your.site/on-tasks-completed", - "headers": { - "Authorization": "Bearer a-secret-token" - } - } - } - })), - (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( - { - "message": "The Authorization header is missing. It must use the bearer authorization method.", - "code": "missing_authorization_header", - "type": "auth", - "link": "https://docs.meilisearch.com/errors#missing_authorization_header" - } - )), - ) -)] -async fn get_webhooks( - index_scheduler: GuardedData, Data>, -) -> Result { - let webhooks = index_scheduler.webhooks(); - debug!(returns = ?webhooks, "Get webhooks"); - Ok(HttpResponse::Ok().json(webhooks)) + .service(web::resource("/{uuid}").route(web::get().to(get_webhook))); } #[derive(Debug, Deserr, ToSchema)] @@ -105,7 +66,73 @@ struct WebhooksSettings { #[schema(value_type = Option>)] #[deserr(default, error = DeserrJsonError)] #[serde(default)] - webhooks: Setting>>, + webhooks: Setting>>, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct WebhookWithMetadata { + uuid: Uuid, + is_editable: bool, + #[serde(flatten)] + webhook: Webhook, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct WebhookResults { + results: Vec, +} + +#[utoipa::path( + get, + path = "", + tag = "Webhooks", + security(("Bearer" = ["webhooks.get", "*.get", "*"])), + responses( + (status = OK, description = "Webhooks are returned", body = WebhooksSettings, content_type = "application/json", example = json!({ + "results": [ + { + "uuid": "550e8400-e29b-41d4-a716-446655440000", + "url": "https://your.site/on-tasks-completed", + "headers": { + "Authorization": "Bearer a-secret-token" + }, + "isEditable": true + }, + { + "uuid": "550e8400-e29b-41d4-a716-446655440001", + "url": "https://another.site/on-tasks-completed", + "isEditable": true + } + ] + })), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] +async fn get_webhooks( + index_scheduler: GuardedData, Data>, +) -> Result { + let webhooks = index_scheduler.webhooks(); + let results = webhooks + .webhooks + .into_iter() + .map(|(uuid, webhook)| WebhookWithMetadata { + uuid, + is_editable: uuid != Uuid::nil(), + webhook, + }) + .collect::>(); + let results = WebhookResults { results }; + debug!(returns = ?results, "Get webhooks"); + Ok(HttpResponse::Ok().json(results)) } #[derive(Serialize, Default)] @@ -115,10 +142,7 @@ pub struct PatchWebhooksAnalytics { impl PatchWebhooksAnalytics { pub fn patch_webhooks() -> Self { - PatchWebhooksAnalytics { - patch_webhooks_count: 1, - ..Self::default() - } + PatchWebhooksAnalytics { patch_webhooks_count: 1 } } } @@ -141,15 +165,15 @@ impl Aggregate for PatchWebhooksAnalytics { #[derive(Debug, thiserror::Error)] enum WebhooksError { #[error("The URL for the webhook `{0}` is missing.")] - MissingUrl(String), + MissingUrl(Uuid), #[error("Defining too many webhooks would crush the server. Please limit the number of webhooks to 20. You may use a third-party proxy server to dispatch events to more than 20 endpoints.")] TooManyWebhooks, #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] - TooManyHeaders(String), + TooManyHeaders(Uuid), #[error("Cannot edit webhook `{0}`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.")] - ReservedWebhook(String), + ReservedWebhook(Uuid), #[error("Webhook `{0}` not found.")] - WebhookNotFound(String), + WebhookNotFound(Uuid), } impl ErrorCode for WebhooksError { @@ -175,10 +199,10 @@ impl ErrorCode for WebhooksError { responses( (status = 200, description = "Returns the updated webhooks", body = WebhooksSettings, content_type = "application/json", example = json!({ "webhooks": { - "name": { + "550e8400-e29b-41d4-a716-446655440000": { "url": "http://example.com/webhook", }, - "anotherName": { + "550e8400-e29b-41d4-a716-446655440001": { "url": "https://your.site/on-tasks-completed", "headers": { "Authorization": "Bearer a-secret-token" @@ -212,7 +236,7 @@ fn patch_webhooks_inner( new_webhooks: WebhooksSettings, ) -> Result { fn merge_webhook( - name: &str, + uuid: &Uuid, old_webhook: Option, new_webhook: WebhookSettings, ) -> Result { @@ -221,8 +245,8 @@ fn patch_webhooks_inner( let url = match new_webhook.url { Setting::Set(url) => url, - Setting::NotSet => old_url.ok_or_else(|| WebhooksError::MissingUrl(name.to_owned()))?, - Setting::Reset => return Err(WebhooksError::MissingUrl(name.to_owned())), + Setting::NotSet => old_url.ok_or_else(|| WebhooksError::MissingUrl(uuid.to_owned()))?, + Setting::Reset => return Err(WebhooksError::MissingUrl(uuid.to_owned())), }; let headers = match new_webhook.headers { @@ -246,7 +270,7 @@ fn patch_webhooks_inner( }; if headers.len() > 200 { - return Err(WebhooksError::TooManyHeaders(name.to_owned())); + return Err(WebhooksError::TooManyHeaders(uuid.to_owned())); } Ok(Webhook { url, headers }) @@ -258,19 +282,19 @@ fn patch_webhooks_inner( match new_webhooks.webhooks { Setting::Set(new_webhooks) => { - for (name, new_webhook) in new_webhooks { - if name.starts_with('_') { - return Err(WebhooksError::ReservedWebhook(name).into()); + for (uuid, new_webhook) in new_webhooks { + if uuid.is_nil() { + return Err(WebhooksError::ReservedWebhook(uuid).into()); } match new_webhook { Setting::Set(new_webhook) => { - let old_webhook = webhooks.remove(&name); - let webhook = merge_webhook(&name, old_webhook, new_webhook)?; - webhooks.insert(name.clone(), webhook); + let old_webhook = webhooks.remove(&uuid); + let webhook = merge_webhook(&uuid, old_webhook, new_webhook)?; + webhooks.insert(uuid, webhook); } Setting::Reset => { - webhooks.remove(&name); + webhooks.remove(&uuid); } Setting::NotSet => (), } @@ -298,25 +322,34 @@ fn patch_webhooks_inner( tag = "Webhooks", security(("Bearer" = ["webhooks.get", "*.get", "*"])), responses( - (status = 200, description = "Webhook found", body = WebhookSettings, content_type = "application/json"), + (status = 200, description = "Webhook found", body = WebhookSettings, content_type = "application/json", example = json!({ + "uuid": "550e8400-e29b-41d4-a716-446655440000", + "url": "https://your.site/on-tasks-completed", + "headers": { + "Authorization": "Bearer a-secret" + }, + "isEditable": true + })), (status = 404, description = "Webhook not found", body = ResponseError, content_type = "application/json"), (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json"), ), params( - ("name" = String, Path, description = "The name of the webhook") + ("uuid" = Uuid, Path, description = "The universally unique identifier of the webhook") ) )] async fn get_webhook( index_scheduler: GuardedData, Data>, - name: Path, + uuid: Path, ) -> Result { - let webhook_name = name.into_inner(); - let webhooks = index_scheduler.webhooks(); + let uuid = uuid.into_inner(); + let mut webhooks = index_scheduler.webhooks(); - if let Some(webhook) = webhooks.webhooks.get(&webhook_name) { - debug!(returns = ?webhook, "Get webhook {}", webhook_name); - Ok(HttpResponse::Ok().json(webhook)) - } else { - Err(WebhooksError::WebhookNotFound(webhook_name).into()) - } + let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?; + + debug!(returns = ?webhook, "Get webhook {}", uuid); + Ok(HttpResponse::Ok().json(WebhookWithMetadata { + uuid, + is_editable: uuid != Uuid::nil(), + webhook, + })) } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 12a8228fa..7fa088eb5 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -145,13 +145,14 @@ async fn cli_only() { server_handle.abort(); } - #[actix_web::test] async fn cli_with_dumps() { let db_path = tempfile::tempdir().unwrap(); let server = Server::new_with_options(Opt { task_webhook_url: Some(Url::parse("http://defined-in-test-cli.com").unwrap()), - task_webhook_authorization_header: Some(String::from("Bearer a-secret-token-defined-in-test-cli")), + task_webhook_authorization_header: Some(String::from( + "Bearer a-secret-token-defined-in-test-cli", + )), import_dump: Some(PathBuf::from("../dump/tests/assets/v6-with-webhooks.dump")), ..default_settings(db_path.path()) }) From ca27bcaac72bfcc08252e9e0fe2207ac442a2ba4 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 11:34:47 +0200 Subject: [PATCH 230/312] Update tests --- crates/dump/src/reader/mod.rs | 16 ++--- .../dump/tests/assets/v6-with-webhooks.dump | Bin 1913 -> 1389 bytes crates/meilisearch/tests/tasks/webhook.rs | 58 +++++++++++------- 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index 85e5df432..844aadc99 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -449,7 +449,7 @@ pub(crate) mod test { let dump = DumpReader::open(dump).unwrap(); // top level infos - insta::assert_snapshot!(dump.date().unwrap(), @"2025-07-31 7:28:28.091553 +00:00:00"); + insta::assert_snapshot!(dump.date().unwrap(), @"2025-07-31 9:21:30.479544 +00:00:00"); insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @r" Some( cb887dcc-34b3-48d1-addd-9815ae721a81, @@ -462,23 +462,23 @@ pub(crate) mod test { insta::assert_json_snapshot!(webhooks, @r#" { "webhooks": { - "_cli": { + "00000000-0000-0000-0000-000000000000": { "url": "https://defined-in-dump.com/", "headers": { "Authorization": "Bearer defined in dump" } }, - "exampleName": { + "627ea538-733d-4545-8d2d-03526eb381ce": { + "url": "https://example.com/authorization-less", + "headers": {} + }, + "771b0a28-ef28-4082-b984-536f82958c65": { "url": "https://example.com/hook", "headers": { "authorization": "TOKEN" } }, - "otherName": { - "url": "https://example.com/authorization-less", - "headers": {} - }, - "third": { + "f3583083-f8a7-4cbf-a5e7-fb3f1e28a7e9": { "url": "https://third.com", "headers": {} } diff --git a/crates/dump/tests/assets/v6-with-webhooks.dump b/crates/dump/tests/assets/v6-with-webhooks.dump index 83989f25c4cc882dafeea5c60d917d594867f019..c8f9649d88429ab8c16f3375f94a6c4474d3e79f 100644 GIT binary patch literal 1389 zcmV-z1(Nz7iwFP!00000|Ls~)Z`(Ey_H%!QrpLl`NQx5WyroGC3MQPdwcIj3zwl?SBAd%IV4f z(EhQEd;1>%OZy-F)j|uSKbxv72D=8DlO{qu#QqWW$H4*c=;J@Wf2rX^o10OKUxVG_ zUi;&{HT&~eM!o$H0G*|&G+L8LWDyamVMG~Y$T(%O0?C<5*`UV{3-~;PI-f%6R6#5# z*VfQ~Ff6K%&FiXlaMc1PcW0Gu z{VZvoUA$;Z?eeO;y00soJ(x&d94!^R&HAs;{|CW5T+IL^eZrMR z$XC8r@N#X9lxsL~Zq^(h9T~8B35Mh)@kQbY%ZVfO&IyLyj>esbC#{=Qb^flxf%;0(@3J-gI57FFPXz zG|i2{_)er`8C5(bm{L{?9=zDU$}R?xLf~x vqbZI3pnW!TcS@#0NH2+{SKdtS5>2@n%bq;x(W6I?9}@onHxdkx05|{u+j_X9 literal 1913 zcmV-<2Zs0`iwFP!00000|Lt4bj@vd9_Vxc1`gK`YS{`-qMS(Qi0!4weXt%v+QKYm; z$7*EBl_+esi+!xVK%cBbO19VDb>vNKH);4`v9hQkDa~kR$eB@iF*-RAVkqVujtIq3 zcSMAB#||UR5hD@ijN0~?Aw1%vGoWp>s6B7=zF0lT>iCf$FQs~;n$aoobopZx!F4D9 z(_<=hHva(S&*R9+{|xBmfAL4FS{44@RAn(bHb9&xVr-xM8H3+D@~4R7(OH=IiOc`i zZ*%ES(Abwsi@a$3$*nKj#qvkf>=Q&diYW>(L!8qn=KkH-2bO6|s4Fds7t<@zyoPh! zU1{y5rp+>?q=Hc}bkR3UwI2IiP^>rjB=fTAnR zCWucs4I|JbPJV{LW_g)6^S6y%)ezkQv-3JTv%*8~fnib*|@LECu z6_Z^*QQmkJ2&lGWh5p_DdP9T{FhmiL2~9t6L&QRko??h-$ll8k5t1BkQSQCeaG!i} zs2Kw4lq}xG2+?S9xDARzGEBZPKqMw1i{8TkAslk{Bm+bvuz7rMm&a@n3vsm122mEq z$ssm~aRkZhyV^$<0@Jp;f(ZE|tEn>euV@S4x}p`Ff=7 zNz9U|YI3cF!K|%K? zRw2dF@$e4$Uk0%zm;XHjO!WCgY2P&l>_4IWf&4LvsI&iP0N8)kWOcsm(-uEA5LwqI zn%qEG7Xq%cx?0#UHvA~HDkZprQY?%cxKuL%TQKj3j`G=eUwhxE8@O&WIlfF&G8Ksm zlt@ItaGC~bmhnK&NF)gtl#2w2Mb_`?e(}30{a^c3jIxcNnR5tJG^2_Knc^g1oT@;i8RWxeOv)Ic zX-a3?-2CiljO3IY$VW@LmyPgN!DJFrH^9(^WtS(iy6O`p*Ym1?bP23Rr=n4N83|Ho zOxEl`c0wXS2H%=o!Q+fgW>ZQcf`cg)3K*M3K_UnWIO8lqsQ_i(Wo!tc3}~{p)BHRV zpCQvr=wXcRN#Dk4n=2Hhe1alBH(hr5!{Ri>(*(n)&M~PN%*Y^%t+u|YWZ*! z+ib?z>G08q$!zh!=(Kp3`d^BP+y9*bR{j^ABYio`i&K<>zk~cq%w7HWInc@fl-1uu z{vY=G5A(l-N5uL6GhkEyZHuC8v1PXoUsXe{&Q}$c_X;YzYRZSjl(&9Y)cA{wF5HhU zj9ql$7}15Xi!Pigx-fRpg@d9CcIHhTr72oHpsddBT<#USU)nn1&X)GWJofp`-FEry zK=uFeKSa1+s_%S;^BK-(IG^Es#)s)MR$odD_>7&;G*0lz$l7TT=QNzta8AQH4d*mI zAg5t&dB2mmU*(rDMDSiGu~iR<5{mabi7is_Jk(`r9UL4S&WV2jsh^k*08Rh^55LRg diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 7fa088eb5..0990561e9 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -13,6 +13,7 @@ use meili_snap::{json_string, snapshot}; use meilisearch::Opt; use tokio::sync::mpsc; use url::Url; +use uuid::Uuid; use crate::common::{self, default_settings, Server}; use crate::json; @@ -129,16 +130,18 @@ async fn cli_only() { let (webhooks, code) = server.get_webhooks().await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(webhooks, { ".webhooks._cli.url" => "[ignored]" }), @r#" + snapshot!(json_string!(webhooks, { ".results[].url" => "[ignored]" }), @r#" { - "webhooks": { - "_cli": { + "results": [ + { + "uuid": "00000000-0000-0000-0000-000000000000", + "isEditable": false, "url": "[ignored]", "headers": { "Authorization": "Bearer a-secret-token" } } - } + ] } "#); @@ -163,28 +166,36 @@ async fn cli_with_dumps() { snapshot!(code, @"200 OK"); snapshot!(webhooks, @r#" { - "webhooks": { - "_cli": { + "results": [ + { + "uuid": "00000000-0000-0000-0000-000000000000", + "isEditable": false, "url": "http://defined-in-test-cli.com/", "headers": { "Authorization": "Bearer a-secret-token-defined-in-test-cli" } }, - "exampleName": { + { + "uuid": "627ea538-733d-4545-8d2d-03526eb381ce", + "isEditable": true, + "url": "https://example.com/authorization-less", + "headers": {} + }, + { + "uuid": "771b0a28-ef28-4082-b984-536f82958c65", + "isEditable": true, "url": "https://example.com/hook", "headers": { "authorization": "TOKEN" } }, - "otherName": { - "url": "https://example.com/authorization-less", - "headers": {} - }, - "third": { + { + "uuid": "f3583083-f8a7-4cbf-a5e7-fb3f1e28a7e9", + "isEditable": true, "url": "https://third.com", "headers": {} } - } + ] } "#); } @@ -194,23 +205,23 @@ async fn reserved_names() { let server = Server::new().await; let (value, code) = server - .set_webhooks(json!({ "webhooks": { "_cli": { "url": "http://localhost:8080" } } })) + .set_webhooks(json!({ "webhooks": { Uuid::nil(): { "url": "http://localhost:8080" } } })) .await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Cannot edit webhook `_cli`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", + "message": "Cannot edit webhook `[uuid]`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", "code": "reserved_webhook", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#reserved_webhook" } "#); - let (value, code) = server.set_webhooks(json!({ "webhooks": { "_cli": null } })).await; + let (value, code) = server.set_webhooks(json!({ "webhooks": { Uuid::nil(): null } })).await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Cannot edit webhook `_cli`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", + "message": "Cannot edit webhook `[uuid]`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", "code": "reserved_webhook", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#reserved_webhook" @@ -223,14 +234,14 @@ async fn over_limits() { let server = Server::new().await; // Too many webhooks - for i in 0..20 { + for _ in 0..20 { let (_value, code) = server - .set_webhooks(json!({ "webhooks": { format!("webhook_{i}"): { "url": "http://localhost:8080" } } })) + .set_webhooks(json!({ "webhooks": { Uuid::new_v4(): { "url": "http://localhost:8080" } } })) .await; snapshot!(code, @"200 OK"); } let (value, code) = server - .set_webhooks(json!({ "webhooks": { "webhook_21": { "url": "http://localhost:8080" } } })) + .set_webhooks(json!({ "webhooks": { Uuid::new_v4(): { "url": "http://localhost:8080" } } })) .await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" @@ -252,20 +263,21 @@ async fn over_limits() { "#); // Test too many headers + let uuid = Uuid::new_v4(); for i in 0..200 { let header_name = format!("header_{i}"); let (_value, code) = server - .set_webhooks(json!({ "webhooks": { "webhook": { "url": "http://localhost:8080", "headers": { header_name: "value" } } } })) + .set_webhooks(json!({ "webhooks": { uuid: { "url": "http://localhost:8080", "headers": { header_name: "value" } } } })) .await; snapshot!(code, @"200 OK"); } let (value, code) = server - .set_webhooks(json!({ "webhooks": { "webhook": { "url": "http://localhost:8080", "headers": { "header_201": "value" } } } })) + .set_webhooks(json!({ "webhooks": { uuid: { "url": "http://localhost:8080", "headers": { "header_201": "value" } } } })) .await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Too many headers for the webhook `webhook`. Please limit the number of headers to 200.", + "message": "Too many headers for the webhook `[uuid]`. Please limit the number of headers to 200.", "code": "invalid_webhooks_headers", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" From 29fb4d5e2a155be419c3b439d024ad2e87d4d0db Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 12:27:12 +0200 Subject: [PATCH 231/312] Add post webhook route --- crates/meilisearch-types/src/error.rs | 3 +- crates/meilisearch/src/routes/webhooks.rs | 69 +++++++++++++++++++---- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 56590e79d..3916012c1 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -423,7 +423,8 @@ InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQU InvalidWebhooks , InvalidRequest , BAD_REQUEST ; InvalidWebhooksUrl , InvalidRequest , BAD_REQUEST ; InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST ; -ReservedWebhook , InvalidRequest , BAD_REQUEST +ReservedWebhook , InvalidRequest , BAD_REQUEST ; +WebhookNotFound , InvalidRequest , NOT_FOUND } impl ErrorCode for JoinError { diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index a78c36b0c..7be6d0386 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -6,9 +6,7 @@ use deserr::actix_web::AwebJson; use deserr::Deserr; use index_scheduler::IndexScheduler; use meilisearch_types::deserr::DeserrJsonError; -use meilisearch_types::error::deserr_codes::{ - InvalidWebhooks, InvalidWebhooksHeaders, InvalidWebhooksUrl, -}; +use meilisearch_types::error::deserr_codes::{InvalidWebhooksHeaders, InvalidWebhooksUrl}; use meilisearch_types::error::{ErrorCode, ResponseError}; use meilisearch_types::keys::actions; use meilisearch_types::milli::update::Setting; @@ -25,7 +23,7 @@ use crate::extractors::sequential_extractor::SeqHandler; #[derive(OpenApi)] #[openapi( - paths(get_webhooks, patch_webhooks, get_webhook), + paths(get_webhooks, patch_webhooks, get_webhook, post_webhook), tags(( name = "Webhooks", description = "The `/webhooks` route allows you to register endpoints to be called once tasks are processed.", @@ -38,13 +36,14 @@ pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") .route(web::get().to(get_webhooks)) - .route(web::patch().to(SeqHandler(patch_webhooks))), + .route(web::patch().to(SeqHandler(patch_webhooks))) + .route(web::post().to(SeqHandler(post_webhook))), ) .service(web::resource("/{uuid}").route(web::get().to(get_webhook))); } #[derive(Debug, Deserr, ToSchema)] -#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] struct WebhookSettings { @@ -64,16 +63,17 @@ struct WebhookSettings { #[schema(rename_all = "camelCase")] struct WebhooksSettings { #[schema(value_type = Option>)] - #[deserr(default, error = DeserrJsonError)] #[serde(default)] webhooks: Setting>>, } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] struct WebhookWithMetadata { uuid: Uuid, is_editable: bool, + #[schema(value_type = WebhookSettings)] #[serde(flatten)] webhook: Webhook, } @@ -138,11 +138,16 @@ async fn get_webhooks( #[derive(Serialize, Default)] pub struct PatchWebhooksAnalytics { patch_webhooks_count: usize, + post_webhook_count: usize, } impl PatchWebhooksAnalytics { pub fn patch_webhooks() -> Self { - PatchWebhooksAnalytics { patch_webhooks_count: 1 } + PatchWebhooksAnalytics { patch_webhooks_count: 1, ..Default::default() } + } + + pub fn post_webhook() -> Self { + PatchWebhooksAnalytics { post_webhook_count: 1, ..Default::default() } } } @@ -154,6 +159,7 @@ impl Aggregate for PatchWebhooksAnalytics { fn aggregate(self: Box, new: Box) -> Box { Box::new(PatchWebhooksAnalytics { patch_webhooks_count: self.patch_webhooks_count + new.patch_webhooks_count, + post_webhook_count: self.post_webhook_count + new.post_webhook_count, }) } @@ -185,7 +191,7 @@ impl ErrorCode for WebhooksError { meilisearch_types::error::Code::InvalidWebhooksHeaders } WebhooksError::ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, - WebhooksError::WebhookNotFound(_) => meilisearch_types::error::Code::InvalidWebhooks, + WebhooksError::WebhookNotFound(_) => meilisearch_types::error::Code::WebhookNotFound, } } } @@ -318,7 +324,7 @@ fn patch_webhooks_inner( #[utoipa::path( get, - path = "/{name}", + path = "/{uuid}", tag = "Webhooks", security(("Bearer" = ["webhooks.get", "*.get", "*"])), responses( @@ -353,3 +359,44 @@ async fn get_webhook( webhook, })) } + +#[utoipa::path( + post, + path = "", + tag = "Webhooks", + request_body = WebhookSettings, + security(("Bearer" = ["webhooks.update", "*"])), + responses( + (status = 201, description = "Webhook created successfully", body = WebhookWithMetadata, content_type = "application/json", example = json!({ + "uuid": "550e8400-e29b-41d4-a716-446655440000", + "url": "https://your.site/on-tasks-completed", + "headers": { + "Authorization": "Bearer a-secret-token" + }, + "isEditable": true + })), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json"), + (status = 400, description = "Bad request", body = ResponseError, content_type = "application/json"), + ) +)] +async fn post_webhook( + index_scheduler: GuardedData, Data>, + webhook_settings: AwebJson, + req: HttpRequest, + analytics: Data, +) -> Result { + let uuid = Uuid::new_v4(); + + let webhooks = patch_webhooks_inner( + &index_scheduler, + WebhooksSettings { + webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Set(webhook_settings.0))])), + }, + )?; + let webhook = webhooks.webhooks.get(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?.clone(); + + analytics.publish(PatchWebhooksAnalytics::post_webhook(), &req); + + debug!(returns = ?webhook, "Created webhook {}", uuid); + Ok(HttpResponse::Created().json(WebhookWithMetadata { uuid, is_editable: true, webhook })) +} From ad68245186e80eb5e23da4853f5a2f7c4078ac93 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 12:33:34 +0200 Subject: [PATCH 232/312] Update tests --- crates/meilisearch/tests/common/server.rs | 9 +++++ crates/meilisearch/tests/tasks/webhook.rs | 41 ++++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 1dfe2e593..113dbc86f 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -186,6 +186,15 @@ impl Server { self.service.patch("/webhooks", value).await } + pub async fn create_webhook(&self, value: Value) -> (Value, StatusCode) { + self.service.post("/webhooks", value).await + } + + pub async fn get_webhook(&self, uuid: impl AsRef) -> (Value, StatusCode) { + let url = format!("/webhooks/{}", uuid.as_ref()); + self.service.get(url).await + } + pub async fn get_metrics(&self) -> (Value, StatusCode) { self.service.get("/metrics").await } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 0990561e9..8c2a59874 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -236,7 +236,9 @@ async fn over_limits() { // Too many webhooks for _ in 0..20 { let (_value, code) = server - .set_webhooks(json!({ "webhooks": { Uuid::new_v4(): { "url": "http://localhost:8080" } } })) + .set_webhooks( + json!({ "webhooks": { Uuid::new_v4(): { "url": "http://localhost:8080" } } }), + ) .await; snapshot!(code, @"200 OK"); } @@ -284,3 +286,40 @@ async fn over_limits() { } "#); } + +#[actix_web::test] +async fn post_and_get() { + let server = Server::new().await; + + let (value, code) = server + .create_webhook(json!({ + "url": "https://example.com/hook", + "headers": { "authorization": "TOKEN" } + })) + .await; + snapshot!(code, @"201 Created"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "https://example.com/hook", + "headers": { + "authorization": "TOKEN" + } + } + "#); + + let uuid = value.get("uuid").unwrap().as_str().unwrap(); + let (value, code) = server.get_webhook(uuid).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "https://example.com/hook", + "headers": { + "authorization": "TOKEN" + } + } + "#); +} From 94733a4a183e34c7fb461f889105f3d5ca764f4a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 12:38:14 +0200 Subject: [PATCH 233/312] Add delete endpoint --- crates/meilisearch/src/routes/webhooks.rs | 46 ++++++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 7be6d0386..7dd5b00d1 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -23,7 +23,7 @@ use crate::extractors::sequential_extractor::SeqHandler; #[derive(OpenApi)] #[openapi( - paths(get_webhooks, patch_webhooks, get_webhook, post_webhook), + paths(get_webhooks, patch_webhooks, get_webhook, post_webhook, delete_webhook), tags(( name = "Webhooks", description = "The `/webhooks` route allows you to register endpoints to be called once tasks are processed.", @@ -39,7 +39,11 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .route(web::patch().to(SeqHandler(patch_webhooks))) .route(web::post().to(SeqHandler(post_webhook))), ) - .service(web::resource("/{uuid}").route(web::get().to(get_webhook))); + .service( + web::resource("/{uuid}") + .route(web::get().to(get_webhook)) + .route(web::delete().to(SeqHandler(delete_webhook))), + ); } #[derive(Debug, Deserr, ToSchema)] @@ -400,3 +404,41 @@ async fn post_webhook( debug!(returns = ?webhook, "Created webhook {}", uuid); Ok(HttpResponse::Created().json(WebhookWithMetadata { uuid, is_editable: true, webhook })) } + +#[utoipa::path( + delete, + path = "/{uuid}", + tag = "Webhooks", + security(("Bearer" = ["webhooks.update", "*"])), + responses( + (status = 204, description = "Webhook deleted successfully"), + (status = 404, description = "Webhook not found", body = ResponseError, content_type = "application/json"), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json"), + ), + params( + ("uuid" = Uuid, Path, description = "The universally unique identifier of the webhook") + ) +)] +async fn delete_webhook( + index_scheduler: GuardedData, Data>, + uuid: Path, + req: HttpRequest, + analytics: Data, +) -> Result { + let uuid = uuid.into_inner(); + + let webhooks = index_scheduler.webhooks(); + if !webhooks.webhooks.contains_key(&uuid) { + return Err(WebhooksError::WebhookNotFound(uuid).into()); + } + + patch_webhooks_inner( + &index_scheduler, + WebhooksSettings { webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Reset)])) }, + )?; + + analytics.publish(PatchWebhooksAnalytics::patch_webhooks(), &req); + + debug!("Deleted webhook {}", uuid); + Ok(HttpResponse::NoContent().finish()) +} From 9e43f7b419b2f7955be045d01a9b4061f2164042 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 12:44:35 +0200 Subject: [PATCH 234/312] Update tests --- crates/meilisearch/src/routes/webhooks.rs | 11 +++++++---- crates/meilisearch/tests/common/server.rs | 5 +++++ crates/meilisearch/tests/tasks/webhook.rs | 8 +++++++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 7dd5b00d1..15b3145d0 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -143,6 +143,7 @@ async fn get_webhooks( pub struct PatchWebhooksAnalytics { patch_webhooks_count: usize, post_webhook_count: usize, + delete_webhook_count: usize, } impl PatchWebhooksAnalytics { @@ -153,6 +154,10 @@ impl PatchWebhooksAnalytics { pub fn post_webhook() -> Self { PatchWebhooksAnalytics { post_webhook_count: 1, ..Default::default() } } + + pub fn delete_webhook() -> Self { + PatchWebhooksAnalytics { delete_webhook_count: 1, ..Default::default() } + } } impl Aggregate for PatchWebhooksAnalytics { @@ -164,6 +169,7 @@ impl Aggregate for PatchWebhooksAnalytics { Box::new(PatchWebhooksAnalytics { patch_webhooks_count: self.patch_webhooks_count + new.patch_webhooks_count, post_webhook_count: self.post_webhook_count + new.post_webhook_count, + delete_webhook_count: self.delete_webhook_count + new.delete_webhook_count, }) } @@ -356,7 +362,6 @@ async fn get_webhook( let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?; - debug!(returns = ?webhook, "Get webhook {}", uuid); Ok(HttpResponse::Ok().json(WebhookWithMetadata { uuid, is_editable: uuid != Uuid::nil(), @@ -401,7 +406,6 @@ async fn post_webhook( analytics.publish(PatchWebhooksAnalytics::post_webhook(), &req); - debug!(returns = ?webhook, "Created webhook {}", uuid); Ok(HttpResponse::Created().json(WebhookWithMetadata { uuid, is_editable: true, webhook })) } @@ -437,8 +441,7 @@ async fn delete_webhook( WebhooksSettings { webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Reset)])) }, )?; - analytics.publish(PatchWebhooksAnalytics::patch_webhooks(), &req); + analytics.publish(PatchWebhooksAnalytics::delete_webhook(), &req); - debug!("Deleted webhook {}", uuid); Ok(HttpResponse::NoContent().finish()) } diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 113dbc86f..dd690c3db 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -195,6 +195,11 @@ impl Server { self.service.get(url).await } + pub async fn delete_webhook(&self, uuid: impl AsRef) -> (Value, StatusCode) { + let url = format!("/webhooks/{}", uuid.as_ref()); + self.service.delete(url).await + } + pub async fn get_metrics(&self) -> (Value, StatusCode) { self.service.get("/metrics").await } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 8c2a59874..9d66800af 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -288,7 +288,7 @@ async fn over_limits() { } #[actix_web::test] -async fn post_and_get() { +async fn post_get_delete() { let server = Server::new().await; let (value, code) = server @@ -322,4 +322,10 @@ async fn post_and_get() { } } "#); + + let (_value, code) = server.delete_webhook(uuid).await; + snapshot!(code, @"204 No Content"); + + let (_value, code) = server.get_webhook(uuid).await; + snapshot!(code, @"404 Not Found"); } From 34590297c1fa31ef8fe1c5fb5628a02d110078e3 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 12:53:57 +0200 Subject: [PATCH 235/312] Add patch webhook endpoint --- crates/meilisearch/src/routes/webhooks.rs | 53 ++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 15b3145d0..4b925c325 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -23,7 +23,7 @@ use crate::extractors::sequential_extractor::SeqHandler; #[derive(OpenApi)] #[openapi( - paths(get_webhooks, patch_webhooks, get_webhook, post_webhook, delete_webhook), + paths(get_webhooks, patch_webhooks, get_webhook, post_webhook, patch_webhook, delete_webhook), tags(( name = "Webhooks", description = "The `/webhooks` route allows you to register endpoints to be called once tasks are processed.", @@ -42,6 +42,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service( web::resource("/{uuid}") .route(web::get().to(get_webhook)) + .route(web::patch().to(SeqHandler(patch_webhook))) .route(web::delete().to(SeqHandler(delete_webhook))), ); } @@ -142,6 +143,7 @@ async fn get_webhooks( #[derive(Serialize, Default)] pub struct PatchWebhooksAnalytics { patch_webhooks_count: usize, + patch_webhook_count: usize, post_webhook_count: usize, delete_webhook_count: usize, } @@ -151,6 +153,10 @@ impl PatchWebhooksAnalytics { PatchWebhooksAnalytics { patch_webhooks_count: 1, ..Default::default() } } + pub fn patch_webhook() -> Self { + PatchWebhooksAnalytics { patch_webhook_count: 1, ..Default::default() } + } + pub fn post_webhook() -> Self { PatchWebhooksAnalytics { post_webhook_count: 1, ..Default::default() } } @@ -168,6 +174,7 @@ impl Aggregate for PatchWebhooksAnalytics { fn aggregate(self: Box, new: Box) -> Box { Box::new(PatchWebhooksAnalytics { patch_webhooks_count: self.patch_webhooks_count + new.patch_webhooks_count, + patch_webhook_count: self.patch_webhook_count + new.patch_webhook_count, post_webhook_count: self.post_webhook_count + new.post_webhook_count, delete_webhook_count: self.delete_webhook_count + new.delete_webhook_count, }) @@ -409,6 +416,50 @@ async fn post_webhook( Ok(HttpResponse::Created().json(WebhookWithMetadata { uuid, is_editable: true, webhook })) } +#[utoipa::path( + patch, + path = "/{uuid}", + tag = "Webhooks", + request_body = WebhookSettings, + security(("Bearer" = ["webhooks.update", "*"])), + responses( + (status = 200, description = "Webhook updated successfully", body = WebhookWithMetadata, content_type = "application/json", example = json!({ + "uuid": "550e8400-e29b-41d4-a716-446655440000", + "url": "https://your.site/on-tasks-completed", + "headers": { + "Authorization": "Bearer a-secret-token" + }, + "isEditable": true + })), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json"), + (status = 400, description = "Bad request", body = ResponseError, content_type = "application/json"), + ), + params( + ("uuid" = Uuid, Path, description = "The universally unique identifier of the webhook") + ) +)] +async fn patch_webhook( + index_scheduler: GuardedData, Data>, + uuid: Path, + webhook_settings: AwebJson, + req: HttpRequest, + analytics: Data, +) -> Result { + let uuid = uuid.into_inner(); + + let webhooks = patch_webhooks_inner( + &index_scheduler, + WebhooksSettings { + webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Set(webhook_settings.0))])), + }, + )?; + let webhook = webhooks.webhooks.get(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?.clone(); + + analytics.publish(PatchWebhooksAnalytics::patch_webhook(), &req); + + Ok(HttpResponse::Ok().json(WebhookWithMetadata { uuid, is_editable: uuid != Uuid::nil(), webhook })) +} + #[utoipa::path( delete, path = "/{uuid}", From bb43bf122ea09f1abdd3b9a20120c8906dd2406b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine?= Date: Thu, 31 Jul 2025 12:55:19 +0200 Subject: [PATCH 236/312] Update .github/pull_request_template.md Co-authored-by: Louis Dureuil --- .github/pull_request_template.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 2e6ee0fff..3665d3303 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -7,7 +7,10 @@ Fixes #... ⚠️ Ensure the following requirements before merging ⚠️ - [] Automated tests have been added. - [] If some tests cannot be automated, manual rigorous tests should be applied. -- [] ⚠️ If there is an change in the DB: it's mandatory to manually test the `--experimental-dumpless-upgrade` on a DB of the previous Meilisearch minor version (e.g. v1.13 for the v1.14 release). +- [] ⚠️ If there is any change in the DB: + - [ ] Test that any impacted DB still works as expected after using `--experimental-dumpless-upgrade` on a DB created with the last released Meilisearch + - [ ] Test that during the upgrade, **search is still available** (artificially make the upgrade longer if needed) + - [ ] Set the `db change` label. - [] If necessary, the feature have been tested in the Cloud production environment (with [prototypes](./documentation/prototypes.md)) and the Cloud UI is ready. - [] If necessary, the [documentation](https://github.com/meilisearch/documentation) related to the implemented feature in the PR is ready. - [] If necessary, the [integrations](https://github.com/meilisearch/integration-guides) related to the implemented feature in the PR are ready. From ee80fc87c95a4c94190b3bb74066aa3c05fa80af Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 13:00:43 +0200 Subject: [PATCH 237/312] Add test for patch endpoint --- crates/meilisearch/src/routes/webhooks.rs | 6 +- crates/meilisearch/tests/common/server.rs | 5 ++ crates/meilisearch/tests/tasks/webhook.rs | 84 +++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 4b925c325..776667829 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -457,7 +457,11 @@ async fn patch_webhook( analytics.publish(PatchWebhooksAnalytics::patch_webhook(), &req); - Ok(HttpResponse::Ok().json(WebhookWithMetadata { uuid, is_editable: uuid != Uuid::nil(), webhook })) + Ok(HttpResponse::Ok().json(WebhookWithMetadata { + uuid, + is_editable: uuid != Uuid::nil(), + webhook, + })) } #[utoipa::path( diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index dd690c3db..0b57ca37a 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -200,6 +200,11 @@ impl Server { self.service.delete(url).await } + pub async fn patch_webhook(&self, uuid: impl AsRef, value: Value) -> (Value, StatusCode) { + let url = format!("/webhooks/{}", uuid.as_ref()); + self.service.patch(url, value).await + } + pub async fn get_metrics(&self) -> (Value, StatusCode) { self.service.get("/metrics").await } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 9d66800af..3d27d6be6 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -329,3 +329,87 @@ async fn post_get_delete() { let (_value, code) = server.get_webhook(uuid).await; snapshot!(code, @"404 Not Found"); } + +#[actix_web::test] +async fn patch() { + let server = Server::new().await; + + let uuid = Uuid::new_v4().to_string(); + let (value, code) = + server.patch_webhook(&uuid, json!({ "headers": { "authorization": "TOKEN" } })).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "The URL for the webhook `[uuid]` is missing.", + "code": "invalid_webhooks_url", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + } + "#); + + let (value, code) = + server.patch_webhook(&uuid, json!({ "url": "https://example.com/hook" })).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "https://example.com/hook", + "headers": {} + } + "#); + + let (value, code) = + server.patch_webhook(&uuid, json!({ "headers": { "authorization": "TOKEN" } })).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "https://example.com/hook", + "headers": { + "authorization": "TOKEN" + } + } + "#); + + let (value, code) = + server.patch_webhook(&uuid, json!({ "headers": { "authorization2": "TOKEN" } })).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "https://example.com/hook", + "headers": { + "authorization": "TOKEN", + "authorization2": "TOKEN" + } + } + "#); + + let (value, code) = + server.patch_webhook(&uuid, json!({ "headers": { "authorization": null } })).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "https://example.com/hook", + "headers": { + "authorization2": "TOKEN" + } + } + "#); + + let (value, code) = server.patch_webhook(&uuid, json!({ "url": null })).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "The URL for the webhook `[uuid]` is missing.", + "code": "invalid_webhooks_url", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + } + "#); +} From 35537e0b0b2065243650af43727bfc5ac016c377 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 14:12:09 +0200 Subject: [PATCH 238/312] Add single_receives_data test --- crates/meilisearch/tests/tasks/webhook.rs | 56 +++++++++++++++-------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 3d27d6be6..14946e415 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -71,17 +71,50 @@ async fn create_webhook_server() -> WebhookHandle { #[actix_web::test] async fn cli_only() { - let WebhookHandle { server_handle, url, mut receiver } = create_webhook_server().await; - let db_path = tempfile::tempdir().unwrap(); let server = Server::new_with_options(Opt { - task_webhook_url: Some(Url::parse(&url).unwrap()), + task_webhook_url: Some(Url::parse("https://example-cli.com/").unwrap()), task_webhook_authorization_header: Some(String::from("Bearer a-secret-token")), ..default_settings(db_path.path()) }) .await .unwrap(); + let (webhooks, code) = server.get_webhooks().await; + snapshot!(code, @"200 OK"); + snapshot!(webhooks, @r#" + { + "results": [ + { + "uuid": "00000000-0000-0000-0000-000000000000", + "isEditable": false, + "url": "https://example-cli.com/", + "headers": { + "Authorization": "Bearer a-secret-token" + } + } + ] + } + "#); +} + +#[actix_web::test] +async fn single_receives_data() { + let WebhookHandle { server_handle, url, mut receiver } = create_webhook_server().await; + + let server = Server::new().await; + + let (value, code) = server.create_webhook(json!({ "url": url })).await; + snapshot!(code, @"201 Created"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]", ".url" => "[ignored]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "[ignored]", + "headers": {} + } + "#); + let index = server.index("tamo"); // May be flaky: we're relying on the fact that while the first document addition is processed, the other // operations will be received and will be batched together. If it doesn't happen it's not a problem @@ -128,23 +161,6 @@ async fn cli_only() { assert!(nb_tasks == 5, "We should have received the 5 tasks but only received {nb_tasks}"); - let (webhooks, code) = server.get_webhooks().await; - snapshot!(code, @"200 OK"); - snapshot!(json_string!(webhooks, { ".results[].url" => "[ignored]" }), @r#" - { - "results": [ - { - "uuid": "00000000-0000-0000-0000-000000000000", - "isEditable": false, - "url": "[ignored]", - "headers": { - "Authorization": "Bearer a-secret-token" - } - } - ] - } - "#); - server_handle.abort(); } From c1a5a545b6f1b4d4f3b46e1faf83e717c32b7ee1 Mon Sep 17 00:00:00 2001 From: curquiza Date: Thu, 31 Jul 2025 15:23:45 +0200 Subject: [PATCH 239/312] Adapt Go CI to recent change in the Go repo --- .github/workflows/sdks-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sdks-tests.yml b/.github/workflows/sdks-tests.yml index dc4d51068..0bcc1d7a7 100644 --- a/.github/workflows/sdks-tests.yml +++ b/.github/workflows/sdks-tests.yml @@ -114,7 +114,7 @@ jobs: dep ensure fi - name: Run integration tests - run: go test -v ./... + run: go test --race -v ./integration meilisearch-java-tests: needs: define-docker-image From ed147f80ac782a0fade13f009b4365859b037ec0 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 16:45:30 +0200 Subject: [PATCH 240/312] Add test and fix bug --- crates/index-scheduler/src/lib.rs | 41 +++++++++++++------ crates/meilisearch/tests/tasks/webhook.rs | 49 ++++++++++++++++++++++- 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index ce8791a63..8d7617b6c 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -828,16 +828,32 @@ impl IndexScheduler { written: 0, }; - enum EitherRead { - Other(T), - Data(Vec), + enum EitherRead<'a, T: Read> { + Other(Option), + Data(&'a [u8]), } - impl Read for &mut EitherRead { + impl EitherRead<'_, T> { + /// A clone that works only once for the Other variant. + fn clone(&mut self) -> Self { + match self { + Self::Other(r) => { + let r = r.take(); + Self::Other(r) + } + Self::Data(arg0) => Self::Data(arg0), + } + } + } + + impl Read for EitherRead<'_, T> { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { match self { - EitherRead::Other(reader) => reader.read(buf), - EitherRead::Data(data) => data.as_slice().read(buf), + EitherRead::Other(Some(reader)) => reader.read(buf), + EitherRead::Other(None) => { + Err(io::Error::new(io::ErrorKind::Other, "No reader available")) + } + EitherRead::Data(data) => data.read(buf), } } } @@ -845,16 +861,17 @@ impl IndexScheduler { let mut reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); // When there is more than one webhook, cache the data in memory + let mut data; let mut reader = match webhooks.webhooks.len() { - 1 => EitherRead::Other(reader), + 1 => EitherRead::Other(Some(reader)), _ => { - let mut data = Vec::new(); + data = Vec::new(); reader.read_to_end(&mut data)?; - EitherRead::Data(data) + EitherRead::Data(&data) } }; - for (name, Webhook { url, headers }) in webhooks.webhooks.iter() { + for (uuid, Webhook { url, headers }) in webhooks.webhooks.iter() { let mut request = ureq::post(url) .timeout(Duration::from_secs(30)) .set("Content-Encoding", "gzip") @@ -863,8 +880,8 @@ impl IndexScheduler { request = request.set(header_name, header_value); } - if let Err(e) = request.send(&mut reader) { - tracing::error!("While sending data to the webhook {name}: {e}"); + if let Err(e) = request.send(reader.clone()) { + tracing::error!("While sending data to the webhook {uuid}: {e}"); } } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 14946e415..4caa7df92 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -115,10 +115,10 @@ async fn single_receives_data() { } "#); - let index = server.index("tamo"); // May be flaky: we're relying on the fact that while the first document addition is processed, the other // operations will be received and will be batched together. If it doesn't happen it's not a problem // the rest of the test won't assume anything about the number of tasks per batch. + let index = server.index("tamo"); for i in 0..5 { let (_, _status) = index.add_documents(json!({ "id": i, "doggo": "bone" }), None).await; } @@ -164,6 +164,53 @@ async fn single_receives_data() { server_handle.abort(); } +#[actix_web::test] +async fn multiple_receive_data() { + let server = Server::new().await; + + let WebhookHandle { server_handle: handle1, url: url1, receiver: mut receiver1 } = + create_webhook_server().await; + let WebhookHandle { server_handle: handle2, url: url2, receiver: mut receiver2 } = + create_webhook_server().await; + let WebhookHandle { server_handle: handle3, url: url3, receiver: mut receiver3 } = + create_webhook_server().await; + + for url in [url1, url2, url3] { + let (value, code) = server.create_webhook(json!({ "url": url })).await; + snapshot!(code, @"201 Created"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]", ".url" => "[ignored]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "[ignored]", + "headers": {} + } + "#); + } + let index = server.index("tamo"); + let (_, status) = index.add_documents(json!({ "id": 1, "doggo": "bone" }), None).await; + snapshot!(status, @"202 Accepted"); + + let mut count1 = 0; + let mut count2 = 0; + let mut count3 = 0; + while count1 == 0 || count2 == 0 || count3 == 0 { + tokio::select! { + msg = receiver1.recv() => { if msg.is_some() { count1 += 1; } }, + msg = receiver2.recv() => { if msg.is_some() { count2 += 1; } }, + msg = receiver3.recv() => { if msg.is_some() { count3 += 1; } }, + } + } + + assert_eq!(count1, 1); + assert_eq!(count2, 1); + assert_eq!(count3, 1); + + handle1.abort(); + handle2.abort(); + handle3.abort(); +} + #[actix_web::test] async fn cli_with_dumps() { let db_path = tempfile::tempdir().unwrap(); From 2ec80a1ae2b67886da052b41d913a57335cbce69 Mon Sep 17 00:00:00 2001 From: ManyTheFish Date: Thu, 31 Jul 2025 17:14:38 +0200 Subject: [PATCH 241/312] update mini-dashboard to v0.2.21 --- crates/meilisearch/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/Cargo.toml b/crates/meilisearch/Cargo.toml index 21f6b58e5..9da316b84 100644 --- a/crates/meilisearch/Cargo.toml +++ b/crates/meilisearch/Cargo.toml @@ -170,5 +170,5 @@ german = ["meilisearch-types/german"] turkish = ["meilisearch-types/turkish"] [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.20/build.zip" -sha1 = "82a7ddd7bf14bb5323c3d235d2b62892a98b6a59" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.21/build.zip" +sha1 = "94f56a8e24e2e3a1bc1bd7d9ceaa23464a5e241a" From e3a6d63b523f0477367c0abc4f2971d6bdd779df Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 1 Aug 2025 08:42:27 +0200 Subject: [PATCH 242/312] Add utoipa types --- crates/meilisearch/src/routes/mod.rs | 5 ++++- crates/meilisearch/src/routes/webhooks.rs | 12 ++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 4ae72b0bd..2a41ce021 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -41,6 +41,9 @@ use crate::routes::indexes::IndexView; use crate::routes::multi_search::SearchResults; use crate::routes::network::{Network, Remote}; use crate::routes::swap_indexes::SwapIndexesPayload; +use crate::routes::webhooks::{ + WebhookResults, WebhookSettings, WebhookWithMetadata, WebhooksSettings, +}; use crate::search::{ FederatedSearch, FederatedSearchResult, Federation, FederationOptions, MergeFacets, SearchQueryWithIndex, SearchResultWithIndex, SimilarQuery, SimilarResult, @@ -101,7 +104,7 @@ mod webhooks; url = "/", description = "Local server", )), - components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export)) + components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export, WebhookSettings, WebhooksSettings, WebhookResults, WebhookWithMetadata)) )] pub struct MeilisearchApi; diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 776667829..054279727 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -51,7 +51,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] -struct WebhookSettings { +pub(super) struct WebhookSettings { #[schema(value_type = Option)] #[deserr(default, error = DeserrJsonError)] #[serde(default)] @@ -66,7 +66,7 @@ struct WebhookSettings { #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] -struct WebhooksSettings { +pub(super) struct WebhooksSettings { #[schema(value_type = Option>)] #[serde(default)] webhooks: Setting>>, @@ -75,7 +75,7 @@ struct WebhooksSettings { #[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] -struct WebhookWithMetadata { +pub(super) struct WebhookWithMetadata { uuid: Uuid, is_editable: bool, #[schema(value_type = WebhookSettings)] @@ -83,9 +83,9 @@ struct WebhookWithMetadata { webhook: Webhook, } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] -struct WebhookResults { +pub(super) struct WebhookResults { results: Vec, } @@ -95,7 +95,7 @@ struct WebhookResults { tag = "Webhooks", security(("Bearer" = ["webhooks.get", "*.get", "*"])), responses( - (status = OK, description = "Webhooks are returned", body = WebhooksSettings, content_type = "application/json", example = json!({ + (status = OK, description = "Webhooks are returned", body = WebhookResults, content_type = "application/json", example = json!({ "results": [ { "uuid": "550e8400-e29b-41d4-a716-446655440000", From beb532e2a752efde58644862e4fa66aab4462ca9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Aug 2025 17:04:58 +0000 Subject: [PATCH 243/312] Bump sigstore/cosign-installer from 3.8.2 to 3.9.2 Bumps [sigstore/cosign-installer](https://github.com/sigstore/cosign-installer) from 3.8.2 to 3.9.2. - [Release notes](https://github.com/sigstore/cosign-installer/releases) - [Commits](https://github.com/sigstore/cosign-installer/compare/3454372f43399081ed03b604cb2d021dabca52bb...d58896d6a1865668819e1d91763c7751a165e159) --- updated-dependencies: - dependency-name: sigstore/cosign-installer dependency-version: 3.9.2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/publish-docker-images.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml index 6d2ce2248..0ac834bbb 100644 --- a/.github/workflows/publish-docker-images.yml +++ b/.github/workflows/publish-docker-images.yml @@ -65,7 +65,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Install cosign - uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # tag=v3.8.2 + uses: sigstore/cosign-installer@d58896d6a1865668819e1d91763c7751a165e159 # tag=v3.9.2 - name: Login to Docker Hub uses: docker/login-action@v3 From ddea0b1570388092f1f7895c1ead473971b04bfb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 1 Aug 2025 17:05:02 +0000 Subject: [PATCH 244/312] Bump svenstaro/upload-release-action from 2.11.1 to 2.11.2 Bumps [svenstaro/upload-release-action](https://github.com/svenstaro/upload-release-action) from 2.11.1 to 2.11.2. - [Release notes](https://github.com/svenstaro/upload-release-action/releases) - [Changelog](https://github.com/svenstaro/upload-release-action/blob/master/CHANGELOG.md) - [Commits](https://github.com/svenstaro/upload-release-action/compare/2.11.1...2.11.2) --- updated-dependencies: - dependency-name: svenstaro/upload-release-action dependency-version: 2.11.2 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/publish-apt-brew-pkg.yml | 2 +- .github/workflows/publish-binaries.yml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish-apt-brew-pkg.yml b/.github/workflows/publish-apt-brew-pkg.yml index 5b6994dcf..9a9c566e3 100644 --- a/.github/workflows/publish-apt-brew-pkg.yml +++ b/.github/workflows/publish-apt-brew-pkg.yml @@ -32,7 +32,7 @@ jobs: - name: Build deb package run: cargo deb -p meilisearch -o target/debian/meilisearch.deb - name: Upload debian pkg to release - uses: svenstaro/upload-release-action@2.11.1 + uses: svenstaro/upload-release-action@2.11.2 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/debian/meilisearch.deb diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-binaries.yml index 3200e778e..27d8c3610 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-binaries.yml @@ -51,7 +51,7 @@ jobs: # No need to upload binaries for dry run (cron) - name: Upload binaries to release if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.11.1 + uses: svenstaro/upload-release-action@2.11.2 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/release/meilisearch @@ -81,7 +81,7 @@ jobs: # No need to upload binaries for dry run (cron) - name: Upload binaries to release if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.11.1 + uses: svenstaro/upload-release-action@2.11.2 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/release/${{ matrix.artifact_name }} @@ -113,7 +113,7 @@ jobs: - name: Upload the binary to release # No need to upload binaries for dry run (cron) if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.11.1 + uses: svenstaro/upload-release-action@2.11.2 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/${{ matrix.target }}/release/meilisearch @@ -178,7 +178,7 @@ jobs: - name: Upload the binary to release # No need to upload binaries for dry run (cron) if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.11.1 + uses: svenstaro/upload-release-action@2.11.2 with: repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} file: target/${{ matrix.target }}/release/meilisearch From 4182e631d60cc6f63c68b2539a16d96a2745b815 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine?= Date: Mon, 4 Aug 2025 09:59:54 +0200 Subject: [PATCH 245/312] Potential fix for code scanning alert no. 63: Workflow does not contain permissions Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .github/workflows/release-drafter.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release-drafter.yml b/.github/workflows/release-drafter.yml index 20f2d83f4..2f8ec04b0 100644 --- a/.github/workflows/release-drafter.yml +++ b/.github/workflows/release-drafter.yml @@ -1,5 +1,9 @@ name: Release Drafter +permissions: + contents: read + pull-requests: write + on: push: branches: From 05dd8e0d6279f91a76e4bb2cd338e8deeec9fc58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Mon, 4 Aug 2025 11:14:10 +0200 Subject: [PATCH 246/312] update mini-dashboard to v0.2.22 --- crates/meilisearch/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/Cargo.toml b/crates/meilisearch/Cargo.toml index 9da316b84..5cbbb6666 100644 --- a/crates/meilisearch/Cargo.toml +++ b/crates/meilisearch/Cargo.toml @@ -170,5 +170,5 @@ german = ["meilisearch-types/german"] turkish = ["meilisearch-types/turkish"] [package.metadata.mini-dashboard] -assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.21/build.zip" -sha1 = "94f56a8e24e2e3a1bc1bd7d9ceaa23464a5e241a" +assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.22/build.zip" +sha1 = "b70b2036b5f167da9ea0b637da8b320c7ea88254" From f8d70249a7d80099f77b3af7f174023f236b4f1c Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 4 Aug 2025 13:59:11 +0200 Subject: [PATCH 247/312] Update process with Ruleset branch addition --- documentation/release.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/documentation/release.md b/documentation/release.md index f308bab98..f70fcf872 100644 --- a/documentation/release.md +++ b/documentation/release.md @@ -47,11 +47,14 @@ git checkout -b release-vX.Y.Z+1 # Increase the Z here git push -u origin release-vX.Y.Z+1 ``` -2. Change the [version in `Cargo.toml` file](https://github.com/meilisearch/meilisearch/blob/e9b62aacb38f2c7a777adfda55293d407e0d6254/Cargo.toml#L21). You can use [our automation](https://github.com/meilisearch/meilisearch/actions/workflows/update-cargo-toml-version.yml) -> click on `Run workflow` -> Fill the appropriate version and run it on the newly created branch `release-vX.Y.Z` -> Click on "Run workflow". A PR updating the version in the `Cargo.toml` and `Cargo.lock` files will be created. +2. Add the newly created branch `release-vX.Y.Z+1` to "Target Branches" of [this GitHub Ruleset](https://github.com/meilisearch/meilisearch/settings/rules/4253297). +Why? GitHub Merge Queue does not work with branch patterns yet, so we have to add the new created branch to the GitHub Ruleset to be able to use GitHub Merge Queue. -3. Open and merge the PRs (fixing your bugs): they should point to `release-vX.Y.Z+1` branch. +3. Change the [version in `Cargo.toml` file](https://github.com/meilisearch/meilisearch/blob/e9b62aacb38f2c7a777adfda55293d407e0d6254/Cargo.toml#L21). You can use [our automation](https://github.com/meilisearch/meilisearch/actions/workflows/update-cargo-toml-version.yml) -> click on `Run workflow` -> Fill the appropriate version and run it on the newly created branch `release-vX.Y.Z` -> Click on "Run workflow". A PR updating the version in the `Cargo.toml` and `Cargo.lock` files will be created. -4. Go to the GitHub interface, in the [`Release` section](https://github.com/meilisearch/meilisearch/releases) and click on `Draft a new release` +4. Open and merge the PRs (fixing your bugs): they should point to `release-vX.Y.Z+1` branch. + +5. Go to the GitHub interface, in the [`Release` section](https://github.com/meilisearch/meilisearch/releases) and click on `Draft a new release` ⚠️⚠️⚠️ Publish on `release-vX.Y.Z+1` branch, not on `main`! ⚠️ If doing a patch release that should NOT be the `latest` release: @@ -60,7 +63,7 @@ git push -u origin release-vX.Y.Z+1 - Once the release is created, you don't have to care about Homebrew, APT and Docker CIs: they will not consider this new release as the latest; the CIs are already adapted for this situation. - However, the [CI updating the `latest` git tag](https://github.com/meilisearch/meilisearch/actions/workflows/latest-git-tag.yml) is not working for this situation currently and will attach the `latest` git tag to the just-created release, which is something we don't want! If you don't succeed in stopping the CI on time, don't worry, you just have to re-run the [old CI](https://github.com/meilisearch/meilisearch/actions/workflows/latest-git-tag.yml) corresponding to the real latest release, and the `latest` git tag will be attached back to the right commit. -5. Bring the new commits back from `release-vX.Y.Z+1` to `main` by merging a PR originating `release-vX.Y.Z+1` and pointing to `main`. +6. Bring the new commits back from `release-vX.Y.Z+1` to `main` by merging a PR originating `release-vX.Y.Z+1` and pointing to `main`. ⚠️ If you encounter any merge conflicts, please do NOT fix the git conflicts directly on the `release-vX.Y.Z` branch. It would bring the changes present in `main` into `release-vX.Y.Z`, which would break a potential future patched release. From 7acbb1e14086b41e5de3cc9813ea9cc827118fdc Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 14:49:27 +0200 Subject: [PATCH 248/312] Remove PATCH /webhooks --- crates/meilisearch/src/routes/mod.rs | 6 +- crates/meilisearch/src/routes/webhooks.rs | 259 ++++++++-------------- crates/meilisearch/tests/common/server.rs | 4 - crates/meilisearch/tests/tasks/webhook.rs | 43 ++-- 4 files changed, 114 insertions(+), 198 deletions(-) diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 2a41ce021..745ac5824 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -41,9 +41,7 @@ use crate::routes::indexes::IndexView; use crate::routes::multi_search::SearchResults; use crate::routes::network::{Network, Remote}; use crate::routes::swap_indexes::SwapIndexesPayload; -use crate::routes::webhooks::{ - WebhookResults, WebhookSettings, WebhookWithMetadata, WebhooksSettings, -}; +use crate::routes::webhooks::{WebhookResults, WebhookSettings, WebhookWithMetadata}; use crate::search::{ FederatedSearch, FederatedSearchResult, Federation, FederationOptions, MergeFacets, SearchQueryWithIndex, SearchResultWithIndex, SimilarQuery, SimilarResult, @@ -104,7 +102,7 @@ mod webhooks; url = "/", description = "Local server", )), - components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export, WebhookSettings, WebhooksSettings, WebhookResults, WebhookWithMetadata)) + components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export, WebhookSettings, WebhookResults, WebhookWithMetadata)) )] pub struct MeilisearchApi; diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 054279727..b6f85f7d0 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -10,7 +10,7 @@ use meilisearch_types::error::deserr_codes::{InvalidWebhooksHeaders, InvalidWebh use meilisearch_types::error::{ErrorCode, ResponseError}; use meilisearch_types::keys::actions; use meilisearch_types::milli::update::Setting; -use meilisearch_types::webhooks::{Webhook, Webhooks}; +use meilisearch_types::webhooks::Webhook; use serde::Serialize; use tracing::debug; use utoipa::{OpenApi, ToSchema}; @@ -23,7 +23,7 @@ use crate::extractors::sequential_extractor::SeqHandler; #[derive(OpenApi)] #[openapi( - paths(get_webhooks, patch_webhooks, get_webhook, post_webhook, patch_webhook, delete_webhook), + paths(get_webhooks, get_webhook, post_webhook, patch_webhook, delete_webhook), tags(( name = "Webhooks", description = "The `/webhooks` route allows you to register endpoints to be called once tasks are processed.", @@ -36,7 +36,6 @@ pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") .route(web::get().to(get_webhooks)) - .route(web::patch().to(SeqHandler(patch_webhooks))) .route(web::post().to(SeqHandler(post_webhook))), ) .service( @@ -62,16 +61,6 @@ pub(super) struct WebhookSettings { headers: Setting>>, } -#[derive(Debug, Deserr, ToSchema)] -#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -#[schema(rename_all = "camelCase")] -pub(super) struct WebhooksSettings { - #[schema(value_type = Option>)] - #[serde(default)] - webhooks: Setting>>, -} - #[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] @@ -83,6 +72,12 @@ pub(super) struct WebhookWithMetadata { webhook: Webhook, } +impl WebhookWithMetadata { + pub fn from(uuid: Uuid, webhook: Webhook) -> Self { + Self { uuid, is_editable: uuid != Uuid::nil(), webhook } + } +} + #[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] pub(super) struct WebhookResults { @@ -142,17 +137,12 @@ async fn get_webhooks( #[derive(Serialize, Default)] pub struct PatchWebhooksAnalytics { - patch_webhooks_count: usize, patch_webhook_count: usize, post_webhook_count: usize, delete_webhook_count: usize, } impl PatchWebhooksAnalytics { - pub fn patch_webhooks() -> Self { - PatchWebhooksAnalytics { patch_webhooks_count: 1, ..Default::default() } - } - pub fn patch_webhook() -> Self { PatchWebhooksAnalytics { patch_webhook_count: 1, ..Default::default() } } @@ -173,7 +163,6 @@ impl Aggregate for PatchWebhooksAnalytics { fn aggregate(self: Box, new: Box) -> Box { Box::new(PatchWebhooksAnalytics { - patch_webhooks_count: self.patch_webhooks_count + new.patch_webhooks_count, patch_webhook_count: self.patch_webhook_count + new.patch_webhook_count, post_webhook_count: self.post_webhook_count + new.post_webhook_count, delete_webhook_count: self.delete_webhook_count + new.delete_webhook_count, @@ -213,130 +202,45 @@ impl ErrorCode for WebhooksError { } } -#[utoipa::path( - patch, - path = "", - tag = "Webhooks", - request_body = WebhooksSettings, - security(("Bearer" = ["webhooks.update", "*"])), - responses( - (status = 200, description = "Returns the updated webhooks", body = WebhooksSettings, content_type = "application/json", example = json!({ - "webhooks": { - "550e8400-e29b-41d4-a716-446655440000": { - "url": "http://example.com/webhook", - }, - "550e8400-e29b-41d4-a716-446655440001": { - "url": "https://your.site/on-tasks-completed", - "headers": { - "Authorization": "Bearer a-secret-token" - } - } - } - })), - (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!({ - "message": "The Authorization header is missing. It must use the bearer authorization method.", - "code": "missing_authorization_header", - "type": "auth", - "link": "https://docs.meilisearch.com/errors#missing_authorization_header" - })), - ) -)] -async fn patch_webhooks( - index_scheduler: GuardedData, Data>, - new_webhooks: AwebJson, - req: HttpRequest, - analytics: Data, -) -> Result { - let webhooks = patch_webhooks_inner(&index_scheduler, new_webhooks.0)?; +fn patch_webhook_inner( + uuid: &Uuid, + old_webhook: Option, + new_webhook: WebhookSettings, +) -> Result { + let (old_url, mut headers) = + old_webhook.map(|w| (Some(w.url), w.headers)).unwrap_or((None, BTreeMap::new())); - analytics.publish(PatchWebhooksAnalytics::patch_webhooks(), &req); - - Ok(HttpResponse::Ok().json(webhooks)) -} - -fn patch_webhooks_inner( - index_scheduler: &GuardedData, Data>, - new_webhooks: WebhooksSettings, -) -> Result { - fn merge_webhook( - uuid: &Uuid, - old_webhook: Option, - new_webhook: WebhookSettings, - ) -> Result { - let (old_url, mut headers) = - old_webhook.map(|w| (Some(w.url), w.headers)).unwrap_or((None, BTreeMap::new())); - - let url = match new_webhook.url { - Setting::Set(url) => url, - Setting::NotSet => old_url.ok_or_else(|| WebhooksError::MissingUrl(uuid.to_owned()))?, - Setting::Reset => return Err(WebhooksError::MissingUrl(uuid.to_owned())), - }; - - let headers = match new_webhook.headers { - Setting::Set(new_headers) => { - for (name, value) in new_headers { - match value { - Setting::Set(value) => { - headers.insert(name, value); - } - Setting::NotSet => continue, - Setting::Reset => { - headers.remove(&name); - continue; - } - } - } - headers - } - Setting::NotSet => headers, - Setting::Reset => BTreeMap::new(), - }; - - if headers.len() > 200 { - return Err(WebhooksError::TooManyHeaders(uuid.to_owned())); - } - - Ok(Webhook { url, headers }) - } - - debug!(parameters = ?new_webhooks, "Patch webhooks"); - - let Webhooks { mut webhooks } = index_scheduler.webhooks(); - - match new_webhooks.webhooks { - Setting::Set(new_webhooks) => { - for (uuid, new_webhook) in new_webhooks { - if uuid.is_nil() { - return Err(WebhooksError::ReservedWebhook(uuid).into()); - } - - match new_webhook { - Setting::Set(new_webhook) => { - let old_webhook = webhooks.remove(&uuid); - let webhook = merge_webhook(&uuid, old_webhook, new_webhook)?; - webhooks.insert(uuid, webhook); - } - Setting::Reset => { - webhooks.remove(&uuid); - } - Setting::NotSet => (), - } - } - } - Setting::Reset => webhooks.clear(), - Setting::NotSet => (), + let url = match new_webhook.url { + Setting::Set(url) => url, + Setting::NotSet => old_url.ok_or_else(|| WebhooksError::MissingUrl(uuid.to_owned()))?, + Setting::Reset => return Err(WebhooksError::MissingUrl(uuid.to_owned())), }; - if webhooks.len() > 20 { - return Err(WebhooksError::TooManyWebhooks.into()); + let headers = match new_webhook.headers { + Setting::Set(new_headers) => { + for (name, value) in new_headers { + match value { + Setting::Set(value) => { + headers.insert(name, value); + } + Setting::NotSet => continue, + Setting::Reset => { + headers.remove(&name); + continue; + } + } + } + headers + } + Setting::NotSet => headers, + Setting::Reset => BTreeMap::new(), + }; + + if headers.len() > 200 { + return Err(WebhooksError::TooManyHeaders(uuid.to_owned())); } - let webhooks = Webhooks { webhooks }; - index_scheduler.put_webhooks(webhooks.clone())?; - - debug!(returns = ?webhooks, "Patch webhooks"); - - Ok(webhooks) + Ok(Webhook { url, headers }) } #[utoipa::path( @@ -401,19 +305,35 @@ async fn post_webhook( req: HttpRequest, analytics: Data, ) -> Result { - let uuid = Uuid::new_v4(); + let webhook_settings = webhook_settings.into_inner(); + debug!(parameters = ?webhook_settings, "Post webhook"); - let webhooks = patch_webhooks_inner( - &index_scheduler, - WebhooksSettings { - webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Set(webhook_settings.0))])), - }, - )?; - let webhook = webhooks.webhooks.get(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?.clone(); + let uuid = Uuid::new_v4(); + if webhook_settings.headers.as_ref().set().is_some_and(|h| h.len() > 200) { + return Err(WebhooksError::TooManyHeaders(uuid).into()); + } + + let mut webhooks = index_scheduler.webhooks(); + if dbg!(webhooks.webhooks.len() >= 20) { + return Err(WebhooksError::TooManyWebhooks.into()); + } + + let webhook = Webhook { + url: webhook_settings.url.set().ok_or(WebhooksError::MissingUrl(uuid))?, + headers: webhook_settings + .headers + .set() + .map(|h| h.into_iter().map(|(k, v)| (k, v.set().unwrap_or_default())).collect()) + .unwrap_or_default(), + }; + webhooks.webhooks.insert(uuid, webhook.clone()); + index_scheduler.put_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::post_webhook(), &req); - Ok(HttpResponse::Created().json(WebhookWithMetadata { uuid, is_editable: true, webhook })) + let response = WebhookWithMetadata::from(uuid, webhook); + debug!(returns = ?response, "Post webhook"); + Ok(HttpResponse::Created().json(response)) } #[utoipa::path( @@ -446,22 +366,29 @@ async fn patch_webhook( analytics: Data, ) -> Result { let uuid = uuid.into_inner(); + let webhook_settings = webhook_settings.into_inner(); + debug!(parameters = ?(uuid, &webhook_settings), "Patch webhook"); - let webhooks = patch_webhooks_inner( - &index_scheduler, - WebhooksSettings { - webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Set(webhook_settings.0))])), - }, - )?; - let webhook = webhooks.webhooks.get(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?.clone(); + if uuid.is_nil() { + return Err(WebhooksError::ReservedWebhook(uuid).into()); + } + + let mut webhooks = index_scheduler.webhooks(); + let old_webhook = webhooks.webhooks.remove(&uuid); + let webhook = patch_webhook_inner(&uuid, old_webhook, webhook_settings)?; + + if webhook.headers.len() > 200 { + return Err(WebhooksError::TooManyHeaders(uuid).into()); + } + + webhooks.webhooks.insert(uuid, webhook.clone()); + index_scheduler.put_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::patch_webhook(), &req); - Ok(HttpResponse::Ok().json(WebhookWithMetadata { - uuid, - is_editable: uuid != Uuid::nil(), - webhook, - })) + let response = WebhookWithMetadata::from(uuid, webhook); + debug!(returns = ?response, "Patch webhook"); + Ok(HttpResponse::Ok().json(response)) } #[utoipa::path( @@ -485,18 +412,18 @@ async fn delete_webhook( analytics: Data, ) -> Result { let uuid = uuid.into_inner(); + debug!(parameters = ?uuid, "Delete webhook"); - let webhooks = index_scheduler.webhooks(); - if !webhooks.webhooks.contains_key(&uuid) { - return Err(WebhooksError::WebhookNotFound(uuid).into()); + if uuid.is_nil() { + return Err(WebhooksError::ReservedWebhook(uuid).into()); } - patch_webhooks_inner( - &index_scheduler, - WebhooksSettings { webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Reset)])) }, - )?; + let mut webhooks = index_scheduler.webhooks(); + webhooks.webhooks.remove(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?; + index_scheduler.put_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::delete_webhook(), &req); + debug!(returns = "No Content", "Delete webhook"); Ok(HttpResponse::NoContent().finish()) } diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 0b57ca37a..4f1e93c88 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -182,10 +182,6 @@ impl Server { self.service.patch("/network", value).await } - pub async fn set_webhooks(&self, value: Value) -> (Value, StatusCode) { - self.service.patch("/webhooks", value).await - } - pub async fn create_webhook(&self, value: Value) -> (Value, StatusCode) { self.service.post("/webhooks", value).await } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 4caa7df92..f8fcd8ff9 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -99,6 +99,7 @@ async fn cli_only() { } #[actix_web::test] +#[ignore = "Broken"] async fn single_receives_data() { let WebhookHandle { server_handle, url, mut receiver } = create_webhook_server().await; @@ -165,6 +166,7 @@ async fn single_receives_data() { } #[actix_web::test] +#[ignore = "Broken"] async fn multiple_receive_data() { let server = Server::new().await; @@ -268,7 +270,7 @@ async fn reserved_names() { let server = Server::new().await; let (value, code) = server - .set_webhooks(json!({ "webhooks": { Uuid::nil(): { "url": "http://localhost:8080" } } })) + .patch_webhook(Uuid::nil().to_string(), json!({ "url": "http://localhost:8080" })) .await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" @@ -280,7 +282,7 @@ async fn reserved_names() { } "#); - let (value, code) = server.set_webhooks(json!({ "webhooks": { Uuid::nil(): null } })).await; + let (value, code) = server.delete_webhook(Uuid::nil().to_string()).await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { @@ -297,17 +299,13 @@ async fn over_limits() { let server = Server::new().await; // Too many webhooks + let mut uuids = Vec::new(); for _ in 0..20 { - let (_value, code) = server - .set_webhooks( - json!({ "webhooks": { Uuid::new_v4(): { "url": "http://localhost:8080" } } }), - ) - .await; - snapshot!(code, @"200 OK"); + let (value, code) = server.create_webhook(json!({ "url": "http://localhost:8080" } )).await; + snapshot!(code, @"201 Created"); + uuids.push(value.get("uuid").unwrap().as_str().unwrap().to_string()); } - let (value, code) = server - .set_webhooks(json!({ "webhooks": { Uuid::new_v4(): { "url": "http://localhost:8080" } } })) - .await; + let (value, code) = server.create_webhook(json!({ "url": "http://localhost:8080" })).await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { @@ -319,26 +317,23 @@ async fn over_limits() { "#); // Reset webhooks - let (value, code) = server.set_webhooks(json!({ "webhooks": null })).await; - snapshot!(code, @"200 OK"); - snapshot!(value, @r#" - { - "webhooks": {} + for uuid in uuids { + let (_value, code) = server.delete_webhook(&uuid).await; + snapshot!(code, @"204 No Content"); } - "#); // Test too many headers - let uuid = Uuid::new_v4(); + let (value, code) = server.create_webhook(json!({ "url": "http://localhost:8080" })).await; + snapshot!(code, @"201 Created"); + let uuid = value.get("uuid").unwrap().as_str().unwrap(); for i in 0..200 { let header_name = format!("header_{i}"); - let (_value, code) = server - .set_webhooks(json!({ "webhooks": { uuid: { "url": "http://localhost:8080", "headers": { header_name: "value" } } } })) - .await; + let (_value, code) = + server.patch_webhook(uuid, json!({ "headers": { header_name: "" } })).await; snapshot!(code, @"200 OK"); } - let (value, code) = server - .set_webhooks(json!({ "webhooks": { uuid: { "url": "http://localhost:8080", "headers": { "header_201": "value" } } } })) - .await; + let (value, code) = + server.patch_webhook(uuid, json!({ "headers": { "header_200": "" } })).await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { From c5caac95dd163e6dd8f868e096d148e38d120463 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 14:51:23 +0200 Subject: [PATCH 249/312] Format --- crates/meilisearch/src/routes/webhooks.rs | 35 +++++++++++------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index b6f85f7d0..07f19c498 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -20,6 +20,7 @@ use crate::analytics::{Aggregate, Analytics}; use crate::extractors::authentication::policies::ActionPolicy; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; +use WebhooksError::*; #[derive(OpenApi)] #[openapi( @@ -191,13 +192,11 @@ enum WebhooksError { impl ErrorCode for WebhooksError { fn error_code(&self) -> meilisearch_types::error::Code { match self { - WebhooksError::MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhooksUrl, - WebhooksError::TooManyWebhooks => meilisearch_types::error::Code::InvalidWebhooks, - WebhooksError::TooManyHeaders(_) => { - meilisearch_types::error::Code::InvalidWebhooksHeaders - } - WebhooksError::ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, - WebhooksError::WebhookNotFound(_) => meilisearch_types::error::Code::WebhookNotFound, + MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhooksUrl, + TooManyWebhooks => meilisearch_types::error::Code::InvalidWebhooks, + TooManyHeaders(_) => meilisearch_types::error::Code::InvalidWebhooksHeaders, + ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, + WebhookNotFound(_) => meilisearch_types::error::Code::WebhookNotFound, } } } @@ -212,8 +211,8 @@ fn patch_webhook_inner( let url = match new_webhook.url { Setting::Set(url) => url, - Setting::NotSet => old_url.ok_or_else(|| WebhooksError::MissingUrl(uuid.to_owned()))?, - Setting::Reset => return Err(WebhooksError::MissingUrl(uuid.to_owned())), + Setting::NotSet => old_url.ok_or_else(|| MissingUrl(uuid.to_owned()))?, + Setting::Reset => return Err(MissingUrl(uuid.to_owned())), }; let headers = match new_webhook.headers { @@ -237,7 +236,7 @@ fn patch_webhook_inner( }; if headers.len() > 200 { - return Err(WebhooksError::TooManyHeaders(uuid.to_owned())); + return Err(TooManyHeaders(uuid.to_owned())); } Ok(Webhook { url, headers }) @@ -271,7 +270,7 @@ async fn get_webhook( let uuid = uuid.into_inner(); let mut webhooks = index_scheduler.webhooks(); - let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?; + let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; Ok(HttpResponse::Ok().json(WebhookWithMetadata { uuid, @@ -310,16 +309,16 @@ async fn post_webhook( let uuid = Uuid::new_v4(); if webhook_settings.headers.as_ref().set().is_some_and(|h| h.len() > 200) { - return Err(WebhooksError::TooManyHeaders(uuid).into()); + return Err(TooManyHeaders(uuid).into()); } let mut webhooks = index_scheduler.webhooks(); if dbg!(webhooks.webhooks.len() >= 20) { - return Err(WebhooksError::TooManyWebhooks.into()); + return Err(TooManyWebhooks.into()); } let webhook = Webhook { - url: webhook_settings.url.set().ok_or(WebhooksError::MissingUrl(uuid))?, + url: webhook_settings.url.set().ok_or(MissingUrl(uuid))?, headers: webhook_settings .headers .set() @@ -370,7 +369,7 @@ async fn patch_webhook( debug!(parameters = ?(uuid, &webhook_settings), "Patch webhook"); if uuid.is_nil() { - return Err(WebhooksError::ReservedWebhook(uuid).into()); + return Err(ReservedWebhook(uuid).into()); } let mut webhooks = index_scheduler.webhooks(); @@ -378,7 +377,7 @@ async fn patch_webhook( let webhook = patch_webhook_inner(&uuid, old_webhook, webhook_settings)?; if webhook.headers.len() > 200 { - return Err(WebhooksError::TooManyHeaders(uuid).into()); + return Err(TooManyHeaders(uuid).into()); } webhooks.webhooks.insert(uuid, webhook.clone()); @@ -415,11 +414,11 @@ async fn delete_webhook( debug!(parameters = ?uuid, "Delete webhook"); if uuid.is_nil() { - return Err(WebhooksError::ReservedWebhook(uuid).into()); + return Err(ReservedWebhook(uuid).into()); } let mut webhooks = index_scheduler.webhooks(); - webhooks.webhooks.remove(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?; + webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; index_scheduler.put_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::delete_webhook(), &req); From 4ec4710811a4b2c14470067430536823b02d1672 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 15:00:26 +0200 Subject: [PATCH 250/312] Improve logs --- crates/meilisearch/src/routes/webhooks.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 07f19c498..a362b8bb1 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -125,13 +125,10 @@ async fn get_webhooks( let results = webhooks .webhooks .into_iter() - .map(|(uuid, webhook)| WebhookWithMetadata { - uuid, - is_editable: uuid != Uuid::nil(), - webhook, - }) + .map(|(uuid, webhook)| WebhookWithMetadata::from(uuid, webhook)) .collect::>(); let results = WebhookResults { results }; + debug!(returns = ?results, "Get webhooks"); Ok(HttpResponse::Ok().json(results)) } @@ -248,7 +245,7 @@ fn patch_webhook_inner( tag = "Webhooks", security(("Bearer" = ["webhooks.get", "*.get", "*"])), responses( - (status = 200, description = "Webhook found", body = WebhookSettings, content_type = "application/json", example = json!({ + (status = 200, description = "Webhook found", body = WebhookWithMetadata, content_type = "application/json", example = json!({ "uuid": "550e8400-e29b-41d4-a716-446655440000", "url": "https://your.site/on-tasks-completed", "headers": { @@ -271,12 +268,10 @@ async fn get_webhook( let mut webhooks = index_scheduler.webhooks(); let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; - - Ok(HttpResponse::Ok().json(WebhookWithMetadata { - uuid, - is_editable: uuid != Uuid::nil(), - webhook, - })) + let webhook = WebhookWithMetadata::from(uuid, webhook); + + debug!(returns = ?webhook, "Get webhook"); + Ok(HttpResponse::Ok().json(webhook)) } #[utoipa::path( From 737ad3ec1908e8c5f58baa26483d3b35ecf49f40 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 15:00:45 +0200 Subject: [PATCH 251/312] Add new api key actions --- crates/meilisearch-auth/src/store.rs | 8 ++++++++ crates/meilisearch-types/src/keys.rs | 21 ++++++++++++++++++++- crates/meilisearch/src/routes/webhooks.rs | 6 +++--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch-auth/src/store.rs b/crates/meilisearch-auth/src/store.rs index eb2170f08..470379e06 100644 --- a/crates/meilisearch-auth/src/store.rs +++ b/crates/meilisearch-auth/src/store.rs @@ -137,6 +137,14 @@ impl HeedAuthStore { Action::ChatsSettingsAll => { actions.extend([Action::ChatsSettingsGet, Action::ChatsSettingsUpdate]); } + Action::WebhooksAll => { + actions.extend([ + Action::WebhooksGet, + Action::WebhooksUpdate, + Action::WebhooksDelete, + Action::WebhooksCreate, + ]); + } other => { actions.insert(*other); } diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 2eddb9547..6763e2661 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -371,6 +371,15 @@ pub enum Action { #[serde(rename = "webhooks.update")] #[deserr(rename = "webhooks.update")] WebhooksUpdate, + #[serde(rename = "webhooks.delete")] + #[deserr(rename = "webhooks.delete")] + WebhooksDelete, + #[serde(rename = "webhooks.create")] + #[deserr(rename = "webhooks.create")] + WebhooksCreate, + #[serde(rename = "webhooks.*")] + #[deserr(rename = "webhooks.*")] + WebhooksAll, } impl Action { @@ -436,7 +445,9 @@ impl Action { match self { // Any action that expands to others must return false, as it wouldn't be able to expand recursively. All | AllGet | DocumentsAll | IndexesAll | ChatsAll | TasksAll | SettingsAll - | StatsAll | MetricsAll | DumpsAll | SnapshotsAll | ChatsSettingsAll => false, + | StatsAll | MetricsAll | DumpsAll | SnapshotsAll | ChatsSettingsAll | WebhooksAll => { + false + } Search => true, DocumentsAdd => false, @@ -473,6 +484,8 @@ impl Action { ChatsSettingsUpdate => false, WebhooksGet => true, WebhooksUpdate => false, + WebhooksDelete => false, + WebhooksCreate => false, } } @@ -535,6 +548,9 @@ pub mod actions { pub const WEBHOOKS_GET: u8 = WebhooksGet.repr(); pub const WEBHOOKS_UPDATE: u8 = WebhooksUpdate.repr(); + pub const WEBHOOKS_DELETE: u8 = WebhooksDelete.repr(); + pub const WEBHOOKS_CREATE: u8 = WebhooksCreate.repr(); + pub const WEBHOOKS_ALL: u8 = WebhooksAll.repr(); } #[cfg(test)] @@ -592,6 +608,9 @@ pub(crate) mod test { assert!(AllGet.repr() == 44 && ALL_GET == 44); assert!(WebhooksGet.repr() == 45 && WEBHOOKS_GET == 45); assert!(WebhooksUpdate.repr() == 46 && WEBHOOKS_UPDATE == 46); + assert!(WebhooksDelete.repr() == 47 && WEBHOOKS_DELETE == 47); + assert!(WebhooksCreate.repr() == 48 && WEBHOOKS_CREATE == 48); + assert!(WebhooksAll.repr() == 49 && WEBHOOKS_ALL == 49); } #[test] diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index a362b8bb1..67036e0b5 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -269,7 +269,7 @@ async fn get_webhook( let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; let webhook = WebhookWithMetadata::from(uuid, webhook); - + debug!(returns = ?webhook, "Get webhook"); Ok(HttpResponse::Ok().json(webhook)) } @@ -294,7 +294,7 @@ async fn get_webhook( ) )] async fn post_webhook( - index_scheduler: GuardedData, Data>, + index_scheduler: GuardedData, Data>, webhook_settings: AwebJson, req: HttpRequest, analytics: Data, @@ -400,7 +400,7 @@ async fn patch_webhook( ) )] async fn delete_webhook( - index_scheduler: GuardedData, Data>, + index_scheduler: GuardedData, Data>, uuid: Path, req: HttpRequest, analytics: Data, From 8dfebbb3e7b5b1f50ea3bc0e42ad6fc793c5ad40 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 15:37:12 +0200 Subject: [PATCH 252/312] Fix tests --- crates/index-scheduler/src/lib.rs | 63 ++++------------------- crates/meilisearch/tests/tasks/webhook.rs | 2 - 2 files changed, 11 insertions(+), 54 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 8d7617b6c..84cb0f752 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -820,58 +820,17 @@ impl IndexScheduler { let rtxn = self.env.read_txn()?; - let task_reader = TaskReader { - rtxn: &rtxn, - index_scheduler: self, - tasks: &mut updated.into_iter(), - buffer: Vec::with_capacity(800), // on average a task is around ~600 bytes - written: 0, - }; - - enum EitherRead<'a, T: Read> { - Other(Option), - Data(&'a [u8]), - } - - impl EitherRead<'_, T> { - /// A clone that works only once for the Other variant. - fn clone(&mut self) -> Self { - match self { - Self::Other(r) => { - let r = r.take(); - Self::Other(r) - } - Self::Data(arg0) => Self::Data(arg0), - } - } - } - - impl Read for EitherRead<'_, T> { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - match self { - EitherRead::Other(Some(reader)) => reader.read(buf), - EitherRead::Other(None) => { - Err(io::Error::new(io::ErrorKind::Other, "No reader available")) - } - EitherRead::Data(data) => data.read(buf), - } - } - } - - let mut reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); - - // When there is more than one webhook, cache the data in memory - let mut data; - let mut reader = match webhooks.webhooks.len() { - 1 => EitherRead::Other(Some(reader)), - _ => { - data = Vec::new(); - reader.read_to_end(&mut data)?; - EitherRead::Data(&data) - } - }; - for (uuid, Webhook { url, headers }) in webhooks.webhooks.iter() { + let task_reader = TaskReader { + rtxn: &rtxn, + index_scheduler: self, + tasks: &mut updated.into_iter(), + buffer: Vec::with_capacity(800), // on average a task is around ~600 bytes + written: 0, + }; + + let reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); + let mut request = ureq::post(url) .timeout(Duration::from_secs(30)) .set("Content-Encoding", "gzip") @@ -880,7 +839,7 @@ impl IndexScheduler { request = request.set(header_name, header_value); } - if let Err(e) = request.send(reader.clone()) { + if let Err(e) = request.send(reader) { tracing::error!("While sending data to the webhook {uuid}: {e}"); } } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index f8fcd8ff9..beef2f5c1 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -99,7 +99,6 @@ async fn cli_only() { } #[actix_web::test] -#[ignore = "Broken"] async fn single_receives_data() { let WebhookHandle { server_handle, url, mut receiver } = create_webhook_server().await; @@ -166,7 +165,6 @@ async fn single_receives_data() { } #[actix_web::test] -#[ignore = "Broken"] async fn multiple_receive_data() { let server = Server::new().await; From 69c59d3de3c709aafa66f751960e15fa5add6508 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 15:43:37 +0200 Subject: [PATCH 253/312] Update security in utoipa --- crates/meilisearch/src/routes/webhooks.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 67036e0b5..9dc448407 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -89,7 +89,7 @@ pub(super) struct WebhookResults { get, path = "", tag = "Webhooks", - security(("Bearer" = ["webhooks.get", "*.get", "*"])), + security(("Bearer" = ["webhooks.get", "webhooks.*", "*.get", "*"])), responses( (status = OK, description = "Webhooks are returned", body = WebhookResults, content_type = "application/json", example = json!({ "results": [ @@ -243,7 +243,7 @@ fn patch_webhook_inner( get, path = "/{uuid}", tag = "Webhooks", - security(("Bearer" = ["webhooks.get", "*.get", "*"])), + security(("Bearer" = ["webhooks.get", "webhooks.*", "*.get", "*"])), responses( (status = 200, description = "Webhook found", body = WebhookWithMetadata, content_type = "application/json", example = json!({ "uuid": "550e8400-e29b-41d4-a716-446655440000", @@ -279,7 +279,7 @@ async fn get_webhook( path = "", tag = "Webhooks", request_body = WebhookSettings, - security(("Bearer" = ["webhooks.update", "*"])), + security(("Bearer" = ["webhooks.create", "webhooks.*", "*"])), responses( (status = 201, description = "Webhook created successfully", body = WebhookWithMetadata, content_type = "application/json", example = json!({ "uuid": "550e8400-e29b-41d4-a716-446655440000", @@ -335,7 +335,7 @@ async fn post_webhook( path = "/{uuid}", tag = "Webhooks", request_body = WebhookSettings, - security(("Bearer" = ["webhooks.update", "*"])), + security(("Bearer" = ["webhooks.update", "webhooks.*", "*"])), responses( (status = 200, description = "Webhook updated successfully", body = WebhookWithMetadata, content_type = "application/json", example = json!({ "uuid": "550e8400-e29b-41d4-a716-446655440000", @@ -389,7 +389,7 @@ async fn patch_webhook( delete, path = "/{uuid}", tag = "Webhooks", - security(("Bearer" = ["webhooks.update", "*"])), + security(("Bearer" = ["webhooks.delete", "webhooks.*", "*"])), responses( (status = 204, description = "Webhook deleted successfully"), (status = 404, description = "Webhook not found", body = ResponseError, content_type = "application/json"), From 1754745c426ec8a1efe46ffd1f845c4d801cf3bb Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 16:26:20 +0200 Subject: [PATCH 254/312] Add URL and header validity checks --- crates/meilisearch/src/routes/webhooks.rs | 51 +++++++++++++++++++---- crates/meilisearch/tests/tasks/webhook.rs | 51 +++++++++++++++++++++++ 2 files changed, 94 insertions(+), 8 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 9dc448407..18edfb63c 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -1,5 +1,9 @@ use std::collections::BTreeMap; +use actix_http::header::{ + HeaderName, HeaderValue, InvalidHeaderName as ActixInvalidHeaderName, + InvalidHeaderValue as ActixInvalidHeaderValue, +}; use actix_web::web::{self, Data, Path}; use actix_web::{HttpRequest, HttpResponse}; use deserr::actix_web::AwebJson; @@ -13,6 +17,7 @@ use meilisearch_types::milli::update::Setting; use meilisearch_types::webhooks::Webhook; use serde::Serialize; use tracing::debug; +use url::Url; use utoipa::{OpenApi, ToSchema}; use uuid::Uuid; @@ -184,6 +189,12 @@ enum WebhooksError { ReservedWebhook(Uuid), #[error("Webhook `{0}` not found.")] WebhookNotFound(Uuid), + #[error("Invalid header name `{0}`: {1}")] + InvalidHeaderName(String, ActixInvalidHeaderName), + #[error("Invalid header value `{0}`: {1}")] + InvalidHeaderValue(String, ActixInvalidHeaderValue), + #[error("Invalid URL `{0}`: {1}")] + InvalidUrl(String, url::ParseError), } impl ErrorCode for WebhooksError { @@ -194,6 +205,9 @@ impl ErrorCode for WebhooksError { TooManyHeaders(_) => meilisearch_types::error::Code::InvalidWebhooksHeaders, ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, WebhookNotFound(_) => meilisearch_types::error::Code::WebhookNotFound, + InvalidHeaderName(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, + InvalidHeaderValue(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, + InvalidUrl(_, _) => meilisearch_types::error::Code::InvalidWebhooksUrl, } } } @@ -239,6 +253,32 @@ fn patch_webhook_inner( Ok(Webhook { url, headers }) } +fn check_changed(uuid: Uuid, webhook: &Webhook) -> Result<(), WebhooksError> { + if uuid.is_nil() { + return Err(ReservedWebhook(uuid)); + } + + if webhook.url.is_empty() { + return Err(MissingUrl(uuid)); + } + + if webhook.headers.len() > 200 { + return Err(TooManyHeaders(uuid)); + } + + for (header, value) in &webhook.headers { + HeaderName::from_bytes(header.as_bytes()) + .map_err(|e| InvalidHeaderName(header.to_owned(), e))?; + HeaderValue::from_str(value).map_err(|e| InvalidHeaderValue(header.to_owned(), e))?; + } + + if let Err(e) = Url::parse(&webhook.url) { + return Err(InvalidUrl(webhook.url.to_owned(), e)); + } + + Ok(()) +} + #[utoipa::path( get, path = "/{uuid}", @@ -320,6 +360,8 @@ async fn post_webhook( .map(|h| h.into_iter().map(|(k, v)| (k, v.set().unwrap_or_default())).collect()) .unwrap_or_default(), }; + + check_changed(uuid, &webhook)?; webhooks.webhooks.insert(uuid, webhook.clone()); index_scheduler.put_webhooks(webhooks)?; @@ -363,18 +405,11 @@ async fn patch_webhook( let webhook_settings = webhook_settings.into_inner(); debug!(parameters = ?(uuid, &webhook_settings), "Patch webhook"); - if uuid.is_nil() { - return Err(ReservedWebhook(uuid).into()); - } - let mut webhooks = index_scheduler.webhooks(); let old_webhook = webhooks.webhooks.remove(&uuid); let webhook = patch_webhook_inner(&uuid, old_webhook, webhook_settings)?; - if webhook.headers.len() > 200 { - return Err(TooManyHeaders(uuid).into()); - } - + check_changed(uuid, &webhook)?; webhooks.webhooks.insert(uuid, webhook.clone()); index_scheduler.put_webhooks(webhooks)?; diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index beef2f5c1..a1029da6d 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -469,3 +469,54 @@ async fn patch() { } "#); } + +#[actix_web::test] +async fn invalid_url_and_headers() { + let server = Server::new().await; + + // Test invalid URL format + let (value, code) = server.create_webhook(json!({ "url": "not-a-valid-url" })).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid URL `not-a-valid-url`: relative URL without a base", + "code": "invalid_webhooks_url", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + } + "#); + + // Test invalid header name (containing spaces) + let (value, code) = server + .create_webhook(json!({ + "url": "https://example.com/hook", + "headers": { "invalid header name": "value" } + })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid header name `invalid header name`: invalid HTTP header name", + "code": "invalid_webhooks_headers", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" + } + "#); + + // Test invalid header value (containing control characters) + let (value, code) = server + .create_webhook(json!({ + "url": "https://example.com/hook", + "headers": { "authorization": "token\nwith\nnewlines" } + })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid header value `authorization`: failed to parse header value", + "code": "invalid_webhooks_headers", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" + } + "#); +} From 454f8b36f45367c9e92df85e8bbb30d67ab77f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Renault?= Date: Mon, 4 Aug 2025 16:14:33 +0200 Subject: [PATCH 255/312] Make clippy happy --- crates/meilisearch/tests/common/mod.rs | 16 ++++++---------- .../meilisearch/tests/documents/get_documents.rs | 2 +- crates/meilisearch/tests/vector/fragments.rs | 2 +- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/crates/meilisearch/tests/common/mod.rs b/crates/meilisearch/tests/common/mod.rs index d6df761d4..03b1271f1 100644 --- a/crates/meilisearch/tests/common/mod.rs +++ b/crates/meilisearch/tests/common/mod.rs @@ -3,10 +3,8 @@ pub mod index; pub mod server; pub mod service; -use std::{ - collections::BTreeMap, - fmt::{self, Display}, -}; +use std::collections::BTreeMap; +use std::fmt::{self, Display}; use actix_http::StatusCode; #[allow(unused)] @@ -17,10 +15,8 @@ use serde::{Deserialize, Serialize}; #[allow(unused)] pub use server::{default_settings, Server}; use tokio::sync::OnceCell; -use wiremock::{ - matchers::{method, path}, - Mock, MockServer, Request, ResponseTemplate, -}; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, Request, ResponseTemplate}; use crate::common::index::Index; @@ -618,7 +614,7 @@ pub async fn init_fragments_index() -> (Server, String, crate::common::Va let (value, code) = index.add_documents(documents, None).await; assert_eq!(code, StatusCode::ACCEPTED); - let _task = index.wait_task(value.uid()).await.succeeded(); + let _task = server.wait_task(value.uid()).await.succeeded(); let uid = index.uid.clone(); (server, uid, settings) @@ -683,7 +679,7 @@ pub async fn init_fragments_index_composite() -> (Server, String, crate:: let (value, code) = index.add_documents(documents, None).await; assert_eq!(code, StatusCode::ACCEPTED); - index.wait_task(value.uid()).await.succeeded(); + server.wait_task(value.uid()).await.succeeded(); let uid = index.uid.clone(); (server, uid, settings) diff --git a/crates/meilisearch/tests/documents/get_documents.rs b/crates/meilisearch/tests/documents/get_documents.rs index 1209b74f0..b3c68351f 100644 --- a/crates/meilisearch/tests/documents/get_documents.rs +++ b/crates/meilisearch/tests/documents/get_documents.rs @@ -87,7 +87,7 @@ async fn get_document() { async fn get_document_sorted() { let server = Server::new_shared(); let index = server.unique_index(); - index.load_test_set().await; + index.load_test_set(server).await; let (task, _status_code) = index.update_settings_sortable_attributes(json!(["age", "email", "gender", "name"])).await; diff --git a/crates/meilisearch/tests/vector/fragments.rs b/crates/meilisearch/tests/vector/fragments.rs index a994eb64c..81c2e3a55 100644 --- a/crates/meilisearch/tests/vector/fragments.rs +++ b/crates/meilisearch/tests/vector/fragments.rs @@ -149,7 +149,7 @@ async fn replace_document() { let (value, code) = index.add_documents(documents, None).await; snapshot!(code, @"202 Accepted"); - index.wait_task(value.uid()).await.succeeded(); + server.wait_task(value.uid()).await.succeeded(); // Make sure kefir now has 2 vectors let (documents, code) = index From 3b0f576d56bb99c38ea03ce25c0b6b745a7dc2d9 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 16:38:00 +0200 Subject: [PATCH 256/312] Improve invalid uuid error message --- crates/meilisearch-types/src/error.rs | 1 + crates/meilisearch/src/routes/webhooks.rs | 16 +++++---- crates/meilisearch/tests/tasks/webhook.rs | 42 +++++++++++++++++++++++ 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 3916012c1..e14a06909 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -424,6 +424,7 @@ InvalidWebhooks , InvalidRequest , BAD_REQU InvalidWebhooksUrl , InvalidRequest , BAD_REQUEST ; InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST ; ReservedWebhook , InvalidRequest , BAD_REQUEST ; +InvalidWebhookUuid , InvalidRequest , BAD_REQUEST ; WebhookNotFound , InvalidRequest , NOT_FOUND } diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 18edfb63c..2454b624b 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::str::FromStr; use actix_http::header::{ HeaderName, HeaderValue, InvalidHeaderName as ActixInvalidHeaderName, @@ -195,6 +196,8 @@ enum WebhooksError { InvalidHeaderValue(String, ActixInvalidHeaderValue), #[error("Invalid URL `{0}`: {1}")] InvalidUrl(String, url::ParseError), + #[error("Invalid UUID: {0}")] + InvalidUuid(uuid::Error), } impl ErrorCode for WebhooksError { @@ -208,6 +211,7 @@ impl ErrorCode for WebhooksError { InvalidHeaderName(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, InvalidHeaderValue(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, InvalidUrl(_, _) => meilisearch_types::error::Code::InvalidWebhooksUrl, + InvalidUuid(_) => meilisearch_types::error::Code::InvalidWebhookUuid, } } } @@ -302,9 +306,9 @@ fn check_changed(uuid: Uuid, webhook: &Webhook) -> Result<(), WebhooksError> { )] async fn get_webhook( index_scheduler: GuardedData, Data>, - uuid: Path, + uuid: Path, ) -> Result { - let uuid = uuid.into_inner(); + let uuid = Uuid::from_str(&uuid.into_inner()).map_err(InvalidUuid)?; let mut webhooks = index_scheduler.webhooks(); let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; @@ -396,12 +400,12 @@ async fn post_webhook( )] async fn patch_webhook( index_scheduler: GuardedData, Data>, - uuid: Path, + uuid: Path, webhook_settings: AwebJson, req: HttpRequest, analytics: Data, ) -> Result { - let uuid = uuid.into_inner(); + let uuid = Uuid::from_str(&uuid.into_inner()).map_err(InvalidUuid)?; let webhook_settings = webhook_settings.into_inner(); debug!(parameters = ?(uuid, &webhook_settings), "Patch webhook"); @@ -436,11 +440,11 @@ async fn patch_webhook( )] async fn delete_webhook( index_scheduler: GuardedData, Data>, - uuid: Path, + uuid: Path, req: HttpRequest, analytics: Data, ) -> Result { - let uuid = uuid.into_inner(); + let uuid = Uuid::from_str(&uuid.into_inner()).map_err(InvalidUuid)?; debug!(parameters = ?uuid, "Delete webhook"); if uuid.is_nil() { diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index a1029da6d..155312b9d 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -520,3 +520,45 @@ async fn invalid_url_and_headers() { } "#); } + +#[actix_web::test] +async fn invalid_uuid() { + let server = Server::new().await; + + // Test get webhook with invalid UUID + let (value, code) = server.get_webhook("invalid-uuid").await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid UUID: invalid character: expected an optional prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `i` at 1", + "code": "invalid_webhook_uuid", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhook_uuid" + } + "#); + + // Test update webhook with invalid UUID + let (value, code) = + server.patch_webhook("invalid-uuid", json!({ "url": "https://example.com/hook" })).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid UUID: invalid character: expected an optional prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `i` at 1", + "code": "invalid_webhook_uuid", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhook_uuid" + } + "#); + + // Test delete webhook with invalid UUID + let (value, code) = server.delete_webhook("invalid-uuid").await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid UUID: invalid character: expected an optional prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `i` at 1", + "code": "invalid_webhook_uuid", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhook_uuid" + } + "#); +} From 3b26d64a5d1504a324b5ca2606a227fe35409913 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 16:39:34 +0200 Subject: [PATCH 257/312] Edit reserved webhook message --- crates/meilisearch/src/routes/webhooks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 2454b624b..fc03f43d0 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -186,7 +186,7 @@ enum WebhooksError { TooManyWebhooks, #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] TooManyHeaders(Uuid), - #[error("Cannot edit webhook `{0}`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.")] + #[error("Cannot edit webhook `{0}`. The webhook defined from the command line cannot be modified using the API.")] ReservedWebhook(Uuid), #[error("Webhook `{0}` not found.")] WebhookNotFound(Uuid), From ddfcacbb621e6ba90484be4462c550f305c4b7e3 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 16:53:41 +0200 Subject: [PATCH 258/312] Add nice error message for users trying to set uuid or isEditable --- crates/meilisearch-types/src/error.rs | 4 +- crates/meilisearch/src/routes/webhooks.rs | 29 ++++++-- crates/meilisearch/tests/tasks/webhook.rs | 86 ++++++++++++++++++++++- 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index e14a06909..e25ce61a4 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -425,7 +425,9 @@ InvalidWebhooksUrl , InvalidRequest , BAD_REQU InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST ; ReservedWebhook , InvalidRequest , BAD_REQUEST ; InvalidWebhookUuid , InvalidRequest , BAD_REQUEST ; -WebhookNotFound , InvalidRequest , NOT_FOUND +WebhookNotFound , InvalidRequest , NOT_FOUND ; +ImmutableWebhookUuid , InvalidRequest , BAD_REQUEST ; +ImmutableWebhookIsEditable , InvalidRequest , BAD_REQUEST } impl ErrorCode for JoinError { diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index fc03f43d0..610e28271 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -7,12 +7,15 @@ use actix_http::header::{ }; use actix_web::web::{self, Data, Path}; use actix_web::{HttpRequest, HttpResponse}; +use core::convert::Infallible; use deserr::actix_web::AwebJson; -use deserr::Deserr; +use deserr::{DeserializeError, Deserr, ValuePointerRef}; use index_scheduler::IndexScheduler; -use meilisearch_types::deserr::DeserrJsonError; -use meilisearch_types::error::deserr_codes::{InvalidWebhooksHeaders, InvalidWebhooksUrl}; -use meilisearch_types::error::{ErrorCode, ResponseError}; +use meilisearch_types::deserr::{immutable_field_error, DeserrJsonError}; +use meilisearch_types::error::deserr_codes::{ + BadRequest, InvalidWebhooksHeaders, InvalidWebhooksUrl, +}; +use meilisearch_types::error::{Code, ErrorCode, ResponseError}; use meilisearch_types::keys::actions; use meilisearch_types::milli::update::Setting; use meilisearch_types::webhooks::Webhook; @@ -54,7 +57,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { } #[derive(Debug, Deserr, ToSchema)] -#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields = deny_immutable_fields_webhook)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] pub(super) struct WebhookSettings { @@ -68,6 +71,22 @@ pub(super) struct WebhookSettings { headers: Setting>>, } +fn deny_immutable_fields_webhook( + field: &str, + accepted: &[&str], + location: ValuePointerRef, +) -> DeserrJsonError { + match field { + "uuid" => immutable_field_error(field, accepted, Code::ImmutableWebhookUuid), + "isEditable" => immutable_field_error(field, accepted, Code::ImmutableWebhookIsEditable), + _ => deserr::take_cf_content(DeserrJsonError::::error::( + None, + deserr::ErrorKind::UnknownKey { key: field, accepted }, + location, + )), + } +} + #[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 155312b9d..03f732f0d 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -273,7 +273,7 @@ async fn reserved_names() { snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Cannot edit webhook `[uuid]`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", + "message": "Cannot edit webhook `[uuid]`. The webhook defined from the command line cannot be modified using the API.", "code": "reserved_webhook", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#reserved_webhook" @@ -284,7 +284,7 @@ async fn reserved_names() { snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Cannot edit webhook `[uuid]`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", + "message": "Cannot edit webhook `[uuid]`. The webhook defined from the command line cannot be modified using the API.", "code": "reserved_webhook", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#reserved_webhook" @@ -562,3 +562,85 @@ async fn invalid_uuid() { } "#); } + +#[actix_web::test] +async fn forbidden_fields() { + let server = Server::new().await; + + // Test creating webhook with uuid field + let custom_uuid = Uuid::new_v4(); + let (value, code) = server + .create_webhook(json!({ + "url": "https://example.com/hook", + "uuid": custom_uuid.to_string(), + "headers": { "authorization": "TOKEN" } + })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Immutable field `uuid`: expected one of `url`, `headers`", + "code": "immutable_webhook_uuid", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#immutable_webhook_uuid" + } + "#); + + // Test creating webhook with isEditable field + let (value, code) = server + .create_webhook(json!({ + "url": "https://example.com/hook2", + "isEditable": false, + "headers": { "authorization": "TOKEN" } + })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Immutable field `isEditable`: expected one of `url`, `headers`", + "code": "immutable_webhook_is_editable", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#immutable_webhook_is_editable" + } + "#); + + // Test patching webhook with uuid field + let (value, code) = server + .patch_webhook( + "uuid-whatever", + json!({ + "uuid": Uuid::new_v4(), + "headers": { "new-header": "value" } + }), + ) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Immutable field `uuid`: expected one of `url`, `headers`", + "code": "immutable_webhook_uuid", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#immutable_webhook_uuid" + } + "#); + + // Test patching webhook with isEditable field + let (value, code) = server + .patch_webhook( + "uuid-whatever", + json!({ + "isEditable": false, + "headers": { "another-header": "value" } + }), + ) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "message": "Immutable field `isEditable`: expected one of `url`, `headers`", + "code": "immutable_webhook_is_editable", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#immutable_webhook_is_editable" + } + "#); +} From 7251cccd0310b17575786f695a9848600d6ebbaa Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 17:13:05 +0200 Subject: [PATCH 259/312] Make notify_webhooks execute in its own thread --- crates/index-scheduler/src/lib.rs | 10 ++-------- crates/index-scheduler/src/scheduler/mod.rs | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 84cb0f752..9af74a3e5 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -766,14 +766,8 @@ impl IndexScheduler { Ok(()) } - /// Once the tasks changes have been committed we must send all the tasks that were updated to our webhook if there is one. - fn notify_webhook(&self, updated: &RoaringBitmap) -> Result<()> { - let webhooks = self.cached_webhooks.read().unwrap_or_else(|poisoned| poisoned.into_inner()); - if webhooks.webhooks.is_empty() { - return Ok(()); - } - let webhooks = Webhooks::clone(&*webhooks); - + /// Once the tasks changes have been committed we must send all the tasks that were updated to our webhooks + fn notify_webhooks(&self, webhooks: Webhooks, updated: &RoaringBitmap) -> Result<()> { struct TaskReader<'a, 'b> { rtxn: &'a RoTxn<'a>, index_scheduler: &'a IndexScheduler, diff --git a/crates/index-scheduler/src/scheduler/mod.rs b/crates/index-scheduler/src/scheduler/mod.rs index 5ac591143..b5acf7582 100644 --- a/crates/index-scheduler/src/scheduler/mod.rs +++ b/crates/index-scheduler/src/scheduler/mod.rs @@ -26,6 +26,7 @@ use meilisearch_types::error::ResponseError; use meilisearch_types::heed::{Env, WithoutTls}; use meilisearch_types::milli; use meilisearch_types::tasks::Status; +use meilisearch_types::webhooks::Webhooks; use process_batch::ProcessBatchInfo; use rayon::current_num_threads; use rayon::iter::{IntoParallelIterator, ParallelIterator}; @@ -446,8 +447,17 @@ impl IndexScheduler { Ok(()) })?; - // We shouldn't crash the tick function if we can't send data to the webhook. - let _ = self.notify_webhook(&ids); + // We shouldn't crash the tick function if we can't send data to the webhooks + let webhooks = self.cached_webhooks.read().unwrap_or_else(|p| p.into_inner()); + if !webhooks.webhooks.is_empty() { + let webhooks = Webhooks::clone(&*webhooks); + let cloned_index_scheduler = self.private_clone(); + std::thread::spawn(move || { + if let Err(e) = cloned_index_scheduler.notify_webhooks(webhooks, &ids) { + tracing::error!("Failure to notify webhooks: {e}"); + } + }); + } #[cfg(test)] self.breakpoint(crate::test_utils::Breakpoint::AfterProcessing); From fc814b7537cfb8c1a8abdeb2835f1ebe0f317e0b Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:25:14 +0200 Subject: [PATCH 260/312] Apply review suggestion --- crates/milli/src/search/facet/filter.rs | 78 +++++++++++++------------ 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index 955e75753..af4a77814 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -1,3 +1,4 @@ +use std::borrow::Cow; use std::collections::BTreeSet; use std::fmt::{Debug, Display}; use std::ops::Bound::{self, Excluded, Included, Unbounded}; @@ -14,9 +15,7 @@ use super::facet_range_search; use crate::constants::RESERVED_GEO_FIELD_NAME; use crate::error::{Error, UserError}; use crate::filterable_attributes_rules::{filtered_matching_patterns, matching_features}; -use crate::heed_codec::facet::{ - FacetGroupKey, FacetGroupKeyCodec, FacetGroupValue, FacetGroupValueCodec, -}; +use crate::heed_codec::facet::{FacetGroupKey, FacetGroupKeyCodec, FacetGroupValueCodec}; use crate::index::db_name::FACET_ID_STRING_DOCIDS; use crate::search::facet::facet_range_search::find_docids_of_facet_within_bounds; use crate::{ @@ -427,44 +426,51 @@ impl<'a> Filter<'a> { // It's used as a fallback. let value = crate::normalize_facet(word.value()); - let mut value2 = value.as_bytes().to_owned(); - if let Some(last) = value2.last_mut() { - if *last != 255 { - *last += 1; - if let Ok(value2) = String::from_utf8(value2) { - // The idea here is that "STARTS WITH baba" is the same as "baba <= value < babb". - // We just increase the last letter to find the upper bound. - // The result could be invalid utf8, so it can fallback. - let mut docids = RoaringBitmap::new(); - find_docids_of_facet_within_bounds( - rtxn, - strings_db, - field_id, - &Included(&value), - &Excluded(&value2), - universe, - &mut docids, - )?; - return Ok(docids); - } + let last = match value2.last_mut() { + Some(last) => last, + None => { + // The prefix is empty, so all documents that have the field will match. + return index + .exists_faceted_documents_ids(rtxn, field_id) + .map_err(|e| e.into()); + } + }; + + if *last == 255 { + // The prefix is invalid utf8, so no documents will match anyway + return Ok(RoaringBitmap::new()); + } + *last += 1; + + // This is very similar to `heed::Bytes` but its `EItem` is `&[u8]` instead of `[u8]` + struct BytesRef; + impl<'a> BytesEncode<'a> for BytesRef { + type EItem = &'a [u8]; + + fn bytes_encode( + item: &'a Self::EItem, + ) -> std::result::Result, heed::BoxedError> { + Ok(Cow::Borrowed(item)) } } - let base = FacetGroupKey { field_id, level: 0, left_bound: value.as_str() }; - let docids = strings_db - .prefix_iter(rtxn, &base)? - .map(|result| -> Result { - match result { - Ok((_facet_group_key, FacetGroupValue { bitmap, .. })) => Ok(bitmap), - Err(_e) => Err(InternalError::from(SerializationError::Decoding { - db_name: Some(FACET_ID_STRING_DOCIDS), - }) - .into()), - } - }) - .union()?; + // The idea here is that "STARTS WITH baba" is the same as "baba <= value < babb". + // We just incremented the last letter to find the upper bound. + // The upper bound may not be valid utf8, but lmdb doesn't care as it works over bytes. + let mut docids = RoaringBitmap::new(); + let bytes_db = + index.facet_id_string_docids.remap_key_type::>(); + find_docids_of_facet_within_bounds::( + rtxn, + bytes_db, + field_id, + &Included(value.as_bytes()), + &Excluded(value2.as_slice()), + universe, + &mut docids, + )?; return Ok(docids); } From afb367c7f4a8003c0d1c45b76c3f8a201e498276 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:29:39 +0200 Subject: [PATCH 261/312] Update old comment --- crates/milli/src/search/facet/filter.rs | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index af4a77814..4d1e51767 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -416,14 +416,9 @@ impl<'a> Filter<'a> { return Ok(docids); } Condition::StartsWith { keyword: _, word } => { - // There are two algorithms: - // - // - The first one is recursive over levels. - // This is more efficient when the prefix is common among many values. - // - // - The second one looks directly at level 0 of the facet group database. - // This pessimistic approach is more efficient when the value is unique. - // It's used as a fallback. + // The idea here is that "STARTS WITH baba" is the same as "baba <= value < babb". + // We just incremented the last letter to find the upper bound. + // The upper bound may not be valid utf8, but lmdb doesn't care as it works over bytes. let value = crate::normalize_facet(word.value()); let mut value2 = value.as_bytes().to_owned(); @@ -456,9 +451,6 @@ impl<'a> Filter<'a> { } } - // The idea here is that "STARTS WITH baba" is the same as "baba <= value < babb". - // We just incremented the last letter to find the upper bound. - // The upper bound may not be valid utf8, but lmdb doesn't care as it works over bytes. let mut docids = RoaringBitmap::new(); let bytes_db = index.facet_id_string_docids.remap_key_type::>(); From d340013d8b28ab0d3bb289cc0e2c6d742481ed88 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:35:12 +0200 Subject: [PATCH 262/312] Change error name --- crates/meilisearch-types/src/error.rs | 2 +- crates/meilisearch/src/routes/webhooks.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index e25ce61a4..bb486726a 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -423,7 +423,7 @@ InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQU InvalidWebhooks , InvalidRequest , BAD_REQUEST ; InvalidWebhooksUrl , InvalidRequest , BAD_REQUEST ; InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST ; -ReservedWebhook , InvalidRequest , BAD_REQUEST ; +ImmutableWebhook , InvalidRequest , BAD_REQUEST ; InvalidWebhookUuid , InvalidRequest , BAD_REQUEST ; WebhookNotFound , InvalidRequest , NOT_FOUND ; ImmutableWebhookUuid , InvalidRequest , BAD_REQUEST ; diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 610e28271..65aff4179 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -205,8 +205,8 @@ enum WebhooksError { TooManyWebhooks, #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] TooManyHeaders(Uuid), - #[error("Cannot edit webhook `{0}`. The webhook defined from the command line cannot be modified using the API.")] - ReservedWebhook(Uuid), + #[error("Webhook `{0}` is immutable. The webhook defined from the command line cannot be modified using the API.")] + ImmutableWebhook(Uuid), #[error("Webhook `{0}` not found.")] WebhookNotFound(Uuid), #[error("Invalid header name `{0}`: {1}")] @@ -225,7 +225,7 @@ impl ErrorCode for WebhooksError { MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhooksUrl, TooManyWebhooks => meilisearch_types::error::Code::InvalidWebhooks, TooManyHeaders(_) => meilisearch_types::error::Code::InvalidWebhooksHeaders, - ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, + ImmutableWebhook(_) => meilisearch_types::error::Code::ImmutableWebhook, WebhookNotFound(_) => meilisearch_types::error::Code::WebhookNotFound, InvalidHeaderName(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, InvalidHeaderValue(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, @@ -278,7 +278,7 @@ fn patch_webhook_inner( fn check_changed(uuid: Uuid, webhook: &Webhook) -> Result<(), WebhooksError> { if uuid.is_nil() { - return Err(ReservedWebhook(uuid)); + return Err(ImmutableWebhook(uuid)); } if webhook.url.is_empty() { @@ -467,7 +467,7 @@ async fn delete_webhook( debug!(parameters = ?uuid, "Delete webhook"); if uuid.is_nil() { - return Err(ReservedWebhook(uuid).into()); + return Err(ImmutableWebhook(uuid).into()); } let mut webhooks = index_scheduler.webhooks(); From 43c20bb3ed6903d709175b32f3639657a6d85c84 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:39:52 +0200 Subject: [PATCH 263/312] Add missing actions in from_repr Co-Authored-By: Thomas Campistron --- crates/meilisearch-types/src/keys.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 6763e2661..06f621e70 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -433,6 +433,9 @@ impl Action { ALL_GET => Some(Self::AllGet), WEBHOOKS_GET => Some(Self::WebhooksGet), WEBHOOKS_UPDATE => Some(Self::WebhooksUpdate), + WEBHOOKS_DELETE => Some(Self::WebhooksDelete), + WEBHOOKS_CREATE => Some(Self::WebhooksCreate), + WEBHOOKS_ALL => Some(Self::WebhooksAll), _otherwise => None, } } From 84651ffd7d63e239e585a4986da6c11db1c73b72 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:41:28 +0200 Subject: [PATCH 264/312] Remove hardcoded buffer size Co-Authored-By: Thomas Campistron --- crates/index-scheduler/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 9af74a3e5..c18efeaf2 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -819,7 +819,7 @@ impl IndexScheduler { rtxn: &rtxn, index_scheduler: self, tasks: &mut updated.into_iter(), - buffer: Vec::with_capacity(800), // on average a task is around ~600 bytes + buffer: Vec::with_capacity(page_size::get()), written: 0, }; From 8ef1a50086b4848b3fe310eef9861c1dad954a56 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:42:39 +0200 Subject: [PATCH 265/312] Add hint Co-Authored-By: Thomas Campistron --- crates/meilisearch/src/routes/webhooks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 65aff4179..c97f2fc5f 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -203,7 +203,7 @@ enum WebhooksError { MissingUrl(Uuid), #[error("Defining too many webhooks would crush the server. Please limit the number of webhooks to 20. You may use a third-party proxy server to dispatch events to more than 20 endpoints.")] TooManyWebhooks, - #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] + #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200. Hint: To remove an already defined header set its value to `null`")] TooManyHeaders(Uuid), #[error("Webhook `{0}` is immutable. The webhook defined from the command line cannot be modified using the API.")] ImmutableWebhook(Uuid), From 386cf832856a6173544d2fec818cf1e085c9738f Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:48:39 +0200 Subject: [PATCH 266/312] Improve webhook settings --- crates/meilisearch/src/routes/webhooks.rs | 25 ++++++++--------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index c97f2fc5f..86847c4bc 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -61,10 +61,9 @@ pub fn configure(cfg: &mut web::ServiceConfig) { #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] pub(super) struct WebhookSettings { - #[schema(value_type = Option)] #[deserr(default, error = DeserrJsonError)] #[serde(default)] - url: Setting, + url: Option, #[schema(value_type = Option>, example = json!({"Authorization":"Bearer a-secret-token"}))] #[deserr(default, error = DeserrJsonError)] #[serde(default)] @@ -237,19 +236,14 @@ impl ErrorCode for WebhooksError { fn patch_webhook_inner( uuid: &Uuid, - old_webhook: Option, + old_webhook: Webhook, new_webhook: WebhookSettings, ) -> Result { - let (old_url, mut headers) = - old_webhook.map(|w| (Some(w.url), w.headers)).unwrap_or((None, BTreeMap::new())); + let Webhook { url: old_url, mut headers } = old_webhook; - let url = match new_webhook.url { - Setting::Set(url) => url, - Setting::NotSet => old_url.ok_or_else(|| MissingUrl(uuid.to_owned()))?, - Setting::Reset => return Err(MissingUrl(uuid.to_owned())), - }; + let url = new_webhook.url.unwrap_or(old_url); - let headers = match new_webhook.headers { + match new_webhook.headers { Setting::Set(new_headers) => { for (name, value) in new_headers { match value { @@ -263,10 +257,9 @@ fn patch_webhook_inner( } } } - headers } - Setting::NotSet => headers, - Setting::Reset => BTreeMap::new(), + Setting::Reset => headers.clear(), + Setting::NotSet => (), }; if headers.len() > 200 { @@ -376,7 +369,7 @@ async fn post_webhook( } let webhook = Webhook { - url: webhook_settings.url.set().ok_or(MissingUrl(uuid))?, + url: webhook_settings.url.ok_or(MissingUrl(uuid))?, headers: webhook_settings .headers .set() @@ -429,7 +422,7 @@ async fn patch_webhook( debug!(parameters = ?(uuid, &webhook_settings), "Patch webhook"); let mut webhooks = index_scheduler.webhooks(); - let old_webhook = webhooks.webhooks.remove(&uuid); + let old_webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; let webhook = patch_webhook_inner(&uuid, old_webhook, webhook_settings)?; check_changed(uuid, &webhook)?; From b2d157a74a7088ccd25679f5c78cdacc2b36dd5a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:49:21 +0200 Subject: [PATCH 267/312] Remove dbg Co-Authored-By: Thomas Campistron --- crates/meilisearch/src/routes/webhooks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 86847c4bc..996377240 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -364,7 +364,7 @@ async fn post_webhook( } let mut webhooks = index_scheduler.webhooks(); - if dbg!(webhooks.webhooks.len() >= 20) { + if webhooks.webhooks.len() >= 20 { return Err(TooManyWebhooks.into()); } From 6cb22966447b347a15439959f904858c9a6881c2 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 11:10:48 +0200 Subject: [PATCH 268/312] Update tests --- crates/meilisearch/tests/auth/api_keys.rs | 2 +- crates/meilisearch/tests/auth/errors.rs | 2 +- crates/meilisearch/tests/tasks/webhook.rs | 48 +++++++++++++---------- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/crates/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs index f16789add..8dca24ac4 100644 --- a/crates/meilisearch/tests/auth/api_keys.rs +++ b/crates/meilisearch/tests/auth/api_keys.rs @@ -421,7 +421,7 @@ async fn error_add_api_key_invalid_parameters_actions() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r#" { - "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`, `webhooks.get`, `webhooks.update`", + "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`, `webhooks.get`, `webhooks.update`, `webhooks.delete`, `webhooks.create`, `webhooks.*`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" diff --git a/crates/meilisearch/tests/auth/errors.rs b/crates/meilisearch/tests/auth/errors.rs index 6d3369144..2a40f4d2b 100644 --- a/crates/meilisearch/tests/auth/errors.rs +++ b/crates/meilisearch/tests/auth/errors.rs @@ -93,7 +93,7 @@ async fn create_api_key_bad_actions() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r#" { - "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`, `webhooks.get`, `webhooks.update`", + "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`, `webhooks.get`, `webhooks.update`, `webhooks.delete`, `webhooks.create`, `webhooks.*`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 03f732f0d..268a37e18 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -265,7 +265,14 @@ async fn cli_with_dumps() { #[actix_web::test] async fn reserved_names() { - let server = Server::new().await; + let db_path = tempfile::tempdir().unwrap(); + let server = Server::new_with_options(Opt { + task_webhook_url: Some(Url::parse("https://example-cli.com/").unwrap()), + task_webhook_authorization_header: Some(String::from("Bearer a-secret-token")), + ..default_settings(db_path.path()) + }) + .await + .unwrap(); let (value, code) = server .patch_webhook(Uuid::nil().to_string(), json!({ "url": "http://localhost:8080" })) @@ -273,10 +280,10 @@ async fn reserved_names() { snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Cannot edit webhook `[uuid]`. The webhook defined from the command line cannot be modified using the API.", - "code": "reserved_webhook", + "message": "Webhook `[uuid]` is immutable. The webhook defined from the command line cannot be modified using the API.", + "code": "immutable_webhook", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#reserved_webhook" + "link": "https://docs.meilisearch.com/errors#immutable_webhook" } "#); @@ -284,10 +291,10 @@ async fn reserved_names() { snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Cannot edit webhook `[uuid]`. The webhook defined from the command line cannot be modified using the API.", - "code": "reserved_webhook", + "message": "Webhook `[uuid]` is immutable. The webhook defined from the command line cannot be modified using the API.", + "code": "immutable_webhook", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#reserved_webhook" + "link": "https://docs.meilisearch.com/errors#immutable_webhook" } "#); } @@ -335,7 +342,7 @@ async fn over_limits() { snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Too many headers for the webhook `[uuid]`. Please limit the number of headers to 200.", + "message": "Too many headers for the webhook `[uuid]`. Please limit the number of headers to 200. Hint: To remove an already defined header set its value to `null`", "code": "invalid_webhooks_headers", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" @@ -387,12 +394,11 @@ async fn post_get_delete() { } #[actix_web::test] -async fn patch() { +async fn create_and_patch() { let server = Server::new().await; - let uuid = Uuid::new_v4().to_string(); let (value, code) = - server.patch_webhook(&uuid, json!({ "headers": { "authorization": "TOKEN" } })).await; + server.create_webhook(json!({ "headers": { "authorization": "TOKEN" } })).await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { @@ -403,9 +409,8 @@ async fn patch() { } "#); - let (value, code) = - server.patch_webhook(&uuid, json!({ "url": "https://example.com/hook" })).await; - snapshot!(code, @"200 OK"); + let (value, code) = server.create_webhook(json!({ "url": "https://example.com/hook" })).await; + snapshot!(code, @"201 Created"); snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" { "uuid": "[uuid]", @@ -415,6 +420,7 @@ async fn patch() { } "#); + let uuid = value.get("uuid").unwrap().as_str().unwrap(); let (value, code) = server.patch_webhook(&uuid, json!({ "headers": { "authorization": "TOKEN" } })).await; snapshot!(code, @"200 OK"); @@ -459,13 +465,15 @@ async fn patch() { "#); let (value, code) = server.patch_webhook(&uuid, json!({ "url": null })).await; - snapshot!(code, @"400 Bad Request"); - snapshot!(value, @r#" + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" { - "message": "The URL for the webhook `[uuid]` is missing.", - "code": "invalid_webhooks_url", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + "uuid": "81ccb94c-74cf-4d40-8070-492055804693", + "isEditable": true, + "url": "https://example.com/hook", + "headers": { + "authorization2": "TOKEN" + } } "#); } From a9c924b433623165859bfac15b55b3c0ed1f8c79 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 11:16:34 +0200 Subject: [PATCH 269/312] Turn url back into a setting --- crates/meilisearch/src/routes/webhooks.rs | 11 ++++++++--- crates/meilisearch/tests/tasks/webhook.rs | 12 +++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 996377240..9adad3284 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -61,9 +61,10 @@ pub fn configure(cfg: &mut web::ServiceConfig) { #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] pub(super) struct WebhookSettings { + #[schema(value_type = Option, example = "https://your.site/on-tasks-completed")] #[deserr(default, error = DeserrJsonError)] #[serde(default)] - url: Option, + url: Setting, #[schema(value_type = Option>, example = json!({"Authorization":"Bearer a-secret-token"}))] #[deserr(default, error = DeserrJsonError)] #[serde(default)] @@ -241,7 +242,11 @@ fn patch_webhook_inner( ) -> Result { let Webhook { url: old_url, mut headers } = old_webhook; - let url = new_webhook.url.unwrap_or(old_url); + let url = match new_webhook.url { + Setting::Set(url) => url, + Setting::NotSet => old_url, + Setting::Reset => return Err(MissingUrl(uuid.to_owned())), + }; match new_webhook.headers { Setting::Set(new_headers) => { @@ -369,7 +374,7 @@ async fn post_webhook( } let webhook = Webhook { - url: webhook_settings.url.ok_or(MissingUrl(uuid))?, + url: webhook_settings.url.set().ok_or(MissingUrl(uuid))?, headers: webhook_settings .headers .set() diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 268a37e18..f457fb697 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -465,15 +465,13 @@ async fn create_and_patch() { "#); let (value, code) = server.patch_webhook(&uuid, json!({ "url": null })).await; - snapshot!(code, @"200 OK"); + snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" { - "uuid": "81ccb94c-74cf-4d40-8070-492055804693", - "isEditable": true, - "url": "https://example.com/hook", - "headers": { - "authorization2": "TOKEN" - } + "message": "The URL for the webhook `[uuid]` is missing.", + "code": "invalid_webhooks_url", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" } "#); } From 8b27dec25c9e5d68b9e42c7d54e55195dc36e866 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 11:19:21 +0200 Subject: [PATCH 270/312] Test that the cli webhook receives data --- crates/meilisearch/tests/tasks/webhook.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index f457fb697..41362566a 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -166,8 +166,6 @@ async fn single_receives_data() { #[actix_web::test] async fn multiple_receive_data() { - let server = Server::new().await; - let WebhookHandle { server_handle: handle1, url: url1, receiver: mut receiver1 } = create_webhook_server().await; let WebhookHandle { server_handle: handle2, url: url2, receiver: mut receiver2 } = @@ -175,7 +173,15 @@ async fn multiple_receive_data() { let WebhookHandle { server_handle: handle3, url: url3, receiver: mut receiver3 } = create_webhook_server().await; - for url in [url1, url2, url3] { + let db_path = tempfile::tempdir().unwrap(); + let server = Server::new_with_options(Opt { + task_webhook_url: Some(Url::parse(&url3).unwrap()), + ..default_settings(db_path.path()) + }) + .await + .unwrap(); + + for url in [url1, url2] { let (value, code) = server.create_webhook(json!({ "url": url })).await; snapshot!(code, @"201 Created"); snapshot!(json_string!(value, { ".uuid" => "[uuid]", ".url" => "[ignored]" }), @r#" From 3d2c204f2d528040f6775ae0102666aa5e6987f6 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 5 Aug 2025 11:26:10 +0200 Subject: [PATCH 271/312] Update crates/milli/src/search/facet/filter.rs --- crates/milli/src/search/facet/filter.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index 4d1e51767..803a0635b 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -433,8 +433,9 @@ impl<'a> Filter<'a> { } }; - if *last == 255 { - // The prefix is invalid utf8, so no documents will match anyway + if *last == u8::MAX { + // u8::MAX is a forbidden UTF-8 byte, we're guaranteed it cannot be sent through a filter to meilisearch, but just in case, we're going to return something + tracing::warn!("Found non utf-8 character in filter. That shouldn't be possible"); return Ok(RoaringBitmap::new()); } *last += 1; From 4c61a227caf32b6e947722d1c4120a75b1e25dbe Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 5 Aug 2025 11:29:54 +0200 Subject: [PATCH 272/312] fmt after my suggestion --- crates/milli/src/search/facet/filter.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/milli/src/search/facet/filter.rs b/crates/milli/src/search/facet/filter.rs index 803a0635b..76d935fc6 100644 --- a/crates/milli/src/search/facet/filter.rs +++ b/crates/milli/src/search/facet/filter.rs @@ -435,7 +435,9 @@ impl<'a> Filter<'a> { if *last == u8::MAX { // u8::MAX is a forbidden UTF-8 byte, we're guaranteed it cannot be sent through a filter to meilisearch, but just in case, we're going to return something - tracing::warn!("Found non utf-8 character in filter. That shouldn't be possible"); + tracing::warn!( + "Found non utf-8 character in filter. That shouldn't be possible" + ); return Ok(RoaringBitmap::new()); } *last += 1; From 4f6a48c32779863bb84afaa01ec841846f726e32 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 11:44:53 +0200 Subject: [PATCH 273/312] Stop storing the cli webhook in the db --- crates/dump/src/reader/mod.rs | 5 +++ crates/index-scheduler/src/insta_snapshot.rs | 2 ++ crates/index-scheduler/src/lib.rs | 32 ++++++++++++++++++-- crates/index-scheduler/src/scheduler/mod.rs | 4 +-- crates/index-scheduler/src/test_utils.rs | 2 ++ crates/meilisearch/src/lib.rs | 29 ++---------------- 6 files changed, 41 insertions(+), 33 deletions(-) diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index 844aadc99..129b01f46 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -458,6 +458,11 @@ pub(crate) mod test { // webhooks + // Important note: You might be surprised to see the cli webhook in the dump, as it's not supposed to be saved. + // This is because the dump comes from a version that did save it. + // It's no longer the case, but that's not what this test is about. + // It's ok to see the cli webhook disappear when this test gets updated. + let webhooks = dump.webhooks().unwrap(); insta::assert_json_snapshot!(webhooks, @r#" { diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index f3431dd33..addd87be8 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -30,6 +30,8 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { index_mapper, features: _, + cli_webhook_url: _, + cli_webhook_authorization: _, cached_webhooks: _, test_breakpoint_sdr: _, planned_failures: _, diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index c18efeaf2..d04b8f9e2 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -73,6 +73,7 @@ use queue::Queue; use roaring::RoaringBitmap; use scheduler::Scheduler; use time::OffsetDateTime; +use uuid::Uuid; use versioning::Versioning; use crate::index_mapper::IndexMapper; @@ -107,6 +108,10 @@ pub struct IndexSchedulerOptions { pub snapshots_path: PathBuf, /// The path to the folder containing the dumps. pub dumps_path: PathBuf, + /// The webhook url that was set by the CLI. + pub cli_webhook_url: Option, + /// The Authorization header to send to the webhook URL that was set by the CLI. + pub cli_webhook_authorization: Option, /// The maximum size, in bytes, of the task index. pub task_db_size: usize, /// The size, in bytes, with which a meilisearch index is opened the first time of each meilisearch index. @@ -179,6 +184,10 @@ pub struct IndexScheduler { /// A database to store single-keyed data that is persisted across restarts. persisted: Database, + /// The webhook url that was set by the CLI. + cli_webhook_url: Option, + /// The Authorization header to send to the webhook URL that was set by the CLI. + cli_webhook_authorization: Option, /// Webhook cached_webhooks: Arc>, @@ -221,7 +230,11 @@ impl IndexScheduler { cleanup_enabled: self.cleanup_enabled, experimental_no_edition_2024_for_dumps: self.experimental_no_edition_2024_for_dumps, persisted: self.persisted, + + cli_webhook_url: self.cli_webhook_url.clone(), + cli_webhook_authorization: self.cli_webhook_authorization.clone(), cached_webhooks: self.cached_webhooks.clone(), + embedders: self.embedders.clone(), #[cfg(test)] test_breakpoint_sdr: self.test_breakpoint_sdr.clone(), @@ -299,7 +312,8 @@ impl IndexScheduler { let persisted = env.create_database(&mut wtxn, Some(db_name::PERSISTED))?; let webhooks_db = persisted.remap_data_type::>(); - let webhooks = webhooks_db.get(&wtxn, db_keys::WEBHOOKS)?.unwrap_or_default(); + let mut webhooks = webhooks_db.get(&wtxn, db_keys::WEBHOOKS)?.unwrap_or_default(); + webhooks.webhooks.remove(&Uuid::nil()); // remove the cli webhook if it was saved by mistake wtxn.commit()?; @@ -317,7 +331,10 @@ impl IndexScheduler { .indexer_config .experimental_no_edition_2024_for_dumps, persisted, + cached_webhooks: Arc::new(RwLock::new(webhooks)), + cli_webhook_url: options.cli_webhook_url, + cli_webhook_authorization: options.cli_webhook_authorization, embedders: Default::default(), @@ -869,9 +886,10 @@ impl IndexScheduler { self.features.network() } - pub fn put_webhooks(&self, webhooks: Webhooks) -> Result<()> { + pub fn put_webhooks(&self, mut webhooks: Webhooks) -> Result<()> { let mut wtxn = self.env.write_txn()?; let webhooks_db = self.persisted.remap_data_type::>(); + webhooks.webhooks.remove(&Uuid::nil()); // the cli webhook must not be saved webhooks_db.put(&mut wtxn, db_keys::WEBHOOKS, &webhooks)?; wtxn.commit()?; *self.cached_webhooks.write().unwrap() = webhooks; @@ -880,7 +898,15 @@ impl IndexScheduler { pub fn webhooks(&self) -> Webhooks { let webhooks = self.cached_webhooks.read().unwrap_or_else(|poisoned| poisoned.into_inner()); - Webhooks::clone(&*webhooks) + let mut webhooks = Webhooks::clone(&*webhooks); + if let Some(url) = self.cli_webhook_url.as_ref().cloned() { + let mut headers = BTreeMap::new(); + if let Some(auth) = self.cli_webhook_authorization.as_ref().cloned() { + headers.insert(String::from("Authorization"), auth); + } + webhooks.webhooks.insert(Uuid::nil(), Webhook { url, headers }); + } + webhooks } pub fn embedders( diff --git a/crates/index-scheduler/src/scheduler/mod.rs b/crates/index-scheduler/src/scheduler/mod.rs index b5acf7582..bbfc4e058 100644 --- a/crates/index-scheduler/src/scheduler/mod.rs +++ b/crates/index-scheduler/src/scheduler/mod.rs @@ -26,7 +26,6 @@ use meilisearch_types::error::ResponseError; use meilisearch_types::heed::{Env, WithoutTls}; use meilisearch_types::milli; use meilisearch_types::tasks::Status; -use meilisearch_types::webhooks::Webhooks; use process_batch::ProcessBatchInfo; use rayon::current_num_threads; use rayon::iter::{IntoParallelIterator, ParallelIterator}; @@ -448,9 +447,8 @@ impl IndexScheduler { })?; // We shouldn't crash the tick function if we can't send data to the webhooks - let webhooks = self.cached_webhooks.read().unwrap_or_else(|p| p.into_inner()); + let webhooks = self.webhooks(); if !webhooks.webhooks.is_empty() { - let webhooks = Webhooks::clone(&*webhooks); let cloned_index_scheduler = self.private_clone(); std::thread::spawn(move || { if let Err(e) = cloned_index_scheduler.notify_webhooks(webhooks, &ids) { diff --git a/crates/index-scheduler/src/test_utils.rs b/crates/index-scheduler/src/test_utils.rs index b7d69b5b3..36de0ed9e 100644 --- a/crates/index-scheduler/src/test_utils.rs +++ b/crates/index-scheduler/src/test_utils.rs @@ -98,6 +98,8 @@ impl IndexScheduler { indexes_path: tempdir.path().join("indexes"), snapshots_path: tempdir.path().join("snapshots"), dumps_path: tempdir.path().join("dumps"), + cli_webhook_url: None, + cli_webhook_authorization: None, task_db_size: 1000 * 1000 * 10, // 10 MB, we don't use MiB on purpose. index_base_map_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. enable_mdb_writemap: false, diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 613268936..533f0327f 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -13,7 +13,6 @@ pub mod routes; pub mod search; pub mod search_queue; -use std::collections::BTreeMap; use std::fs::File; use std::io::{BufReader, BufWriter}; use std::path::Path; @@ -49,14 +48,12 @@ use meilisearch_types::tasks::KindWithContent; use meilisearch_types::versioning::{ create_current_version_file, get_version, VersionFileError, VERSION_MINOR, VERSION_PATCH, }; -use meilisearch_types::webhooks::Webhook; use meilisearch_types::{compression, heed, milli, VERSION_FILE_NAME}; pub use option::Opt; use option::ScheduleSnapshot; use search_queue::SearchQueue; use tracing::{error, info_span}; use tracing_subscriber::filter::Targets; -use uuid::Uuid; use crate::error::MeilisearchHttpError; @@ -226,6 +223,8 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< indexes_path: opt.db_path.join("indexes"), snapshots_path: opt.snapshot_dir.clone(), dumps_path: opt.dump_dir.clone(), + cli_webhook_url: opt.task_webhook_url.as_ref().map(|url| url.to_string()), + cli_webhook_authorization: opt.task_webhook_authorization_header.clone(), task_db_size: opt.max_task_db_size.as_u64() as usize, index_base_map_size: opt.max_index_size.as_u64() as usize, enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage, @@ -328,30 +327,6 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< .unwrap(); } - // We set the webhook url - let cli_webhook = opt.task_webhook_url.as_ref().map(|u| Webhook { - url: u.to_string(), - headers: { - let mut headers = BTreeMap::new(); - if let Some(value) = &opt.task_webhook_authorization_header { - headers.insert(String::from("Authorization"), value.to_string()); - } - headers - }, - }); - let mut webhooks = index_scheduler.webhooks(); - if webhooks.webhooks.get(&Uuid::nil()) != cli_webhook.as_ref() { - match cli_webhook { - Some(webhook) => { - webhooks.webhooks.insert(Uuid::nil(), webhook); - } - None => { - webhooks.webhooks.remove(&Uuid::nil()); - } - } - index_scheduler.put_webhooks(webhooks)?; - } - Ok((index_scheduler, auth_controller)) } From c4e7bf2e6022d946260e559366767f87af999082 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 12:14:16 +0200 Subject: [PATCH 274/312] Stabilize `STARTS WITH` filter --- crates/filter-parser/src/lib.rs | 6 +++--- crates/index-scheduler/src/features.rs | 2 +- crates/meilisearch/tests/search/errors.rs | 24 +++++++++++------------ 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 938702103..e25636812 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -165,9 +165,9 @@ impl<'a> FilterCondition<'a> { | Condition::Exists | Condition::LowerThan(_) | Condition::LowerThanOrEqual(_) - | Condition::Between { .. } => None, - Condition::Contains { keyword, word: _ } - | Condition::StartsWith { keyword, word: _ } => Some(keyword), + | Condition::Between { .. } + | Condition::StartsWith { .. } => None, + Condition::Contains { keyword, word: _ } => Some(keyword), }, FilterCondition::Not(this) => this.use_contains_operator(), FilterCondition::Or(seq) | FilterCondition::And(seq) => { diff --git a/crates/index-scheduler/src/features.rs b/crates/index-scheduler/src/features.rs index b52a659a6..00e706a74 100644 --- a/crates/index-scheduler/src/features.rs +++ b/crates/index-scheduler/src/features.rs @@ -85,7 +85,7 @@ impl RoFeatures { Ok(()) } else { Err(FeatureNotEnabledError { - disabled_action: "Using `CONTAINS` or `STARTS WITH` in a filter", + disabled_action: "Using `CONTAINS` in a filter", feature: "contains filter", issue_link: "https://github.com/orgs/meilisearch/discussions/763", } diff --git a/crates/meilisearch/tests/search/errors.rs b/crates/meilisearch/tests/search/errors.rs index 363ece067..9cc7e06dd 100644 --- a/crates/meilisearch/tests/search/errors.rs +++ b/crates/meilisearch/tests/search/errors.rs @@ -1270,27 +1270,27 @@ async fn search_with_contains_without_enabling_the_feature() { index .search(json!({ "filter": "doggo CONTAINS kefir" }), |response, code| { snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Using `CONTAINS` or `STARTS WITH` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n7:15 doggo CONTAINS kefir", + "message": "Using `CONTAINS` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n7:15 doggo CONTAINS kefir", "code": "feature_not_enabled", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#feature_not_enabled" } - "###); + "#); }) .await; index .search(json!({ "filter": "doggo != echo AND doggo CONTAINS kefir" }), |response, code| { snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Using `CONTAINS` or `STARTS WITH` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n25:33 doggo != echo AND doggo CONTAINS kefir", + "message": "Using `CONTAINS` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n25:33 doggo != echo AND doggo CONTAINS kefir", "code": "feature_not_enabled", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#feature_not_enabled" } - "###); + "#); }) .await; @@ -1299,24 +1299,24 @@ async fn search_with_contains_without_enabling_the_feature() { index.search_post(json!({ "filter": ["doggo != echo", "doggo CONTAINS kefir"] })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Using `CONTAINS` or `STARTS WITH` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n7:15 doggo CONTAINS kefir", + "message": "Using `CONTAINS` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n7:15 doggo CONTAINS kefir", "code": "feature_not_enabled", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#feature_not_enabled" } - "###); + "#); let (response, code) = index.search_post(json!({ "filter": ["doggo != echo", ["doggo CONTAINS kefir"]] })).await; snapshot!(code, @"400 Bad Request"); - snapshot!(json_string!(response), @r###" + snapshot!(json_string!(response), @r#" { - "message": "Using `CONTAINS` or `STARTS WITH` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n7:15 doggo CONTAINS kefir", + "message": "Using `CONTAINS` in a filter requires enabling the `contains filter` experimental feature. See https://github.com/orgs/meilisearch/discussions/763\n7:15 doggo CONTAINS kefir", "code": "feature_not_enabled", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#feature_not_enabled" } - "###); + "#); } From 3a9b08960abe17e46f31cf0196c2a8453df68772 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 13:49:28 +0200 Subject: [PATCH 275/312] Add test --- .../milli/src/search/new/tests/integration.rs | 2 +- crates/milli/tests/search/filters.rs | 13 ++++++++++-- crates/milli/tests/search/mod.rs | 21 +++++++++++++++++-- 3 files changed, 31 insertions(+), 5 deletions(-) diff --git a/crates/milli/src/search/new/tests/integration.rs b/crates/milli/src/search/new/tests/integration.rs index 38f39e18b..6b8c25ab8 100644 --- a/crates/milli/src/search/new/tests/integration.rs +++ b/crates/milli/src/search/new/tests/integration.rs @@ -17,7 +17,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { let path = tempfile::tempdir().unwrap(); let options = EnvOpenOptions::new(); let mut options = options.read_txn_without_tls(); - options.map_size(10 * 1024 * 1024); // 10 MB + options.map_size(10 * 1024 * 1024); // 10 MiB let index = Index::new(options, &path, true).unwrap(); let mut wtxn = index.write_txn().unwrap(); diff --git a/crates/milli/tests/search/filters.rs b/crates/milli/tests/search/filters.rs index bb5943782..c97143d48 100644 --- a/crates/milli/tests/search/filters.rs +++ b/crates/milli/tests/search/filters.rs @@ -25,13 +25,16 @@ macro_rules! test_filter { let SearchResult { documents_ids, .. } = search.execute().unwrap(); let filtered_ids = search::expected_filtered_ids($filter); - let expected_external_ids: Vec<_> = + let mut expected_external_ids: Vec<_> = search::expected_order(&criteria, TermsMatchingStrategy::default(), &[]) .into_iter() .filter_map(|d| if filtered_ids.contains(&d.id) { Some(d.id) } else { None }) .collect(); - let documents_ids = search::internal_to_external_ids(&index, &documents_ids); + let mut documents_ids = search::internal_to_external_ids(&index, &documents_ids); + + expected_external_ids.sort_unstable(); + documents_ids.sort_unstable(); assert_eq!(documents_ids, expected_external_ids); } }; @@ -102,3 +105,9 @@ test_filter!(empty_filter_1_double_not, vec![Right("NOT opt1 IS NOT EMPTY")]); test_filter!(in_filter, vec![Right("tag_in IN[1, 2, 3, four, five]")]); test_filter!(not_in_filter, vec![Right("tag_in NOT IN[1, 2, 3, four, five]")]); test_filter!(not_not_in_filter, vec![Right("NOT tag_in NOT IN[1, 2, 3, four, five]")]); + +test_filter!(starts_with_filter_single_letter, vec![Right("tag STARTS WITH e")]); +test_filter!(starts_with_filter_diacritic, vec![Right("tag STARTS WITH é")]); +test_filter!(starts_with_filter_empty_prefix, vec![Right("tag STARTS WITH ''")]); +test_filter!(starts_with_filter_hell, vec![Right("title STARTS WITH hell")]); +test_filter!(starts_with_filter_hello, vec![Right("title STARTS WITH hello")]); diff --git a/crates/milli/tests/search/mod.rs b/crates/milli/tests/search/mod.rs index fa03f1cc1..578a22009 100644 --- a/crates/milli/tests/search/mod.rs +++ b/crates/milli/tests/search/mod.rs @@ -12,7 +12,8 @@ use milli::update::new::indexer; use milli::update::{IndexerConfig, Settings}; use milli::vector::RuntimeEmbedders; use milli::{ - AscDesc, Criterion, DocumentId, FilterableAttributesRule, Index, Member, TermsMatchingStrategy, + normalize_facet, AscDesc, Criterion, DocumentId, FilterableAttributesRule, Index, Member, + TermsMatchingStrategy, }; use serde::{Deserialize, Deserializer}; use slice_group_by::GroupBy; @@ -36,7 +37,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { let path = tempfile::tempdir().unwrap(); let options = EnvOpenOptions::new(); let mut options = options.read_txn_without_tls(); - options.map_size(10 * 1024 * 1024); // 10 MB + options.map_size(10 * 1024 * 1024); // 10 MiB let index = Index::new(options, &path, true).unwrap(); let mut wtxn = index.write_txn().unwrap(); @@ -46,6 +47,7 @@ pub fn setup_search_index_with_criteria(criteria: &[Criterion]) -> Index { builder.set_criteria(criteria.to_vec()); builder.set_filterable_fields(vec![ + FilterableAttributesRule::Field(S("title")), FilterableAttributesRule::Field(S("tag")), FilterableAttributesRule::Field(S("asc_desc_rank")), FilterableAttributesRule::Field(S("_geo")), @@ -220,6 +222,19 @@ fn execute_filter(filter: &str, document: &TestDocument) -> Option { { id = Some(document.id.clone()) } + } else if let Some((field, prefix)) = filter.split_once("STARTS WITH") { + let field = match field.trim() { + "tag" => &document.tag, + "title" => &document.title, + "description" => &document.description, + _ => panic!("Unknown field: {field}"), + }; + + let field = normalize_facet(field); + let prefix = normalize_facet(prefix.trim().trim_matches('\'')); + if field.starts_with(&prefix) { + id = Some(document.id.clone()); + } } else if let Some(("asc_desc_rank", filter)) = filter.split_once('<') { if document.asc_desc_rank < filter.parse().unwrap() { id = Some(document.id.clone()) @@ -271,6 +286,8 @@ fn execute_filter(filter: &str, document: &TestDocument) -> Option { } else if matches!(filter, "tag_in NOT IN[1, 2, 3, four, five]") { id = (!matches!(document.id.as_str(), "A" | "B" | "C" | "D" | "E")) .then(|| document.id.clone()); + } else { + panic!("Unknown filter: {filter}"); } id } From 2121819c66a0d66b9627a46962316045f77a1af0 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 14:18:45 +0200 Subject: [PATCH 276/312] Fix tests --- crates/meilisearch/tests/search/filters.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 76f252a6d..8e3ee9249 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -1035,7 +1035,7 @@ async fn vector_filter_document_template() { ]); let (value, code) = index.add_documents(documents, None).await; snapshot!(code, @"202 Accepted"); - index.wait_task(value.uid()).await.succeeded(); + server.wait_task(value.uid()).await.succeeded(); let (value, _code) = index .search_post(json!({ From 095cba8fba6bc4faeeac364e1cdc93a72572152b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine?= Date: Tue, 5 Aug 2025 15:29:42 +0200 Subject: [PATCH 277/312] Minor docs update about release.md --- documentation/release.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/documentation/release.md b/documentation/release.md index f70fcf872..69ce32aff 100644 --- a/documentation/release.md +++ b/documentation/release.md @@ -7,7 +7,7 @@ This guide is to describe how to make releases for the current repository. 1. A weekly meeting is done every Monday to define the release and to ensure minimal checks before the release.

Check out the TODO 👇👇👇 -- [ ] Define the version of the release (`vX.Y.Z`) +- [ ] Define the version of the release (`vX.Y.Z`) based on our Versioning Policy
. - [ ] Manually test `--experimental-dumpless-upgrade` on a DB of the previous Meilisearch minor version
- [ ] Check recent automated tests on `main`
- [ ] Scheduled test suite
From 45bb13bf4346f0a38370a125dd1d17d1b08eed63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine?= Date: Tue, 5 Aug 2025 15:42:56 +0200 Subject: [PATCH 278/312] Minor fix in PR template --- .github/pull_request_template.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 3665d3303..0fbc68c1d 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -5,12 +5,12 @@ Fixes #... ## Requirements ⚠️ Ensure the following requirements before merging ⚠️ -- [] Automated tests have been added. -- [] If some tests cannot be automated, manual rigorous tests should be applied. -- [] ⚠️ If there is any change in the DB: +- [ ] Automated tests have been added. +- [ ] If some tests cannot be automated, manual rigorous tests should be applied. +- [ ] ⚠️ If there is any change in the DB: - [ ] Test that any impacted DB still works as expected after using `--experimental-dumpless-upgrade` on a DB created with the last released Meilisearch - [ ] Test that during the upgrade, **search is still available** (artificially make the upgrade longer if needed) - [ ] Set the `db change` label. -- [] If necessary, the feature have been tested in the Cloud production environment (with [prototypes](./documentation/prototypes.md)) and the Cloud UI is ready. -- [] If necessary, the [documentation](https://github.com/meilisearch/documentation) related to the implemented feature in the PR is ready. -- [] If necessary, the [integrations](https://github.com/meilisearch/integration-guides) related to the implemented feature in the PR are ready. +- [ ] If necessary, the feature have been tested in the Cloud production environment (with [prototypes](./documentation/prototypes.md)) and the Cloud UI is ready. +- [ ] If necessary, the [documentation](https://github.com/meilisearch/documentation) related to the implemented feature in the PR is ready. +- [ ] If necessary, the [integrations](https://github.com/meilisearch/integration-guides) related to the implemented feature in the PR are ready. From c385cf985b679207781a17e65ef20df2bb1a39b6 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 15:55:31 +0200 Subject: [PATCH 279/312] Fix tests --- crates/meilisearch/src/search/mod.rs | 1 + crates/meilisearch/tests/vector/binary_quantized.rs | 5 ----- crates/meilisearch/tests/vector/mod.rs | 10 ---------- crates/milli/src/search/hybrid.rs | 1 + crates/milli/src/search/mod.rs | 11 ++++++++++- 5 files changed, 12 insertions(+), 16 deletions(-) diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index d4775c66a..c681a11a5 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -1056,6 +1056,7 @@ pub fn prepare_search<'t>( .map(|x| x as usize) .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS); + search.retrieve_vectors(query.retrieve_vectors); search.exhaustive_number_hits(is_finite_pagination); search.max_total_hits(Some(max_total_hits)); search.scoring_strategy( diff --git a/crates/meilisearch/tests/vector/binary_quantized.rs b/crates/meilisearch/tests/vector/binary_quantized.rs index e0fa9a37c..adb0da441 100644 --- a/crates/meilisearch/tests/vector/binary_quantized.rs +++ b/crates/meilisearch/tests/vector/binary_quantized.rs @@ -327,11 +327,6 @@ async fn binary_quantize_clear_documents() { { "hits": [], "query": "", - "queryVector": [ - 1.0, - 1.0, - 1.0 - ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, diff --git a/crates/meilisearch/tests/vector/mod.rs b/crates/meilisearch/tests/vector/mod.rs index 0eb1f063e..551b82178 100644 --- a/crates/meilisearch/tests/vector/mod.rs +++ b/crates/meilisearch/tests/vector/mod.rs @@ -690,11 +690,6 @@ async fn clear_documents() { { "hits": [], "query": "", - "queryVector": [ - 1.0, - 1.0, - 1.0 - ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, @@ -754,11 +749,6 @@ async fn add_remove_one_vector_4588() { } ], "query": "", - "queryVector": [ - 1.0, - 1.0, - 1.0 - ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, diff --git a/crates/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs index 353c69dab..75ac547b4 100644 --- a/crates/milli/src/search/hybrid.rs +++ b/crates/milli/src/search/hybrid.rs @@ -212,6 +212,7 @@ impl Search<'_> { terms_matching_strategy: self.terms_matching_strategy, scoring_strategy: ScoringStrategy::Detailed, words_limit: self.words_limit, + retrieve_vectors: self.retrieve_vectors, exhaustive_number_hits: self.exhaustive_number_hits, max_total_hits: self.max_total_hits, rtxn: self.rtxn, diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index 2192e842f..e7871031b 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -53,6 +53,7 @@ pub struct Search<'a> { terms_matching_strategy: TermsMatchingStrategy, scoring_strategy: ScoringStrategy, words_limit: usize, + retrieve_vectors: bool, exhaustive_number_hits: bool, max_total_hits: Option, rtxn: &'a heed::RoTxn<'a>, @@ -76,6 +77,7 @@ impl<'a> Search<'a> { geo_param: GeoSortParameter::default(), terms_matching_strategy: TermsMatchingStrategy::default(), scoring_strategy: Default::default(), + retrieve_vectors: false, exhaustive_number_hits: false, max_total_hits: None, words_limit: 10, @@ -188,6 +190,11 @@ impl<'a> Search<'a> { self } + pub fn retrieve_vectors(&mut self, retrieve_vectors: bool) -> &mut Search<'a> { + self.retrieve_vectors = retrieve_vectors; + self + } + /// Forces the search to exhaustively compute the number of candidates, /// this will increase the search time but allows finite pagination. pub fn exhaustive_number_hits(&mut self, exhaustive_number_hits: bool) -> &mut Search<'a> { @@ -277,7 +284,7 @@ impl<'a> Search<'a> { quantized, media: _, }) => { - if *auto_embedded { + if *auto_embedded && self.retrieve_vectors { query_vector = Some(vector.clone()); } execute_vector_search( @@ -359,6 +366,7 @@ impl fmt::Debug for Search<'_> { terms_matching_strategy, scoring_strategy, words_limit, + retrieve_vectors, exhaustive_number_hits, max_total_hits, rtxn: _, @@ -379,6 +387,7 @@ impl fmt::Debug for Search<'_> { .field("searchable_attributes", searchable_attributes) .field("terms_matching_strategy", terms_matching_strategy) .field("scoring_strategy", scoring_strategy) + .field("retrieve_vectors", retrieve_vectors) .field("exhaustive_number_hits", exhaustive_number_hits) .field("max_total_hits", max_total_hits) .field("words_limit", words_limit) From 2b5b41790edfb72530bc8db00cf604eda889fef0 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 5 Aug 2025 16:21:14 +0200 Subject: [PATCH 280/312] update the dump so it doesn't contains the null-uuid webhook --- crates/dump/src/reader/mod.rs | 12 ------------ .../dump/tests/assets/v6-with-webhooks.dump | Bin 1389 -> 1391 bytes 2 files changed, 12 deletions(-) diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index 129b01f46..da55bb4a8 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -457,22 +457,10 @@ pub(crate) mod test { "); // webhooks - - // Important note: You might be surprised to see the cli webhook in the dump, as it's not supposed to be saved. - // This is because the dump comes from a version that did save it. - // It's no longer the case, but that's not what this test is about. - // It's ok to see the cli webhook disappear when this test gets updated. - let webhooks = dump.webhooks().unwrap(); insta::assert_json_snapshot!(webhooks, @r#" { "webhooks": { - "00000000-0000-0000-0000-000000000000": { - "url": "https://defined-in-dump.com/", - "headers": { - "Authorization": "Bearer defined in dump" - } - }, "627ea538-733d-4545-8d2d-03526eb381ce": { "url": "https://example.com/authorization-less", "headers": {} diff --git a/crates/dump/tests/assets/v6-with-webhooks.dump b/crates/dump/tests/assets/v6-with-webhooks.dump index c8f9649d88429ab8c16f3375f94a6c4474d3e79f..955c2a63dc79b51bfc121b30b0f5843480c75361 100644 GIT binary patch literal 1391 zcmV-#1(5n5iwFRO5t3*C1MOPfZ`(Ey&vX9@O^<~si(itQw=QXcp};nDZ68()1w|ff zEwbcE)Y{C*|GuMSCyldw0k*Ss`oYi#b<)vyygMFw(e!BVh~qewGU(6AaE_T6&T9lz z#zKhUI;M;!@+gq|@7mE(*E-c;SBeJytnqLg9g<={d3OQsQ*JCntq0SUe{-B^m|)Zd*OldNQAgu{~}>q_Ak%~-c zZeBOVU&;pJjqbp4g1dWTq2&Hxu9A zYo86TvBkIE{`C6gkD=#gxy*Q+abh!-5}|WTl!TPnoLdSE8)0^no0}Jn@w~Qre{u2b zsWrA+MAI`kA8fze7ms`FpK-bE{{>^|X#a=6Dm2jQrdX~G=Uody;R06&EvQD%1KZS# zz|Fy5R1PYG7E!4Mx`FT%tm;Y^{4!9!`1yz6HJsz~t}ysmq?#%!a$+r-pP-f!G;K|= zky)TBp~@n$(w;S%*N1Rm8U8j2i^{+k=o_E=d-$?M-LxN-3y-U=EGMCEKshjdOUPm- zNt_Z+-^R0J7UM6HX)n2duIeLZa6ADAQplE}tc7fW6sa`_U zJD(^xL1%;)-vy_2PlJ80htJd%zOd}_ve0*k^MzRy)eUJ{W6@vQ84($!L>Q(>o@-5% z5|-*D6?&Flm*(?b2{K7}CLSipHR-K_#Bqh}Oi)xP(nT+@K<_QM2p@AfO1BBFd%|ag;&}SR6!MsZEnTZKz$y40*6h# z@AQ|ulC+fUfr9QS?~=3j!&Ft~5r%x?U)h#Up7B|RZkeQzV*s_YgeVioB+X==YRi<8 z=DKt|=abq;>!E3~W5Z5poY4o0X(0X9qW10(O}(~=zET%y!8!DEyUPh5pZ}$tiZTB? z1fuBzobn#`>GD6y*!KJ{kz@XM5Mcc17R&dT-wg5NeS&_~R$iWN?uA4_&`Q)a3fjMijsEsPuO{33xG@>ag&Z$bnM@;vl4i7u$ zac}&`hu7_2N)eCte+cM2%Tl8?;UecmWQG!Dj3Kj(N(CvSDx*)`#}WHOG|iRMbJ%AE zz~k3{ctVqL{pS$y{=Z+5{g>%qjO%{~!Il00(Lo1$*$BQj{@>8QOw)1y_aNBRf9+5E z_v)X=Lf)AFjr;$H!D`s$izi|42j2OcVJ)bS3t!!%09=R3SJ-x10~qkTe(rqP7XL0056E(^dcg literal 1389 zcmV-z1(Nz7iwFP!00000|Ls~)Z`(Ey_H%!QrpLl`NQx5WyroGC3MQPdwcIj3zwl?SBAd%IV4f z(EhQEd;1>%OZy-F)j|uSKbxv72D=8DlO{qu#QqWW$H4*c=;J@Wf2rX^o10OKUxVG_ zUi;&{HT&~eM!o$H0G*|&G+L8LWDyamVMG~Y$T(%O0?C<5*`UV{3-~;PI-f%6R6#5# z*VfQ~Ff6K%&FiXlaMc1PcW0Gu z{VZvoUA$;Z?eeO;y00soJ(x&d94!^R&HAs;{|CW5T+IL^eZrMR z$XC8r@N#X9lxsL~Zq^(h9T~8B35Mh)@kQbY%ZVfO&IyLyj>esbC#{=Qb^flxf%;0(@3J-gI57FFPXz zG|i2{_)er`8C5(bm{L{?9=zDU$}R?xLf~x vqbZI3pnW!TcS@#0NH2+{SKdtS5>2@n%bq;x(W6I?9}@onHxdkx05|{u+j_X9 From 3f1e172c6f8adf9db2d782e043032c0848b63d45 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 5 Aug 2025 16:47:35 +0200 Subject: [PATCH 281/312] fix race condition: take the rtxn before entering the thread so we're sure we won't try to retrieve deleted tasks --- crates/index-scheduler/src/lib.rs | 65 ++++++++++++--------- crates/index-scheduler/src/scheduler/mod.rs | 11 +--- 2 files changed, 40 insertions(+), 36 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index d04b8f9e2..419e6f21e 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -784,7 +784,7 @@ impl IndexScheduler { } /// Once the tasks changes have been committed we must send all the tasks that were updated to our webhooks - fn notify_webhooks(&self, webhooks: Webhooks, updated: &RoaringBitmap) -> Result<()> { + fn notify_webhooks(&self, updated: RoaringBitmap) { struct TaskReader<'a, 'b> { rtxn: &'a RoTxn<'a>, index_scheduler: &'a IndexScheduler, @@ -829,33 +829,46 @@ impl IndexScheduler { } } - let rtxn = self.env.read_txn()?; - - for (uuid, Webhook { url, headers }) in webhooks.webhooks.iter() { - let task_reader = TaskReader { - rtxn: &rtxn, - index_scheduler: self, - tasks: &mut updated.into_iter(), - buffer: Vec::with_capacity(page_size::get()), - written: 0, - }; - - let reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); - - let mut request = ureq::post(url) - .timeout(Duration::from_secs(30)) - .set("Content-Encoding", "gzip") - .set("Content-Type", "application/x-ndjson"); - for (header_name, header_value) in headers.iter() { - request = request.set(header_name, header_value); - } - - if let Err(e) = request.send(reader) { - tracing::error!("While sending data to the webhook {uuid}: {e}"); - } + let webhooks = self.webhooks(); + if webhooks.webhooks.is_empty() { + return; } + let this = self.private_clone(); + // We must take the RoTxn before entering the thread::spawn otherwise another batch may be + // processed before we had the time to take our txn. + let rtxn = match self.env.clone().static_read_txn() { + Ok(rtxn) => rtxn, + Err(e) => { + tracing::error!("Couldn't get an rtxn to notify the webhook: {e}"); + return; + } + }; - Ok(()) + std::thread::spawn(move || { + for (uuid, Webhook { url, headers }) in webhooks.webhooks.iter() { + let task_reader = TaskReader { + rtxn: &rtxn, + index_scheduler: &this, + tasks: &mut updated.iter(), + buffer: Vec::with_capacity(page_size::get()), + written: 0, + }; + + let reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); + + let mut request = ureq::post(url) + .timeout(Duration::from_secs(30)) + .set("Content-Encoding", "gzip") + .set("Content-Type", "application/x-ndjson"); + for (header_name, header_value) in headers.iter() { + request = request.set(header_name, header_value); + } + + if let Err(e) = request.send(reader) { + tracing::error!("While sending data to the webhook {uuid}: {e}"); + } + } + }); } pub fn index_stats(&self, index_uid: &str) -> Result { diff --git a/crates/index-scheduler/src/scheduler/mod.rs b/crates/index-scheduler/src/scheduler/mod.rs index bbfc4e058..b2bb90c0b 100644 --- a/crates/index-scheduler/src/scheduler/mod.rs +++ b/crates/index-scheduler/src/scheduler/mod.rs @@ -446,16 +446,7 @@ impl IndexScheduler { Ok(()) })?; - // We shouldn't crash the tick function if we can't send data to the webhooks - let webhooks = self.webhooks(); - if !webhooks.webhooks.is_empty() { - let cloned_index_scheduler = self.private_clone(); - std::thread::spawn(move || { - if let Err(e) = cloned_index_scheduler.notify_webhooks(webhooks, &ids) { - tracing::error!("Failure to notify webhooks: {e}"); - } - }); - } + self.notify_webhooks(ids); #[cfg(test)] self.breakpoint(crate::test_utils::Breakpoint::AfterProcessing); From b5158e1e8362dd5a6ea5bb0b751ec356e44f3f54 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 16:49:54 +0200 Subject: [PATCH 282/312] Fix cli webhook getting stored in dumps --- crates/dump/src/writer.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/dump/src/writer.rs b/crates/dump/src/writer.rs index 84a76e483..448896e06 100644 --- a/crates/dump/src/writer.rs +++ b/crates/dump/src/writer.rs @@ -75,10 +75,11 @@ impl DumpWriter { Ok(std::fs::write(self.dir.path().join("network.json"), serde_json::to_string(&network)?)?) } - pub fn create_webhooks(&self, webhooks: Webhooks) -> Result<()> { + pub fn create_webhooks(&self, mut webhooks: Webhooks) -> Result<()> { if webhooks == Webhooks::default() { return Ok(()); } + webhooks.webhooks.remove(&Uuid::nil()); // Don't store the cli webhook Ok(std::fs::write( self.dir.path().join("webhooks.json"), serde_json::to_string(&webhooks)?, From 1ff6da63e8d1236e5332067b4729a2bd513f885f Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 16:58:21 +0200 Subject: [PATCH 283/312] Make errors singular --- crates/meilisearch-types/src/error.rs | 4 ++-- crates/meilisearch/src/routes/webhooks.rs | 16 +++++++-------- crates/meilisearch/tests/tasks/webhook.rs | 24 +++++++++++------------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index bb486726a..4360d947b 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -421,8 +421,8 @@ InvalidChatCompletionSearchIndexUidParamPrompt , InvalidRequest , BAD_REQU InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQUEST ; // Webhooks InvalidWebhooks , InvalidRequest , BAD_REQUEST ; -InvalidWebhooksUrl , InvalidRequest , BAD_REQUEST ; -InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST ; +InvalidWebhookUrl , InvalidRequest , BAD_REQUEST ; +InvalidWebhookHeaders , InvalidRequest , BAD_REQUEST ; ImmutableWebhook , InvalidRequest , BAD_REQUEST ; InvalidWebhookUuid , InvalidRequest , BAD_REQUEST ; WebhookNotFound , InvalidRequest , NOT_FOUND ; diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 9adad3284..e3547996a 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -13,7 +13,7 @@ use deserr::{DeserializeError, Deserr, ValuePointerRef}; use index_scheduler::IndexScheduler; use meilisearch_types::deserr::{immutable_field_error, DeserrJsonError}; use meilisearch_types::error::deserr_codes::{ - BadRequest, InvalidWebhooksHeaders, InvalidWebhooksUrl, + BadRequest, InvalidWebhookHeaders, InvalidWebhookUrl, }; use meilisearch_types::error::{Code, ErrorCode, ResponseError}; use meilisearch_types::keys::actions; @@ -62,11 +62,11 @@ pub fn configure(cfg: &mut web::ServiceConfig) { #[schema(rename_all = "camelCase")] pub(super) struct WebhookSettings { #[schema(value_type = Option, example = "https://your.site/on-tasks-completed")] - #[deserr(default, error = DeserrJsonError)] + #[deserr(default, error = DeserrJsonError)] #[serde(default)] url: Setting, #[schema(value_type = Option>, example = json!({"Authorization":"Bearer a-secret-token"}))] - #[deserr(default, error = DeserrJsonError)] + #[deserr(default, error = DeserrJsonError)] #[serde(default)] headers: Setting>>, } @@ -222,14 +222,14 @@ enum WebhooksError { impl ErrorCode for WebhooksError { fn error_code(&self) -> meilisearch_types::error::Code { match self { - MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhooksUrl, + MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhookUrl, TooManyWebhooks => meilisearch_types::error::Code::InvalidWebhooks, - TooManyHeaders(_) => meilisearch_types::error::Code::InvalidWebhooksHeaders, + TooManyHeaders(_) => meilisearch_types::error::Code::InvalidWebhookHeaders, ImmutableWebhook(_) => meilisearch_types::error::Code::ImmutableWebhook, WebhookNotFound(_) => meilisearch_types::error::Code::WebhookNotFound, - InvalidHeaderName(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, - InvalidHeaderValue(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, - InvalidUrl(_, _) => meilisearch_types::error::Code::InvalidWebhooksUrl, + InvalidHeaderName(_, _) => meilisearch_types::error::Code::InvalidWebhookHeaders, + InvalidHeaderValue(_, _) => meilisearch_types::error::Code::InvalidWebhookHeaders, + InvalidUrl(_, _) => meilisearch_types::error::Code::InvalidWebhookUrl, InvalidUuid(_) => meilisearch_types::error::Code::InvalidWebhookUuid, } } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 41362566a..a3be37a4e 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -349,9 +349,9 @@ async fn over_limits() { snapshot!(value, @r#" { "message": "Too many headers for the webhook `[uuid]`. Please limit the number of headers to 200. Hint: To remove an already defined header set its value to `null`", - "code": "invalid_webhooks_headers", + "code": "invalid_webhook_headers", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" + "link": "https://docs.meilisearch.com/errors#invalid_webhook_headers" } "#); } @@ -409,9 +409,9 @@ async fn create_and_patch() { snapshot!(value, @r#" { "message": "The URL for the webhook `[uuid]` is missing.", - "code": "invalid_webhooks_url", + "code": "invalid_webhook_url", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + "link": "https://docs.meilisearch.com/errors#invalid_webhook_url" } "#); @@ -475,9 +475,9 @@ async fn create_and_patch() { snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" { "message": "The URL for the webhook `[uuid]` is missing.", - "code": "invalid_webhooks_url", + "code": "invalid_webhook_url", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + "link": "https://docs.meilisearch.com/errors#invalid_webhook_url" } "#); } @@ -492,9 +492,9 @@ async fn invalid_url_and_headers() { snapshot!(value, @r#" { "message": "Invalid URL `not-a-valid-url`: relative URL without a base", - "code": "invalid_webhooks_url", + "code": "invalid_webhook_url", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + "link": "https://docs.meilisearch.com/errors#invalid_webhook_url" } "#); @@ -509,9 +509,9 @@ async fn invalid_url_and_headers() { snapshot!(value, @r#" { "message": "Invalid header name `invalid header name`: invalid HTTP header name", - "code": "invalid_webhooks_headers", + "code": "invalid_webhook_headers", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" + "link": "https://docs.meilisearch.com/errors#invalid_webhook_headers" } "#); @@ -526,9 +526,9 @@ async fn invalid_url_and_headers() { snapshot!(value, @r#" { "message": "Invalid header value `authorization`: failed to parse header value", - "code": "invalid_webhooks_headers", + "code": "invalid_webhook_headers", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" + "link": "https://docs.meilisearch.com/errors#invalid_webhook_headers" } "#); } From 899be9c3ffa6935f56d24026a0050357bc23bfc4 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 5 Aug 2025 18:55:32 +0200 Subject: [PATCH 284/312] make sure we NEVER ever write the cli defined webhook to the database or dumps --- crates/dump/src/reader/v6/mod.rs | 2 +- crates/dump/src/writer.rs | 8 +- crates/index-scheduler/src/insta_snapshot.rs | 4 +- crates/index-scheduler/src/lib.rs | 128 +++++++++++++----- .../src/scheduler/process_dump_creation.rs | 2 +- crates/meilisearch-types/src/webhooks.rs | 13 +- crates/meilisearch/src/lib.rs | 2 +- crates/meilisearch/src/routes/webhooks.rs | 30 ++-- crates/meilisearch/tests/tasks/webhook.rs | 4 +- 9 files changed, 130 insertions(+), 63 deletions(-) diff --git a/crates/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs index d8ce430f9..9bc4b33c5 100644 --- a/crates/dump/src/reader/v6/mod.rs +++ b/crates/dump/src/reader/v6/mod.rs @@ -25,7 +25,7 @@ pub type Key = meilisearch_types::keys::Key; pub type ChatCompletionSettings = meilisearch_types::features::ChatCompletionSettings; pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures; pub type Network = meilisearch_types::features::Network; -pub type Webhooks = meilisearch_types::webhooks::Webhooks; +pub type Webhooks = meilisearch_types::webhooks::WebhooksDumpView; // ===== Other types to clarify the code of the compat module // everything related to the tasks diff --git a/crates/dump/src/writer.rs b/crates/dump/src/writer.rs index 448896e06..1d41b6aa5 100644 --- a/crates/dump/src/writer.rs +++ b/crates/dump/src/writer.rs @@ -8,7 +8,7 @@ use meilisearch_types::batches::Batch; use meilisearch_types::features::{ChatCompletionSettings, Network, RuntimeTogglableFeatures}; use meilisearch_types::keys::Key; use meilisearch_types::settings::{Checked, Settings}; -use meilisearch_types::webhooks::Webhooks; +use meilisearch_types::webhooks::WebhooksDumpView; use serde_json::{Map, Value}; use tempfile::TempDir; use time::OffsetDateTime; @@ -75,11 +75,7 @@ impl DumpWriter { Ok(std::fs::write(self.dir.path().join("network.json"), serde_json::to_string(&network)?)?) } - pub fn create_webhooks(&self, mut webhooks: Webhooks) -> Result<()> { - if webhooks == Webhooks::default() { - return Ok(()); - } - webhooks.webhooks.remove(&Uuid::nil()); // Don't store the cli webhook + pub fn create_webhooks(&self, webhooks: WebhooksDumpView) -> Result<()> { Ok(std::fs::write( self.dir.path().join("webhooks.json"), serde_json::to_string(&webhooks)?, diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index addd87be8..cb804d9b4 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -30,9 +30,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { index_mapper, features: _, - cli_webhook_url: _, - cli_webhook_authorization: _, - cached_webhooks: _, + webhooks: _, test_breakpoint_sdr: _, planned_failures: _, run_loop_iteration: _, diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 419e6f21e..6ad7a8397 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -65,13 +65,14 @@ use meilisearch_types::milli::vector::{ use meilisearch_types::milli::{self, Index}; use meilisearch_types::task_view::TaskView; use meilisearch_types::tasks::{KindWithContent, Task}; -use meilisearch_types::webhooks::{Webhook, Webhooks}; +use meilisearch_types::webhooks::{Webhook, WebhooksDumpView, WebhooksView}; use milli::vector::db::IndexEmbeddingConfig; use processing::ProcessingTasks; pub use queue::Query; use queue::Queue; use roaring::RoaringBitmap; use scheduler::Scheduler; +use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use uuid::Uuid; use versioning::Versioning; @@ -184,12 +185,8 @@ pub struct IndexScheduler { /// A database to store single-keyed data that is persisted across restarts. persisted: Database, - /// The webhook url that was set by the CLI. - cli_webhook_url: Option, - /// The Authorization header to send to the webhook URL that was set by the CLI. - cli_webhook_authorization: Option, - /// Webhook - cached_webhooks: Arc>, + /// Webhook, loaded and stored in the `persisted` database + webhooks: Arc, /// A map to retrieve the runtime representation of an embedder depending on its configuration. /// @@ -231,10 +228,7 @@ impl IndexScheduler { experimental_no_edition_2024_for_dumps: self.experimental_no_edition_2024_for_dumps, persisted: self.persisted, - cli_webhook_url: self.cli_webhook_url.clone(), - cli_webhook_authorization: self.cli_webhook_authorization.clone(), - cached_webhooks: self.cached_webhooks.clone(), - + webhooks: self.webhooks.clone(), embedders: self.embedders.clone(), #[cfg(test)] test_breakpoint_sdr: self.test_breakpoint_sdr.clone(), @@ -313,7 +307,8 @@ impl IndexScheduler { let persisted = env.create_database(&mut wtxn, Some(db_name::PERSISTED))?; let webhooks_db = persisted.remap_data_type::>(); let mut webhooks = webhooks_db.get(&wtxn, db_keys::WEBHOOKS)?.unwrap_or_default(); - webhooks.webhooks.remove(&Uuid::nil()); // remove the cli webhook if it was saved by mistake + webhooks + .with_cli(options.cli_webhook_url.clone(), options.cli_webhook_authorization.clone()); wtxn.commit()?; @@ -331,11 +326,7 @@ impl IndexScheduler { .indexer_config .experimental_no_edition_2024_for_dumps, persisted, - - cached_webhooks: Arc::new(RwLock::new(webhooks)), - cli_webhook_url: options.cli_webhook_url, - cli_webhook_authorization: options.cli_webhook_authorization, - + webhooks: Arc::new(webhooks), embedders: Default::default(), #[cfg(test)] @@ -829,8 +820,8 @@ impl IndexScheduler { } } - let webhooks = self.webhooks(); - if webhooks.webhooks.is_empty() { + let webhooks = self.webhooks.get_all(); + if webhooks.is_empty() { return; } let this = self.private_clone(); @@ -845,7 +836,7 @@ impl IndexScheduler { }; std::thread::spawn(move || { - for (uuid, Webhook { url, headers }) in webhooks.webhooks.iter() { + for (uuid, Webhook { url, headers }) in webhooks.iter() { let task_reader = TaskReader { rtxn: &rtxn, index_scheduler: &this, @@ -899,27 +890,27 @@ impl IndexScheduler { self.features.network() } - pub fn put_webhooks(&self, mut webhooks: Webhooks) -> Result<()> { + pub fn update_runtime_webhooks(&self, runtime: RuntimeWebhooks) -> Result<()> { + let webhooks = Webhooks::from_runtime(runtime); let mut wtxn = self.env.write_txn()?; let webhooks_db = self.persisted.remap_data_type::>(); - webhooks.webhooks.remove(&Uuid::nil()); // the cli webhook must not be saved webhooks_db.put(&mut wtxn, db_keys::WEBHOOKS, &webhooks)?; wtxn.commit()?; - *self.cached_webhooks.write().unwrap() = webhooks; + self.webhooks.update_runtime(webhooks.into_runtime()); Ok(()) } - pub fn webhooks(&self) -> Webhooks { - let webhooks = self.cached_webhooks.read().unwrap_or_else(|poisoned| poisoned.into_inner()); - let mut webhooks = Webhooks::clone(&*webhooks); - if let Some(url) = self.cli_webhook_url.as_ref().cloned() { - let mut headers = BTreeMap::new(); - if let Some(auth) = self.cli_webhook_authorization.as_ref().cloned() { - headers.insert(String::from("Authorization"), auth); - } - webhooks.webhooks.insert(Uuid::nil(), Webhook { url, headers }); - } - webhooks + pub fn webhooks_dump_view(&self) -> WebhooksDumpView { + // We must not dump the cli api key + WebhooksDumpView { webhooks: self.webhooks.get_runtime() } + } + + pub fn webhooks_view(&self) -> WebhooksView { + WebhooksView { webhooks: self.webhooks.get_all() } + } + + pub fn retrieve_runtime_webhooks(&self) -> RuntimeWebhooks { + self.webhooks.get_runtime() } pub fn embedders( @@ -1050,3 +1041,72 @@ pub struct IndexStats { /// Internal stats computed from the index. pub inner_stats: index_mapper::IndexStats, } + +/// These structure are not meant to be exposed to the end user, if needed, use the meilisearch-types::webhooks structure instead. +/// /!\ Everytime you deserialize this structure you should fill the cli_webhook later on with the `with_cli` method. /!\ +#[derive(Debug, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +struct Webhooks { + // The cli webhook should *never* be stored in a database. + // It represent a state that only exists for this execution of meilisearch + #[serde(skip)] + pub cli: Option, + + #[serde(default)] + pub runtime: RwLock, +} + +type RuntimeWebhooks = BTreeMap; + +impl Webhooks { + pub fn with_cli(&mut self, url: Option, auth: Option) { + if let Some(url) = url { + let webhook = CliWebhook { url, auth }; + self.cli = Some(webhook); + } + } + + pub fn from_runtime(webhooks: RuntimeWebhooks) -> Self { + Self { cli: None, runtime: RwLock::new(webhooks) } + } + + pub fn into_runtime(self) -> RuntimeWebhooks { + // safe because we own self and it cannot be cloned + self.runtime.into_inner().unwrap() + } + + pub fn update_runtime(&self, webhooks: RuntimeWebhooks) { + *self.runtime.write().unwrap() = webhooks; + } + + /// Returns all the webhooks in an unified view. The cli webhook is represented with an uuid set to 0 + pub fn get_all(&self) -> BTreeMap { + self.cli + .as_ref() + .map(|wh| (Uuid::nil(), Webhook::from(wh))) + .into_iter() + .chain(self.runtime.read().unwrap().iter().map(|(uuid, wh)| (*uuid, wh.clone()))) + .collect() + } + + /// Returns all the runtime webhooks. + pub fn get_runtime(&self) -> BTreeMap { + self.runtime.read().unwrap().iter().map(|(uuid, wh)| (*uuid, wh.clone())).collect() + } +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, PartialEq)] +struct CliWebhook { + pub url: String, + pub auth: Option, +} + +impl From<&CliWebhook> for Webhook { + fn from(webhook: &CliWebhook) -> Self { + let mut headers = BTreeMap::new(); + if let Some(ref auth) = webhook.auth { + headers.insert("Authorization".to_string(), auth.to_string()); + } + Self { url: webhook.url.to_string(), headers } + } +} diff --git a/crates/index-scheduler/src/scheduler/process_dump_creation.rs b/crates/index-scheduler/src/scheduler/process_dump_creation.rs index 8f47cbd0c..4f3ec0fdd 100644 --- a/crates/index-scheduler/src/scheduler/process_dump_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_dump_creation.rs @@ -272,7 +272,7 @@ impl IndexScheduler { // 7. Dump the webhooks progress.update_progress(DumpCreationProgress::DumpTheWebhooks); - let webhooks = self.webhooks(); + let webhooks = self.webhooks_dump_view(); dump.create_webhooks(webhooks)?; let dump_uid = started_at.format(format_description!( diff --git a/crates/meilisearch-types/src/webhooks.rs b/crates/meilisearch-types/src/webhooks.rs index 0f0741d69..7a35850ab 100644 --- a/crates/meilisearch-types/src/webhooks.rs +++ b/crates/meilisearch-types/src/webhooks.rs @@ -11,9 +11,18 @@ pub struct Webhook { pub headers: BTreeMap, } -#[derive(Debug, Serialize, Deserialize, Default, Clone, PartialEq)] +#[derive(Debug, Serialize, Default, Clone, PartialEq)] #[serde(rename_all = "camelCase")] -pub struct Webhooks { +pub struct WebhooksView { + #[serde(default)] + pub webhooks: BTreeMap, +} + +// Same as the WebhooksView instead it should never contains the CLI webhooks. +// It's the right structure to use in the dump +#[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksDumpView { #[serde(default)] pub webhooks: BTreeMap, } diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 533f0327f..ca9bb6f50 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -493,7 +493,7 @@ fn import_dump( // 2. Import the webhooks if let Some(webhooks) = dump_reader.webhooks() { - index_scheduler.put_webhooks(webhooks.clone())?; + index_scheduler.update_runtime_webhooks(webhooks.webhooks.clone())?; } // 3. Import the `Key`s. diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index e3547996a..7b3275a87 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -146,7 +146,7 @@ pub(super) struct WebhookResults { async fn get_webhooks( index_scheduler: GuardedData, Data>, ) -> Result { - let webhooks = index_scheduler.webhooks(); + let webhooks = index_scheduler.webhooks_view(); let results = webhooks .webhooks .into_iter() @@ -326,7 +326,7 @@ async fn get_webhook( uuid: Path, ) -> Result { let uuid = Uuid::from_str(&uuid.into_inner()).map_err(InvalidUuid)?; - let mut webhooks = index_scheduler.webhooks(); + let mut webhooks = index_scheduler.webhooks_view(); let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; let webhook = WebhookWithMetadata::from(uuid, webhook); @@ -368,8 +368,8 @@ async fn post_webhook( return Err(TooManyHeaders(uuid).into()); } - let mut webhooks = index_scheduler.webhooks(); - if webhooks.webhooks.len() >= 20 { + let mut webhooks = index_scheduler.retrieve_runtime_webhooks(); + if webhooks.len() >= 20 { return Err(TooManyWebhooks.into()); } @@ -383,8 +383,8 @@ async fn post_webhook( }; check_changed(uuid, &webhook)?; - webhooks.webhooks.insert(uuid, webhook.clone()); - index_scheduler.put_webhooks(webhooks)?; + webhooks.insert(uuid, webhook.clone()); + index_scheduler.update_runtime_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::post_webhook(), &req); @@ -426,13 +426,17 @@ async fn patch_webhook( let webhook_settings = webhook_settings.into_inner(); debug!(parameters = ?(uuid, &webhook_settings), "Patch webhook"); - let mut webhooks = index_scheduler.webhooks(); - let old_webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; + if uuid.is_nil() { + return Err(ImmutableWebhook(uuid).into()); + } + + let mut webhooks = index_scheduler.retrieve_runtime_webhooks(); + let old_webhook = webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; let webhook = patch_webhook_inner(&uuid, old_webhook, webhook_settings)?; check_changed(uuid, &webhook)?; - webhooks.webhooks.insert(uuid, webhook.clone()); - index_scheduler.put_webhooks(webhooks)?; + webhooks.insert(uuid, webhook.clone()); + index_scheduler.update_runtime_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::patch_webhook(), &req); @@ -468,9 +472,9 @@ async fn delete_webhook( return Err(ImmutableWebhook(uuid).into()); } - let mut webhooks = index_scheduler.webhooks(); - webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; - index_scheduler.put_webhooks(webhooks)?; + let mut webhooks = index_scheduler.retrieve_runtime_webhooks(); + webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; + index_scheduler.update_runtime_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::delete_webhook(), &req); diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index a3be37a4e..bf2477b25 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -283,7 +283,6 @@ async fn reserved_names() { let (value, code) = server .patch_webhook(Uuid::nil().to_string(), json!({ "url": "http://localhost:8080" })) .await; - snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { "message": "Webhook `[uuid]` is immutable. The webhook defined from the command line cannot be modified using the API.", @@ -292,9 +291,9 @@ async fn reserved_names() { "link": "https://docs.meilisearch.com/errors#immutable_webhook" } "#); + snapshot!(code, @"400 Bad Request"); let (value, code) = server.delete_webhook(Uuid::nil().to_string()).await; - snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { "message": "Webhook `[uuid]` is immutable. The webhook defined from the command line cannot be modified using the API.", @@ -303,6 +302,7 @@ async fn reserved_names() { "link": "https://docs.meilisearch.com/errors#immutable_webhook" } "#); + snapshot!(code, @"400 Bad Request"); } #[actix_web::test] From 1fdf8209317edcdc086a7192f09fb12ce4008746 Mon Sep 17 00:00:00 2001 From: Tamo Date: Wed, 6 Aug 2025 12:12:52 +0200 Subject: [PATCH 285/312] Update version to v1.17.0 --- Cargo.lock | 34 +++++++++---------- Cargo.toml | 2 +- .../after_processing_everything.snap | 4 +-- .../register_automatic_upgrade_task.snap | 2 +- ...sk_while_the_upgrade_task_is_enqueued.snap | 2 +- .../upgrade_failure/upgrade_task_failed.snap | 4 +-- .../upgrade_task_failed_again.snap | 4 +-- .../upgrade_task_succeeded.snap | 4 +-- crates/meilisearch/tests/upgrade/mod.rs | 4 +-- ...rEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...rFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...erStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...rEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...rFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...erStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...ue_once_everything_has_been_processed.snap | 2 +- ...ue_once_everything_has_been_processed.snap | 2 +- crates/milli/src/update/upgrade/mod.rs | 3 ++ crates/milli/src/update/upgrade/v1_16.rs | 19 +++++++++++ 19 files changed, 60 insertions(+), 38 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8413b3d14..6894e4856 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -580,7 +580,7 @@ source = "git+https://github.com/meilisearch/bbqueue#cbb87cc707b5af415ef203bdaf2 [[package]] name = "benchmarks" -version = "1.16.0" +version = "1.17.0" dependencies = [ "anyhow", "bumpalo", @@ -770,7 +770,7 @@ dependencies = [ [[package]] name = "build-info" -version = "1.16.0" +version = "1.17.0" dependencies = [ "anyhow", "time", @@ -1774,7 +1774,7 @@ dependencies = [ [[package]] name = "dump" -version = "1.16.0" +version = "1.17.0" dependencies = [ "anyhow", "big_s", @@ -2006,7 +2006,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-store" -version = "1.16.0" +version = "1.17.0" dependencies = [ "tempfile", "thiserror 2.0.12", @@ -2028,7 +2028,7 @@ dependencies = [ [[package]] name = "filter-parser" -version = "1.16.0" +version = "1.17.0" dependencies = [ "insta", "nom", @@ -2049,7 +2049,7 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "1.16.0" +version = "1.17.0" dependencies = [ "criterion", "serde_json", @@ -2194,7 +2194,7 @@ dependencies = [ [[package]] name = "fuzzers" -version = "1.16.0" +version = "1.17.0" dependencies = [ "arbitrary", "bumpalo", @@ -2994,7 +2994,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "1.16.0" +version = "1.17.0" dependencies = [ "anyhow", "backoff", @@ -3230,7 +3230,7 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "1.16.0" +version = "1.17.0" dependencies = [ "criterion", "serde_json", @@ -3724,7 +3724,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "1.16.0" +version = "1.17.0" dependencies = [ "insta", "md5", @@ -3735,7 +3735,7 @@ dependencies = [ [[package]] name = "meilisearch" -version = "1.16.0" +version = "1.17.0" dependencies = [ "actix-cors", "actix-http", @@ -3831,7 +3831,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "1.16.0" +version = "1.17.0" dependencies = [ "base64 0.22.1", "enum-iterator", @@ -3850,7 +3850,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "1.16.0" +version = "1.17.0" dependencies = [ "actix-web", "anyhow", @@ -3885,7 +3885,7 @@ dependencies = [ [[package]] name = "meilitool" -version = "1.16.0" +version = "1.17.0" dependencies = [ "anyhow", "clap", @@ -3919,7 +3919,7 @@ dependencies = [ [[package]] name = "milli" -version = "1.16.0" +version = "1.17.0" dependencies = [ "allocator-api2 0.3.0", "arroy", @@ -4471,7 +4471,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "permissive-json-pointer" -version = "1.16.0" +version = "1.17.0" dependencies = [ "big_s", "serde_json", @@ -7259,7 +7259,7 @@ dependencies = [ [[package]] name = "xtask" -version = "1.16.0" +version = "1.17.0" dependencies = [ "anyhow", "build-info", diff --git a/Cargo.toml b/Cargo.toml index 3e57563b6..1fa86c671 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ members = [ ] [workspace.package] -version = "1.16.0" +version = "1.17.0" authors = [ "Quentin de Quelen ", "Clément Renault ", diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index 0b5d4409d..0b87ffa2c 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 16, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} } [timestamp] [4,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.16.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap index 0bfb9c6da..d0444d3ab 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 16, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index 8d374479b..0533b162a 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 16, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index 9fc28abbe..8a52efde5 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 16, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: @@ -37,7 +37,7 @@ catto [1,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.16.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index 33ddf7193..40faa3504 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 16, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- @@ -40,7 +40,7 @@ doggo [2,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.16.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index 05d366d1e..8fe59f5c4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 16, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -43,7 +43,7 @@ doggo [2,3,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.16.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/meilisearch/tests/upgrade/mod.rs b/crates/meilisearch/tests/upgrade/mod.rs index 8114ed58b..5585e50c4 100644 --- a/crates/meilisearch/tests/upgrade/mod.rs +++ b/crates/meilisearch/tests/upgrade/mod.rs @@ -43,7 +43,7 @@ async fn version_too_old() { std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.16.0"); + snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.17.0"); } #[actix_rt::test] @@ -58,7 +58,7 @@ async fn version_requires_downgrade() { std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.16.1 is higher than the Meilisearch version 1.16.0. Downgrade is not supported"); + snapshot!(err, @"Database version 1.17.1 is higher than the Meilisearch version 1.17.0. Downgrade is not supported"); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index b56cc5ca3..c7bcd543f 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.16.0" + "upgradeTo": "v1.17.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index b56cc5ca3..c7bcd543f 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.16.0" + "upgradeTo": "v1.17.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index b56cc5ca3..c7bcd543f 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.16.0" + "upgradeTo": "v1.17.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index a52072f56..8674d2c24 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.16.0" + "upgradeTo": "v1.17.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index a52072f56..8674d2c24 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.16.0" + "upgradeTo": "v1.17.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index a52072f56..8674d2c24 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.16.0" + "upgradeTo": "v1.17.0" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap index 81b50fb92..842e3f536 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.16.0" + "upgradeTo": "v1.17.0" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap index 1ec334fed..18d5b4e20 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.16.0" + "upgradeTo": "v1.17.0" }, "error": null, "duration": "[duration]", diff --git a/crates/milli/src/update/upgrade/mod.rs b/crates/milli/src/update/upgrade/mod.rs index f53319a37..ecd1cec6c 100644 --- a/crates/milli/src/update/upgrade/mod.rs +++ b/crates/milli/src/update/upgrade/mod.rs @@ -8,6 +8,7 @@ use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3}; use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13}; use v1_14::Latest_V1_13_To_Latest_V1_14; use v1_15::Latest_V1_14_To_Latest_V1_15; +use v1_16::Latest_V1_16_To_V1_17_0; use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH}; use crate::progress::{Progress, VariableNameStep}; @@ -34,6 +35,7 @@ const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[ &Latest_V1_13_To_Latest_V1_14 {}, &Latest_V1_14_To_Latest_V1_15 {}, &Latest_V1_15_To_V1_16_0 {}, + &Latest_V1_16_To_V1_17_0 {}, // This is the last upgrade function, it will be called when the index is up to date. // any other upgrade function should be added before this one. &ToCurrentNoOp {}, @@ -62,6 +64,7 @@ const fn start(from: (u32, u32, u32)) -> Option { // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other. (1, 15, _) => function_index!(6), (1, 16, _) => function_index!(7), + (1, 17, _) => function_index!(8), // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually // considering dumpless upgrade. (_major, _minor, _patch) => return None, diff --git a/crates/milli/src/update/upgrade/v1_16.rs b/crates/milli/src/update/upgrade/v1_16.rs index f43efd77d..02dd136ce 100644 --- a/crates/milli/src/update/upgrade/v1_16.rs +++ b/crates/milli/src/update/upgrade/v1_16.rs @@ -46,3 +46,22 @@ impl UpgradeIndex for Latest_V1_15_To_V1_16_0 { (1, 16, 0) } } + +#[allow(non_camel_case_types)] +pub(super) struct Latest_V1_16_To_V1_17_0(); + +impl UpgradeIndex for Latest_V1_16_To_V1_17_0 { + fn upgrade( + &self, + _wtxn: &mut RwTxn, + _index: &Index, + _original: (u32, u32, u32), + _progress: Progress, + ) -> Result { + Ok(false) + } + + fn target_version(&self) -> (u32, u32, u32) { + (1, 17, 0) + } +} From e302e9edd392ef3901ed96607dfffa829b8c3b74 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 6 Aug 2025 15:02:15 +0200 Subject: [PATCH 286/312] Add test for task --- crates/meilisearch/tests/common/index.rs | 5 ++ crates/meilisearch/tests/settings/chat.rs | 66 +++++++++++++++++++++++ crates/meilisearch/tests/settings/mod.rs | 1 + 3 files changed, 72 insertions(+) create mode 100644 crates/meilisearch/tests/settings/chat.rs diff --git a/crates/meilisearch/tests/common/index.rs b/crates/meilisearch/tests/common/index.rs index bb1506022..012c9bebe 100644 --- a/crates/meilisearch/tests/common/index.rs +++ b/crates/meilisearch/tests/common/index.rs @@ -249,6 +249,11 @@ impl<'a> Index<'a, Owned> { self.service.put_encoded(url, settings, self.encoder).await } + pub async fn update_settings_chat(&self, settings: Value) -> (Value, StatusCode) { + let url = format!("/indexes/{}/settings/chat", urlencode(self.uid.as_ref())); + self.service.patch_encoded(url, settings, self.encoder).await + } + pub async fn delete_settings(&self) -> (Value, StatusCode) { let url = format!("/indexes/{}/settings", urlencode(self.uid.as_ref())); self.service.delete(url).await diff --git a/crates/meilisearch/tests/settings/chat.rs b/crates/meilisearch/tests/settings/chat.rs new file mode 100644 index 000000000..891a22431 --- /dev/null +++ b/crates/meilisearch/tests/settings/chat.rs @@ -0,0 +1,66 @@ +use crate::common::Server; +use crate::json; +use meili_snap::{json_string, snapshot}; + +#[actix_rt::test] +async fn set_reset_chat_issue_5772() { + let server = Server::new().await; + let index = server.unique_index(); + + let (_, code) = server + .set_features(json!({ + "chatCompletions": true, + })) + .await; + snapshot!(code, @r#"200 OK"#); + + let (task1, _code) = index.update_settings_chat(json!({ + "description": "test!", + "documentTemplate": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}", + "documentTemplateMaxBytes": 400, + "searchParameters": { + "limit": 15, + "sort": [], + "attributesToSearchOn": [] + } + })).await; + server.wait_task(task1.uid()).await.succeeded(); + + let (response, _) = index.settings().await; + snapshot!(json_string!(response["chat"]), @r#" + { + "description": "test!", + "documentTemplate": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}", + "documentTemplateMaxBytes": 400, + "searchParameters": { + "limit": 15, + "sort": [], + "attributesToSearchOn": [] + } + } + "#); + + let (task2, _status_code) = index.update_settings_chat(json!({ + "description": "test!", + "documentTemplate": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}", + "documentTemplateMaxBytes": 400, + "searchParameters": { + "limit": 16 + } + })).await; + server.wait_task(task2.uid()).await.succeeded(); + + let (response, _) = index.settings().await; + snapshot!(json_string!(response["chat"]), @r#" + { + "description": "test!", + "documentTemplate": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}", + "documentTemplateMaxBytes": 400, + "searchParameters": { + "limit": 16, + "sort": [], + "attributesToSearchOn": [] + } + } + "#); +} diff --git a/crates/meilisearch/tests/settings/mod.rs b/crates/meilisearch/tests/settings/mod.rs index 6b61e6be0..b3a956c25 100644 --- a/crates/meilisearch/tests/settings/mod.rs +++ b/crates/meilisearch/tests/settings/mod.rs @@ -1,3 +1,4 @@ +mod chat; mod distinct; mod errors; mod get_settings; From 3ead985cafe2b4e095fd97f720f11441544c54ed Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 6 Aug 2025 15:02:25 +0200 Subject: [PATCH 287/312] Fix issue #5772 --- crates/meilisearch/src/routes/indexes/settings.rs | 2 +- crates/meilisearch/tests/settings/get_settings.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/src/routes/indexes/settings.rs b/crates/meilisearch/src/routes/indexes/settings.rs index 308977a6e..10120ebff 100644 --- a/crates/meilisearch/src/routes/indexes/settings.rs +++ b/crates/meilisearch/src/routes/indexes/settings.rs @@ -511,7 +511,7 @@ make_setting_routes!( }, { route: "/chat", - update_verb: put, + update_verb: patch, value_type: ChatSettings, err_type: meilisearch_types::deserr::DeserrJsonError< meilisearch_types::error::deserr_codes::InvalidSettingsIndexChat, diff --git a/crates/meilisearch/tests/settings/get_settings.rs b/crates/meilisearch/tests/settings/get_settings.rs index f50f7f940..8419f640d 100644 --- a/crates/meilisearch/tests/settings/get_settings.rs +++ b/crates/meilisearch/tests/settings/get_settings.rs @@ -186,7 +186,7 @@ test_setting_routes!( }, { setting: chat, - update_verb: put, + update_verb: patch, default_value: { "description": "", "documentTemplate": "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}", From 74992560b00bb34a0f369167b0f9447696072f19 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 7 Aug 2025 09:28:45 +0200 Subject: [PATCH 288/312] Simplify conditions --- crates/meilisearch/src/search/mod.rs | 2 +- crates/meilisearch/tests/search/hybrid.rs | 12 ++++++++++++ crates/milli/src/search/hybrid.rs | 7 +------ crates/milli/src/search/mod.rs | 24 +---------------------- 4 files changed, 15 insertions(+), 30 deletions(-) diff --git a/crates/meilisearch/src/search/mod.rs b/crates/meilisearch/src/search/mod.rs index c681a11a5..bb406aed9 100644 --- a/crates/meilisearch/src/search/mod.rs +++ b/crates/meilisearch/src/search/mod.rs @@ -1020,7 +1020,7 @@ pub fn prepare_search<'t>( .map_err(milli::Error::from)? } }; - search.semantic_auto_embedded( + search.semantic( embedder_name.clone(), embedder.clone(), *quantized, diff --git a/crates/meilisearch/tests/search/hybrid.rs b/crates/meilisearch/tests/search/hybrid.rs index 172242e47..b2970f233 100644 --- a/crates/meilisearch/tests/search/hybrid.rs +++ b/crates/meilisearch/tests/search/hybrid.rs @@ -201,6 +201,10 @@ async fn simple_search() { } ], "query": "Captain", + "queryVector": [ + 1.0, + 1.0 + ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, @@ -272,6 +276,10 @@ async fn simple_search() { } ], "query": "Captain", + "queryVector": [ + 1.0, + 1.0 + ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, @@ -343,6 +351,10 @@ async fn simple_search() { } ], "query": "Captain", + "queryVector": [ + 1.0, + 1.0 + ], "processingTimeMs": "[duration]", "limit": 20, "offset": 0, diff --git a/crates/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs index 75ac547b4..c4d440043 100644 --- a/crates/milli/src/search/hybrid.rs +++ b/crates/milli/src/search/hybrid.rs @@ -234,7 +234,6 @@ impl Search<'_> { // no embedder, no semantic search let Some(SemanticSearch { vector, - mut auto_embedded, embedder_name, embedder, quantized, @@ -262,10 +261,7 @@ impl Search<'_> { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); match embedder.embed_search(query, Some(deadline)) { - Ok(embedding) => { - auto_embedded = true; - embedding - } + Ok(embedding) => embedding, Err(error) => { tracing::error!(error=%error, "Embedding failed"); return Ok(return_keyword_results( @@ -280,7 +276,6 @@ impl Search<'_> { search.semantic = Some(SemanticSearch { vector: Some(vector_query.clone()), - auto_embedded, embedder_name, embedder, quantized, diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index e7871031b..155a0ecf7 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -33,7 +33,6 @@ pub mod similar; #[derive(Debug, Clone)] pub struct SemanticSearch { vector: Option>, - auto_embedded: bool, media: Option, embedder_name: String, embedder: Arc, @@ -105,26 +104,6 @@ impl<'a> Search<'a> { ) -> &mut Search<'a> { self.semantic = Some(SemanticSearch { embedder_name, - auto_embedded: false, - embedder, - quantized, - vector, - media, - }); - self - } - - pub fn semantic_auto_embedded( - &mut self, - embedder_name: String, - embedder: Arc, - quantized: bool, - vector: Option, - media: Option, - ) -> &mut Search<'a> { - self.semantic = Some(SemanticSearch { - embedder_name, - auto_embedded: true, embedder, quantized, vector, @@ -278,13 +257,12 @@ impl<'a> Search<'a> { } = match self.semantic.as_ref() { Some(SemanticSearch { vector: Some(vector), - auto_embedded, embedder_name, embedder, quantized, media: _, }) => { - if *auto_embedded && self.retrieve_vectors { + if self.retrieve_vectors { query_vector = Some(vector.clone()); } execute_vector_search( From 5df125cbb78fdaf82b5941ad1f6bb2d968382c09 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 7 Aug 2025 09:31:05 +0200 Subject: [PATCH 289/312] Format --- crates/milli/src/search/hybrid.rs | 8 +------- crates/milli/src/search/mod.rs | 8 +------- 2 files changed, 2 insertions(+), 14 deletions(-) diff --git a/crates/milli/src/search/hybrid.rs b/crates/milli/src/search/hybrid.rs index c4d440043..1535c73ba 100644 --- a/crates/milli/src/search/hybrid.rs +++ b/crates/milli/src/search/hybrid.rs @@ -232,13 +232,7 @@ impl Search<'_> { } // no embedder, no semantic search - let Some(SemanticSearch { - vector, - embedder_name, - embedder, - quantized, - media, - }) = semantic + let Some(SemanticSearch { vector, embedder_name, embedder, quantized, media }) = semantic else { return Ok(return_keyword_results(self.limit, self.offset, keyword_results)); }; diff --git a/crates/milli/src/search/mod.rs b/crates/milli/src/search/mod.rs index 155a0ecf7..2ae931ff5 100644 --- a/crates/milli/src/search/mod.rs +++ b/crates/milli/src/search/mod.rs @@ -102,13 +102,7 @@ impl<'a> Search<'a> { vector: Option, media: Option, ) -> &mut Search<'a> { - self.semantic = Some(SemanticSearch { - embedder_name, - embedder, - quantized, - vector, - media, - }); + self.semantic = Some(SemanticSearch { embedder_name, embedder, quantized, vector, media }); self } From 759beed5604c869c9db428308f788adba0e00f6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine?= Date: Thu, 7 Aug 2025 18:15:29 +0200 Subject: [PATCH 290/312] Add category in release draft --- .github/release-draft-template.yml | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/release-draft-template.yml b/.github/release-draft-template.yml index 1088be33b..8d687c1ef 100644 --- a/.github/release-draft-template.yml +++ b/.github/release-draft-template.yml @@ -3,13 +3,23 @@ tag-template: 'v$RESOLVED_VERSION' exclude-labels: - 'skip changelog' version-resolver: - major: - labels: - - 'breaking-change' minor: labels: - 'enhancement' default: patch +categories: + - title: '⚠️ Breaking changes' + label: 'breaking-change' + - title: '🚀 Enhancements' + label: 'enhancement' + - title: '🐛 Bug Fixes' + label: 'bug' + - title: '🔒 Security' + label: 'security' + - title: '⚙️ Maintenance/misc' + label: + - 'maintenance' + - 'documentation' template: | $CHANGES From b265c92852451540de52427d8e2ce7a2ea15671b Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 12:17:10 +0200 Subject: [PATCH 291/312] Thank contributors better --- .github/release-draft-template.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/release-draft-template.yml b/.github/release-draft-template.yml index 8d687c1ef..ffe2fa5b7 100644 --- a/.github/release-draft-template.yml +++ b/.github/release-draft-template.yml @@ -23,7 +23,7 @@ categories: template: | $CHANGES - Thanks again to $CONTRIBUTORS! 🎉 + ❤️ Huge thanks to our contributors: $CONTRIBUTORS. no-changes-template: 'Changes are coming soon 😎' sort-direction: 'ascending' replacers: From 68280bad9ecad53e031df8c1e120e02b116c5ca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine?= Date: Mon, 11 Aug 2025 14:28:38 +0200 Subject: [PATCH 292/312] Minor update for release process --- documentation/release.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/documentation/release.md b/documentation/release.md index 69ce32aff..5f08e8f5a 100644 --- a/documentation/release.md +++ b/documentation/release.md @@ -22,7 +22,7 @@ This guide is to describe how to make releases for the current repository. 2. Go to the GitHub interface, in the [`Release` section](https://github.com/meilisearch/meilisearch/releases). 3. Select the already drafted release or click on the `Draft a new release` button if you want to start a blank one, and fill the form with the appropriate information. -⚠️ Publish on `main` +⚠️ Publish on a specific commit defined by the team. Or publish on `main`, but ensure you do want all the PRs merged in your release. ⚙️ The CIs will be triggered to: - [Upload binaries](https://github.com/meilisearch/meilisearch/actions/workflows/publish-binaries.yml) to the associated GitHub release. @@ -31,7 +31,7 @@ This guide is to describe how to make releases for the current repository. - [Move the `latest` git tag to the release commit](https://github.com/meilisearch/meilisearch/actions/workflows/latest-git-tag.yml). -### 🔥 How to do a patch release for an hotfix +### 🔥 How to do a patch release for a hotfix It happens some releases come with impactful bugs in production (e.g. indexation or search issues): we obviously don't wait for the next cycle to fix them and we release a patched version of Meilisearch. From 562c620fec298b68f654b48b1aacdb994fa131ce Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 11 Aug 2025 16:21:14 +0200 Subject: [PATCH 293/312] Update webhook telemetry events --- crates/meilisearch/src/routes/webhooks.rs | 53 ++++++++++------------- 1 file changed, 22 insertions(+), 31 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 7b3275a87..b25b19336 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -159,37 +159,32 @@ async fn get_webhooks( } #[derive(Serialize, Default)] -pub struct PatchWebhooksAnalytics { - patch_webhook_count: usize, - post_webhook_count: usize, - delete_webhook_count: usize, -} - -impl PatchWebhooksAnalytics { - pub fn patch_webhook() -> Self { - PatchWebhooksAnalytics { patch_webhook_count: 1, ..Default::default() } - } - - pub fn post_webhook() -> Self { - PatchWebhooksAnalytics { post_webhook_count: 1, ..Default::default() } - } - - pub fn delete_webhook() -> Self { - PatchWebhooksAnalytics { delete_webhook_count: 1, ..Default::default() } - } -} +pub struct PatchWebhooksAnalytics; impl Aggregate for PatchWebhooksAnalytics { fn event_name(&self) -> &'static str { "Webhooks Updated" } - fn aggregate(self: Box, new: Box) -> Box { - Box::new(PatchWebhooksAnalytics { - patch_webhook_count: self.patch_webhook_count + new.patch_webhook_count, - post_webhook_count: self.post_webhook_count + new.post_webhook_count, - delete_webhook_count: self.delete_webhook_count + new.delete_webhook_count, - }) + fn aggregate(self: Box, _new: Box) -> Box { + self + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(*self).unwrap_or_default() + } +} + +#[derive(Serialize, Default)] +pub struct PostWebhooksAnalytics; + +impl Aggregate for PostWebhooksAnalytics { + fn event_name(&self) -> &'static str { + "Webhooks Created" + } + + fn aggregate(self: Box, _new: Box) -> Box { + self } fn into_event(self: Box) -> serde_json::Value { @@ -386,7 +381,7 @@ async fn post_webhook( webhooks.insert(uuid, webhook.clone()); index_scheduler.update_runtime_webhooks(webhooks)?; - analytics.publish(PatchWebhooksAnalytics::post_webhook(), &req); + analytics.publish(PostWebhooksAnalytics, &req); let response = WebhookWithMetadata::from(uuid, webhook); debug!(returns = ?response, "Post webhook"); @@ -438,7 +433,7 @@ async fn patch_webhook( webhooks.insert(uuid, webhook.clone()); index_scheduler.update_runtime_webhooks(webhooks)?; - analytics.publish(PatchWebhooksAnalytics::patch_webhook(), &req); + analytics.publish(PatchWebhooksAnalytics, &req); let response = WebhookWithMetadata::from(uuid, webhook); debug!(returns = ?response, "Patch webhook"); @@ -462,8 +457,6 @@ async fn patch_webhook( async fn delete_webhook( index_scheduler: GuardedData, Data>, uuid: Path, - req: HttpRequest, - analytics: Data, ) -> Result { let uuid = Uuid::from_str(&uuid.into_inner()).map_err(InvalidUuid)?; debug!(parameters = ?uuid, "Delete webhook"); @@ -476,8 +469,6 @@ async fn delete_webhook( webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; index_scheduler.update_runtime_webhooks(webhooks)?; - analytics.publish(PatchWebhooksAnalytics::delete_webhook(), &req); - debug!(returns = "No Content", "Delete webhook"); Ok(HttpResponse::NoContent().finish()) } From 54b85b86445a9060e52db7d1bdd1ee794f3c6614 Mon Sep 17 00:00:00 2001 From: Tamo Date: Mon, 11 Aug 2025 16:37:09 +0200 Subject: [PATCH 294/312] fix the dumpless upgrade again --- crates/index-scheduler/src/upgrade/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/index-scheduler/src/upgrade/mod.rs b/crates/index-scheduler/src/upgrade/mod.rs index 2053caa92..a749b31d5 100644 --- a/crates/index-scheduler/src/upgrade/mod.rs +++ b/crates/index-scheduler/src/upgrade/mod.rs @@ -39,6 +39,7 @@ pub fn upgrade_index_scheduler( (1, 13, _) => 0, (1, 14, _) => 0, (1, 15, _) => 0, + (1, 16, _) => 0, (major, minor, patch) => { if major > current_major || (major == current_major && minor > current_minor) From 0881810780c0398645f1d09b7b2b603f1068ee05 Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 18:09:54 +0200 Subject: [PATCH 295/312] Add CI to publish OpenAPI file --- ...inaries.yml => publish-release-assets.yml} | 30 ++++++++++++- .gitignore | 3 ++ Cargo.lock | 11 +++++ Cargo.toml | 1 + crates/openapi-generator/Cargo.toml | 12 ++++++ crates/openapi-generator/src/main.rs | 42 +++++++++++++++++++ 6 files changed, 98 insertions(+), 1 deletion(-) rename .github/workflows/{publish-binaries.yml => publish-release-assets.yml} (87%) create mode 100644 crates/openapi-generator/Cargo.toml create mode 100644 crates/openapi-generator/src/main.rs diff --git a/.github/workflows/publish-binaries.yml b/.github/workflows/publish-release-assets.yml similarity index 87% rename from .github/workflows/publish-binaries.yml rename to .github/workflows/publish-release-assets.yml index 27d8c3610..204480887 100644 --- a/.github/workflows/publish-binaries.yml +++ b/.github/workflows/publish-release-assets.yml @@ -1,4 +1,4 @@ -name: Publish binaries to GitHub release +name: Publish assets to GitHub release on: workflow_dispatch: @@ -184,3 +184,31 @@ jobs: file: target/${{ matrix.target }}/release/meilisearch asset_name: ${{ matrix.asset_name }} tag: ${{ github.ref }} + + publish-openapi: + name: Publish OpenAPI file + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Setup Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + + - name: Generate OpenAPI specification + run: | + cd crates/openapi-generator + cargo run --release -- --pretty --output ../../meilisearch.json + + - name: Upload OpenAPI to Release + # No need to upload for dry run (cron) + if: github.event_name == 'release' + uses: svenstaro/upload-release-action@2.11.2 + with: + repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} + file: ./meilisearch.json + asset_name: meilisearch-openapi.json + tag: ${{ github.ref }} diff --git a/.gitignore b/.gitignore index 44cfa8f75..d9a945b88 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ crates/meilisearch/db.snapshot # Fuzzcheck data for the facet indexing fuzz test crates/milli/fuzz/update::facet::incremental::fuzz::fuzz/ + +# OpenAPI +meilisearch.json diff --git a/Cargo.lock b/Cargo.lock index 6894e4856..061cefb3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4338,6 +4338,17 @@ version = "11.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" +[[package]] +name = "openapi-generator" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap", + "meilisearch", + "serde_json", + "utoipa", +] + [[package]] name = "openssl-probe" version = "0.1.6" diff --git a/Cargo.toml b/Cargo.toml index 1fa86c671..dd2747c5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "crates/tracing-trace", "crates/xtask", "crates/build-info", + "crates/openapi-generator", ] [workspace.package] diff --git a/crates/openapi-generator/Cargo.toml b/crates/openapi-generator/Cargo.toml new file mode 100644 index 000000000..87c41625e --- /dev/null +++ b/crates/openapi-generator/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "openapi-generator" +version = "0.1.0" +edition = "2021" +publish = false + +[dependencies] +meilisearch = { path = "../meilisearch" } +serde_json = "1.0" +clap = { version = "4.5.40", features = ["derive"] } +anyhow = "1.0.98" +utoipa = "5.4.0" \ No newline at end of file diff --git a/crates/openapi-generator/src/main.rs b/crates/openapi-generator/src/main.rs new file mode 100644 index 000000000..5630a6fea --- /dev/null +++ b/crates/openapi-generator/src/main.rs @@ -0,0 +1,42 @@ +use anyhow::Result; +use clap::Parser; +use meilisearch::routes::MeilisearchApi; +use utoipa::OpenApi; +use std::path::PathBuf; + +#[derive(Parser)] +#[command(name = "openapi-generator")] +#[command(about = "Generate OpenAPI specification for Meilisearch")] +struct Cli { + /// Output file path (default: meilisearch.json) + #[arg(short, long, value_name = "FILE")] + output: Option, + + /// Pretty print the JSON output + #[arg(short, long)] + pretty: bool, +} + +fn main() -> Result<()> { + let cli = Cli::parse(); + + // Generate the OpenAPI specification + let openapi = MeilisearchApi::openapi(); + + // Determine output path + let output_path = cli.output.unwrap_or_else(|| PathBuf::from("meilisearch.json")); + + // Serialize to JSON + let json = if cli.pretty { + serde_json::to_string_pretty(&openapi)? + } else { + serde_json::to_string(&openapi)? + }; + + // Write to file + std::fs::write(&output_path, json)?; + + println!("OpenAPI specification written to: {}", output_path.display()); + + Ok(()) +} From 3c583ce7a4f23d86aa8748bfa930c9cdffa45368 Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 18:10:39 +0200 Subject: [PATCH 296/312] Fix linting --- crates/openapi-generator/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/openapi-generator/Cargo.toml b/crates/openapi-generator/Cargo.toml index 87c41625e..652f6fc57 100644 --- a/crates/openapi-generator/Cargo.toml +++ b/crates/openapi-generator/Cargo.toml @@ -9,4 +9,4 @@ meilisearch = { path = "../meilisearch" } serde_json = "1.0" clap = { version = "4.5.40", features = ["derive"] } anyhow = "1.0.98" -utoipa = "5.4.0" \ No newline at end of file +utoipa = "5.4.0" From 100a6f96e45f737e273860fab8e7a15134576de4 Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 18:11:23 +0200 Subject: [PATCH 297/312] Minor change --- .github/workflows/publish-release-assets.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-release-assets.yml b/.github/workflows/publish-release-assets.yml index 204480887..f9db0c91b 100644 --- a/.github/workflows/publish-release-assets.yml +++ b/.github/workflows/publish-release-assets.yml @@ -185,7 +185,7 @@ jobs: asset_name: ${{ matrix.asset_name }} tag: ${{ github.ref }} - publish-openapi: + publish-openapi-file: name: Publish OpenAPI file runs-on: ubuntu-latest steps: From c5b325de30efa1ce3d9d3e784af362d1c45781d3 Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 18:15:42 +0200 Subject: [PATCH 298/312] Fix rustfmt --- crates/openapi-generator/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/openapi-generator/src/main.rs b/crates/openapi-generator/src/main.rs index 5630a6fea..a6196f771 100644 --- a/crates/openapi-generator/src/main.rs +++ b/crates/openapi-generator/src/main.rs @@ -1,8 +1,9 @@ +use std::path::PathBuf; + use anyhow::Result; use clap::Parser; use meilisearch::routes::MeilisearchApi; use utoipa::OpenApi; -use std::path::PathBuf; #[derive(Parser)] #[command(name = "openapi-generator")] From 1f126a2d8a90960051695e9ee850b3c1bfb54e68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9mentine?= Date: Mon, 11 Aug 2025 18:23:37 +0200 Subject: [PATCH 299/312] Update release doc (again) --- documentation/release.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/documentation/release.md b/documentation/release.md index 69ce32aff..14a10f77b 100644 --- a/documentation/release.md +++ b/documentation/release.md @@ -4,10 +4,11 @@ This guide is to describe how to make releases for the current repository. ## 📅 Weekly Meilisearch release -1. A weekly meeting is done every Monday to define the release and to ensure minimal checks before the release. +1. A weekly meeting is held every Thursday afternoon to define the release and to ensure minimal checks before the release.
Check out the TODO 👇👇👇 - [ ] Define the version of the release (`vX.Y.Z`) based on our Versioning Policy
. +- [ ] Define the commit that will reference the tag release. Every PR merged after this commit will not be taken into account in the future release - [ ] Manually test `--experimental-dumpless-upgrade` on a DB of the previous Meilisearch minor version
- [ ] Check recent automated tests on `main`
- [ ] Scheduled test suite
From a69af611e30565f53f35a78f08d6986f4fe11615 Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 18:29:52 +0200 Subject: [PATCH 300/312] Add documentation --- .gitignore | 3 --- CONTRIBUTING.md | 12 +++++++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index d9a945b88..44cfa8f75 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,3 @@ crates/meilisearch/db.snapshot # Fuzzcheck data for the facet indexing fuzz test crates/milli/fuzz/update::facet::incremental::fuzz::fuzz/ - -# OpenAPI -meilisearch.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 72a91a765..7f718c899 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -107,12 +107,18 @@ Run `cargo xtask --help` from the root of the repository to find out what is ava To update the openAPI file in the code, see [sprint_issue.md](https://github.com/meilisearch/meilisearch/blob/main/.github/ISSUE_TEMPLATE/sprint_issue.md#reminders-when-modifying-the-api). -If you want to update the openAPI file on the [open-api repository](https://github.com/meilisearch/open-api): -- Pull the latest version of the latest rc of Meilisearch `git checkout release-vX.Y.Z; git pull` +If you want to generate OpenAPI file manually: + +With swagger: - Starts Meilisearch with the `swagger` feature flag: `cargo run --features swagger` - On a browser, open the following URL: http://localhost:7700/scalar - Click the « Download openAPI file » -- Open a PR replacing [this file](https://github.com/meilisearch/open-api/blob/main/open-api.json) with the one downloaded + +With the internal crate: +```bash +cd crates/openapi-generator +cargo run --release -- --pretty --output meilisearch.json +``` ### Logging From 3c84010403607a63e149ebd67d0c2d67c6e5ddb1 Mon Sep 17 00:00:00 2001 From: curquiza Date: Mon, 11 Aug 2025 18:31:30 +0200 Subject: [PATCH 301/312] Minor change in CI manifest --- .github/workflows/publish-release-assets.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/publish-release-assets.yml b/.github/workflows/publish-release-assets.yml index f9db0c91b..cb70b23fc 100644 --- a/.github/workflows/publish-release-assets.yml +++ b/.github/workflows/publish-release-assets.yml @@ -191,18 +191,15 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - - name: Setup Rust uses: actions-rs/toolchain@v1 with: toolchain: stable override: true - - - name: Generate OpenAPI specification + - name: Generate OpenAPI file run: | cd crates/openapi-generator cargo run --release -- --pretty --output ../../meilisearch.json - - name: Upload OpenAPI to Release # No need to upload for dry run (cron) if: github.event_name == 'release' From 14b1a3300bb3a94fd55e9b5031ddcec2772a172e Mon Sep 17 00:00:00 2001 From: curquiza Date: Tue, 12 Aug 2025 10:07:34 +0200 Subject: [PATCH 302/312] Fix indentation --- .github/workflows/publish-release-assets.yml | 48 ++++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/publish-release-assets.yml b/.github/workflows/publish-release-assets.yml index cb70b23fc..ec0d36711 100644 --- a/.github/workflows/publish-release-assets.yml +++ b/.github/workflows/publish-release-assets.yml @@ -185,27 +185,27 @@ jobs: asset_name: ${{ matrix.asset_name }} tag: ${{ github.ref }} - publish-openapi-file: - name: Publish OpenAPI file - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v4 - - name: Setup Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - - name: Generate OpenAPI file - run: | - cd crates/openapi-generator - cargo run --release -- --pretty --output ../../meilisearch.json - - name: Upload OpenAPI to Release - # No need to upload for dry run (cron) - if: github.event_name == 'release' - uses: svenstaro/upload-release-action@2.11.2 - with: - repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} - file: ./meilisearch.json - asset_name: meilisearch-openapi.json - tag: ${{ github.ref }} + publish-openapi-file: + name: Publish OpenAPI file + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Setup Rust + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + - name: Generate OpenAPI file + run: | + cd crates/openapi-generator + cargo run --release -- --pretty --output ../../meilisearch.json + - name: Upload OpenAPI to Release + # No need to upload for dry run (cron) + if: github.event_name == 'release' + uses: svenstaro/upload-release-action@2.11.2 + with: + repo_token: ${{ secrets.MEILI_BOT_GH_PAT }} + file: ./meilisearch.json + asset_name: meilisearch-openapi.json + tag: ${{ github.ref }} From de52fe91f5a1b815c90ea4017f2504f964bf75a0 Mon Sep 17 00:00:00 2001 From: curquiza Date: Tue, 12 Aug 2025 14:52:50 +0200 Subject: [PATCH 303/312] Update version --- Cargo.lock | 34 +++++++++++++++++----------------- Cargo.toml | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6894e4856..28a51d495 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -580,7 +580,7 @@ source = "git+https://github.com/meilisearch/bbqueue#cbb87cc707b5af415ef203bdaf2 [[package]] name = "benchmarks" -version = "1.17.0" +version = "1.17.1" dependencies = [ "anyhow", "bumpalo", @@ -770,7 +770,7 @@ dependencies = [ [[package]] name = "build-info" -version = "1.17.0" +version = "1.17.1" dependencies = [ "anyhow", "time", @@ -1774,7 +1774,7 @@ dependencies = [ [[package]] name = "dump" -version = "1.17.0" +version = "1.17.1" dependencies = [ "anyhow", "big_s", @@ -2006,7 +2006,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "file-store" -version = "1.17.0" +version = "1.17.1" dependencies = [ "tempfile", "thiserror 2.0.12", @@ -2028,7 +2028,7 @@ dependencies = [ [[package]] name = "filter-parser" -version = "1.17.0" +version = "1.17.1" dependencies = [ "insta", "nom", @@ -2049,7 +2049,7 @@ dependencies = [ [[package]] name = "flatten-serde-json" -version = "1.17.0" +version = "1.17.1" dependencies = [ "criterion", "serde_json", @@ -2194,7 +2194,7 @@ dependencies = [ [[package]] name = "fuzzers" -version = "1.17.0" +version = "1.17.1" dependencies = [ "arbitrary", "bumpalo", @@ -2994,7 +2994,7 @@ dependencies = [ [[package]] name = "index-scheduler" -version = "1.17.0" +version = "1.17.1" dependencies = [ "anyhow", "backoff", @@ -3230,7 +3230,7 @@ dependencies = [ [[package]] name = "json-depth-checker" -version = "1.17.0" +version = "1.17.1" dependencies = [ "criterion", "serde_json", @@ -3724,7 +3724,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "meili-snap" -version = "1.17.0" +version = "1.17.1" dependencies = [ "insta", "md5", @@ -3735,7 +3735,7 @@ dependencies = [ [[package]] name = "meilisearch" -version = "1.17.0" +version = "1.17.1" dependencies = [ "actix-cors", "actix-http", @@ -3831,7 +3831,7 @@ dependencies = [ [[package]] name = "meilisearch-auth" -version = "1.17.0" +version = "1.17.1" dependencies = [ "base64 0.22.1", "enum-iterator", @@ -3850,7 +3850,7 @@ dependencies = [ [[package]] name = "meilisearch-types" -version = "1.17.0" +version = "1.17.1" dependencies = [ "actix-web", "anyhow", @@ -3885,7 +3885,7 @@ dependencies = [ [[package]] name = "meilitool" -version = "1.17.0" +version = "1.17.1" dependencies = [ "anyhow", "clap", @@ -3919,7 +3919,7 @@ dependencies = [ [[package]] name = "milli" -version = "1.17.0" +version = "1.17.1" dependencies = [ "allocator-api2 0.3.0", "arroy", @@ -4471,7 +4471,7 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "permissive-json-pointer" -version = "1.17.0" +version = "1.17.1" dependencies = [ "big_s", "serde_json", @@ -7259,7 +7259,7 @@ dependencies = [ [[package]] name = "xtask" -version = "1.17.0" +version = "1.17.1" dependencies = [ "anyhow", "build-info", diff --git a/Cargo.toml b/Cargo.toml index 1fa86c671..9708aebbc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,7 +22,7 @@ members = [ ] [workspace.package] -version = "1.17.0" +version = "1.17.1" authors = [ "Quentin de Quelen ", "Clément Renault ", From 9021cb4258951341f5fcc747abb81f6aaf38ecc5 Mon Sep 17 00:00:00 2001 From: curquiza Date: Tue, 12 Aug 2025 14:55:57 +0200 Subject: [PATCH 304/312] Fix update-cargo-version CI --- .github/workflows/update-cargo-toml-version.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/update-cargo-toml-version.yml b/.github/workflows/update-cargo-toml-version.yml index d13a4404a..4118cd651 100644 --- a/.github/workflows/update-cargo-toml-version.yml +++ b/.github/workflows/update-cargo-toml-version.yml @@ -41,5 +41,4 @@ jobs: --title "Update version for the next release ($NEW_VERSION) in Cargo.toml" \ --body '⚠️ This PR is automatically generated. Check the new version is the expected one and Cargo.lock has been updated before merging.' \ --label 'skip changelog' \ - --milestone $NEW_VERSION \ --base $GITHUB_REF_NAME From b9e014c04496a21c2ea188ab37c47071394a0517 Mon Sep 17 00:00:00 2001 From: curquiza Date: Tue, 12 Aug 2025 15:36:28 +0200 Subject: [PATCH 305/312] Update snapshots --- .../upgrade_failure/after_processing_everything.snap | 4 ++-- .../upgrade_failure/register_automatic_upgrade_task.snap | 2 +- .../registered_a_task_while_the_upgrade_task_is_enqueued.snap | 2 +- .../test_failure.rs/upgrade_failure/upgrade_task_failed.snap | 4 ++-- .../upgrade_failure/upgrade_task_failed_again.snap | 4 ++-- .../upgrade_failure/upgrade_task_succeeded.snap | 4 ++-- crates/meilisearch/tests/upgrade/mod.rs | 4 ++-- ...ches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...ches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...tches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...asks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...asks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap | 2 +- ...tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap | 2 +- ..._whole_batch_queue_once_everything_has_been_processed.snap | 2 +- ...e_whole_task_queue_once_everything_has_been_processed.snap | 2 +- 15 files changed, 20 insertions(+), 20 deletions(-) diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap index 0b87ffa2c..d700dd3db 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} } [timestamp] [4,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", } 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", } 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", } diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap index d0444d3ab..ee3cefba4 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} ---------------------------------------------------------------------- ### Status: enqueued [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap index 0533b162a..abaffbb1b 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap index 8a52efde5..9569ecfe3 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} ---------------------------------------------------------------------- ### Status: @@ -37,7 +37,7 @@ catto [1,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap index 40faa3504..1d7945023 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} ---------------------------------------------------------------------- @@ -40,7 +40,7 @@ doggo [2,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap index 8fe59f5c4..869d1d0b2 100644 --- a/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap +++ b/crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap @@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs [] ---------------------------------------------------------------------- ### All Tasks: -0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }} +0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 17, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }} 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }} 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }} @@ -43,7 +43,7 @@ doggo [2,3,] [timestamp] [0,] ---------------------------------------------------------------------- ### All Batches: -0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } +0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.17.1"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", } ---------------------------------------------------------------------- ### Batch to tasks mapping: 0 [0,] diff --git a/crates/meilisearch/tests/upgrade/mod.rs b/crates/meilisearch/tests/upgrade/mod.rs index 5585e50c4..5d120ba2f 100644 --- a/crates/meilisearch/tests/upgrade/mod.rs +++ b/crates/meilisearch/tests/upgrade/mod.rs @@ -43,7 +43,7 @@ async fn version_too_old() { std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.17.0"); + snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.17.1"); } #[actix_rt::test] @@ -58,7 +58,7 @@ async fn version_requires_downgrade() { std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap(); let options = Opt { experimental_dumpless_upgrade: true, ..default_settings }; let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err(); - snapshot!(err, @"Database version 1.17.1 is higher than the Meilisearch version 1.17.0. Downgrade is not supported"); + snapshot!(err, @"Database version 1.17.2 is higher than the Meilisearch version 1.17.1. Downgrade is not supported"); } #[actix_rt::test] diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index c7bcd543f..e7d8768be 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index c7bcd543f..e7d8768be 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index c7bcd543f..e7d8768be 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap index 8674d2c24..61dd95786 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap index 8674d2c24..61dd95786 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap index 8674d2c24..61dd95786 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "error": null, "duration": "[duration]", diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap index 842e3f536..8103ceed2 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap @@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "progress": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "stats": { "totalNbTasks": 1, diff --git a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap index 18d5b4e20..81259377c 100644 --- a/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap +++ b/crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap @@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs "canceledBy": null, "details": { "upgradeFrom": "v1.12.0", - "upgradeTo": "v1.17.0" + "upgradeTo": "v1.17.1" }, "error": null, "duration": "[duration]", From b5ba0e42b32ced20909a961b056a2b329be7c479 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 09:58:16 +0200 Subject: [PATCH 306/312] Add new error --- crates/filter-parser/src/condition.rs | 7 +++++++ crates/filter-parser/src/error.rs | 10 +++++++++- crates/filter-parser/src/lib.rs | 12 ++++++------ crates/filter-parser/src/value.rs | 18 ++++-------------- 4 files changed, 26 insertions(+), 21 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index 4d156c269..24c6c50cc 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -135,6 +135,13 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> // From this point, we are certain this is a vector filter, so our errors must be final. // We could use nom's `cut` but it's better to be explicit about the errors + if let Ok((_, space)) = tag::<_, _, ()>(" ")(input) { + return Err(crate::Error::new_failure_from_kind( + space, + ErrorKind::VectorFilterMissingEmbedder, + )); + } + let (input, embedder_name) = parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidEmbedder)?; diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index bbf2c8d17..a5905b1cb 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -54,7 +54,7 @@ impl<'a, T> IResultExt<'a> for IResult<'a, T> { nom::Err::Error(e) => *e.context(), nom::Err::Failure(e) => *e.context(), }; - nom::Err::Failure(Error::new_from_kind(input, kind)) + Error::new_failure_from_kind(input, kind) }) } } @@ -79,6 +79,7 @@ pub enum ErrorKind<'a> { MisusedGeoRadius, MisusedGeoBoundingBox, VectorFilterLeftover, + VectorFilterMissingEmbedder, VectorFilterInvalidEmbedder, VectorFilterMissingFragment, VectorFilterInvalidFragment, @@ -112,6 +113,10 @@ impl<'a> Error<'a> { Self { context, kind } } + pub fn new_failure_from_kind(context: Span<'a>, kind: ErrorKind<'a>) -> nom::Err { + nom::Err::Failure(Self::new_from_kind(context, kind)) + } + pub fn new_from_external(context: Span<'a>, error: impl std::error::Error) -> Self { Self::new_from_kind(context, ErrorKind::External(error.to_string())) } @@ -199,6 +204,9 @@ impl Display for Error<'_> { ErrorKind::VectorFilterMissingFragment => { writeln!(f, "The vector filter is missing a fragment name.")? } + ErrorKind::VectorFilterMissingEmbedder => { + writeln!(f, "Was expecting embedder name but found nothing.")? + } ErrorKind::VectorFilterInvalidEmbedder => { writeln!(f, "The vector filter's embedder is invalid.")? } diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index ae11ccf55..8ecbf5dc4 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -437,7 +437,7 @@ fn parse_geo_bounding_box(input: Span) -> IResult { let (input, args) = parsed?; if args.len() != 2 || args[0].len() != 2 || args[1].len() != 2 { - return Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::GeoBoundingBox))); + return Err(Error::new_failure_from_kind(input, ErrorKind::GeoBoundingBox)); } let res = FilterCondition::GeoBoundingBox { @@ -458,7 +458,7 @@ fn parse_geo_point(input: Span) -> IResult { ))(input) .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))))?; // if we succeeded we still return a `Failure` because geoPoints are not allowed - Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoPoint")))) + Err(Error::new_failure_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))) } /// geoPoint = WS* "_geoDistance(float WS* "," WS* float WS* "," WS* float) @@ -472,7 +472,7 @@ fn parse_geo_distance(input: Span) -> IResult { ))(input) .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))))?; // if we succeeded we still return a `Failure` because `geoDistance` filters are not allowed - Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoDistance")))) + Err(Error::new_failure_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))) } /// geo = WS* "_geo(float WS* "," WS* float WS* "," WS* float) @@ -486,7 +486,7 @@ fn parse_geo(input: Span) -> IResult { ))(input) .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geo"))))?; // if we succeeded we still return a `Failure` because `_geo` filter is not allowed - Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geo")))) + Err(Error::new_failure_from_kind(input, ErrorKind::ReservedGeo("_geo"))) } fn parse_error_reserved_keyword(input: Span) -> IResult { @@ -1006,8 +1006,8 @@ pub mod tests { 1:25 _vectors _vectors EXISTS "); insta::assert_snapshot!(p(r#"_vectors. embedderName EXISTS"#), @r" - The vector filter's embedder is invalid. - 10:30 _vectors. embedderName EXISTS + Was expecting embedder name but found nothing. + 10:11 _vectors. embedderName EXISTS "); insta::assert_snapshot!(p(r#"_vectors .embedderName EXISTS"#), @r" Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors .embedderName EXISTS`. diff --git a/crates/filter-parser/src/value.rs b/crates/filter-parser/src/value.rs index 345f0b0a2..ac645799b 100644 --- a/crates/filter-parser/src/value.rs +++ b/crates/filter-parser/src/value.rs @@ -132,31 +132,21 @@ pub fn parse_value(input: Span) -> IResult { } match parse_geo_radius(input) { - Ok(_) => { - return Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::MisusedGeoRadius))) - } + Ok(_) => return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoRadius)), // if we encountered a failure it means the user badly wrote a _geoRadius filter. // But instead of showing them how to fix his syntax we are going to tell them they should not use this filter as a value. Err(e) if e.is_failure() => { - return Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::MisusedGeoRadius))) + return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoRadius)) } _ => (), } match parse_geo_bounding_box(input) { - Ok(_) => { - return Err(nom::Err::Failure(Error::new_from_kind( - input, - ErrorKind::MisusedGeoBoundingBox, - ))) - } + Ok(_) => return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)), // if we encountered a failure it means the user badly wrote a _geoBoundingBox filter. // But instead of showing them how to fix his syntax we are going to tell them they should not use this filter as a value. Err(e) if e.is_failure() => { - return Err(nom::Err::Failure(Error::new_from_kind( - input, - ErrorKind::MisusedGeoBoundingBox, - ))) + return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)) } _ => (), } From f6559258ced2f025a8e54dc9be0d7e41be82d0dc Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 10:32:28 +0200 Subject: [PATCH 307/312] Improve operation error on vector filters --- crates/filter-parser/src/condition.rs | 27 +++++++++++++++------------ crates/filter-parser/src/error.rs | 4 ++++ crates/filter-parser/src/lib.rs | 24 ++++++++++++------------ 3 files changed, 31 insertions(+), 24 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index 24c6c50cc..12542110a 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -164,21 +164,24 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> Ok((input, (Token::from(fid), Some(embedder_name), filter))) } -/// vectors_exists = vectors "EXISTS" +/// vectors_exists = vectors ("EXISTS" | ("NOT" WS+ "EXISTS")) pub fn parse_vectors_exists(input: Span) -> IResult { - let (input, (fid, embedder, filter)) = terminated(parse_vectors, tag("EXISTS"))(input)?; - - Ok((input, FilterCondition::VectorExists { fid, embedder, filter })) -} -/// vectors_not_exists = vectors "NOT" WS+ "EXISTS" -pub fn parse_vectors_not_exists(input: Span) -> IResult { let (input, (fid, embedder, filter)) = parse_vectors(input)?; - let (input, _) = tuple((tag("NOT"), multispace1, tag("EXISTS")))(input)?; - Ok(( - input, - FilterCondition::Not(Box::new(FilterCondition::VectorExists { fid, embedder, filter })), - )) + // Try parsing "EXISTS" first + if let Ok((input, _)) = tag::<_, _, ()>("EXISTS")(input) { + return Ok((input, FilterCondition::VectorExists { fid, embedder, filter })); + } + + // Try parsing "NOT EXISTS" + if let Ok((input, _)) = tuple::<_, _, (), _>((tag("NOT"), multispace1, tag("EXISTS")))(input) { + return Ok(( + input, + FilterCondition::Not(Box::new(FilterCondition::VectorExists { fid, embedder, filter })), + )); + } + + Err(crate::Error::new_failure_from_kind(input, ErrorKind::VectorFilterOperation)) } /// contains = value "CONTAINS" value diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index a5905b1cb..f6356f3fd 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -83,6 +83,7 @@ pub enum ErrorKind<'a> { VectorFilterInvalidEmbedder, VectorFilterMissingFragment, VectorFilterInvalidFragment, + VectorFilterOperation, InvalidPrimary, InvalidEscapedNumber, ExpectedEof, @@ -210,6 +211,9 @@ impl Display for Error<'_> { ErrorKind::VectorFilterInvalidEmbedder => { writeln!(f, "The vector filter's embedder is invalid.")? } + ErrorKind::VectorFilterOperation => { + writeln!(f, "Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter.")? + } ErrorKind::ReservedKeyword(word) => { writeln!(f, "`{word}` is a reserved keyword and thus cannot be used as a field name unless it is put inside quotes. Use \"{word}\" or \'{word}\' instead.")? } diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 8ecbf5dc4..2c8dfac80 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -65,7 +65,7 @@ use nom_locate::LocatedSpan; pub(crate) use value::parse_value; use value::word_exact; -use crate::condition::{parse_vectors_exists, parse_vectors_not_exists}; +use crate::condition::parse_vectors_exists; use crate::error::IResultExt; pub type Span<'a> = LocatedSpan<&'a str, &'a str>; @@ -525,7 +525,7 @@ fn parse_primary(input: Span, depth: usize) -> IResult { parse_is_not_null, parse_is_empty, parse_is_not_empty, - alt((parse_vectors_exists, parse_vectors_not_exists, parse_exists, parse_not_exists)), + alt((parse_vectors_exists, parse_exists, parse_not_exists)), parse_to, parse_contains, parse_not_contains, @@ -1002,16 +1002,16 @@ pub mod tests { ); insta::assert_snapshot!(p(r#"_vectors _vectors EXISTS"#), @r" - Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors _vectors EXISTS`. - 1:25 _vectors _vectors EXISTS + Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter. + 10:25 _vectors _vectors EXISTS "); insta::assert_snapshot!(p(r#"_vectors. embedderName EXISTS"#), @r" Was expecting embedder name but found nothing. 10:11 _vectors. embedderName EXISTS "); insta::assert_snapshot!(p(r#"_vectors .embedderName EXISTS"#), @r" - Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors .embedderName EXISTS`. - 1:30 _vectors .embedderName EXISTS + Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter. + 10:30 _vectors .embedderName EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName. EXISTS"#), @r" The vector filter has leftover tokens. @@ -1038,20 +1038,20 @@ pub mod tests { 33:40 _vectors.embedderName.fragments. EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments.test test EXISTS"#), @r" - Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors.embedderName.fragments.test test EXISTS`. - 1:49 _vectors.embedderName.fragments.test test EXISTS + Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter. + 38:49 _vectors.embedderName.fragments.test test EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments. test EXISTS"#), @r" The vector filter's fragment is invalid. 33:45 _vectors.embedderName.fragments. test EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName .fragments. test EXISTS"#), @r" - Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors.embedderName .fragments. test EXISTS`. - 1:46 _vectors.embedderName .fragments. test EXISTS + Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter. + 23:46 _vectors.embedderName .fragments. test EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName .fragments.test EXISTS"#), @r" - Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `_vectors.embedderName .fragments.test EXISTS`. - 1:45 _vectors.embedderName .fragments.test EXISTS + Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter. + 23:45 _vectors.embedderName .fragments.test EXISTS "); insta::assert_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###" From 666ae1a3e745f167946cacc039ca511bd9fc52c9 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 13:00:38 +0200 Subject: [PATCH 308/312] Add "did you mean" message --- Cargo.lock | 1 + crates/filter-parser/Cargo.toml | 1 + crates/filter-parser/src/condition.rs | 18 ++++++++++++----- crates/filter-parser/src/error.rs | 29 +++++++++++++++++++++++++-- crates/filter-parser/src/lib.rs | 20 ++++++++++-------- crates/filter-parser/src/value.rs | 8 ++++---- 6 files changed, 58 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8413b3d14..43f491f1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2031,6 +2031,7 @@ name = "filter-parser" version = "1.16.0" dependencies = [ "insta", + "levenshtein_automata", "nom", "nom_locate", "unescaper", diff --git a/crates/filter-parser/Cargo.toml b/crates/filter-parser/Cargo.toml index 6eeb0794b..173cabd4b 100644 --- a/crates/filter-parser/Cargo.toml +++ b/crates/filter-parser/Cargo.toml @@ -15,6 +15,7 @@ license.workspace = true nom = "7.1.3" nom_locate = "4.2.0" unescaper = "0.1.6" +levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] } [dev-dependencies] # fixed version due to format breakages in v1.40 diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index 12542110a..e4795d048 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -19,6 +19,7 @@ use Condition::*; use crate::error::IResultExt; use crate::value::parse_vector_value; +use crate::Error; use crate::ErrorKind; use crate::VectorFilter; use crate::{parse_value, FilterCondition, IResult, Span, Token}; @@ -136,10 +137,7 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> // We could use nom's `cut` but it's better to be explicit about the errors if let Ok((_, space)) = tag::<_, _, ()>(" ")(input) { - return Err(crate::Error::new_failure_from_kind( - space, - ErrorKind::VectorFilterMissingEmbedder, - )); + return Err(crate::Error::failure_from_kind(space, ErrorKind::VectorFilterMissingEmbedder)); } let (input, embedder_name) = @@ -159,6 +157,16 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> value(VectorFilter::None, nom::combinator::success("")), ))(input)?; + if let Ok((input, point)) = tag::<_, _, ()>(".")(input) { + let opt_value = parse_vector_value(input).ok().map(|(_, v)| v); + let value = opt_value + .as_ref() + .map(|v| v.original_span().to_string()) + .unwrap_or_else(|| point.to_string()); + let context = opt_value.map(|v| v.original_span()).unwrap_or(point); + return Err(Error::failure_from_kind(context, ErrorKind::VectorFilterUnknownSuffix(value))); + } + let (input, _) = multispace1(input).map_cut(ErrorKind::VectorFilterLeftover)?; Ok((input, (Token::from(fid), Some(embedder_name), filter))) @@ -181,7 +189,7 @@ pub fn parse_vectors_exists(input: Span) -> IResult { )); } - Err(crate::Error::new_failure_from_kind(input, ErrorKind::VectorFilterOperation)) + Err(crate::Error::failure_from_kind(input, ErrorKind::VectorFilterOperation)) } /// contains = value "CONTAINS" value diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index f6356f3fd..91ae2e33c 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -54,7 +54,7 @@ impl<'a, T> IResultExt<'a> for IResult<'a, T> { nom::Err::Error(e) => *e.context(), nom::Err::Failure(e) => *e.context(), }; - Error::new_failure_from_kind(input, kind) + Error::failure_from_kind(input, kind) }) } } @@ -83,6 +83,7 @@ pub enum ErrorKind<'a> { VectorFilterInvalidEmbedder, VectorFilterMissingFragment, VectorFilterInvalidFragment, + VectorFilterUnknownSuffix(String), VectorFilterOperation, InvalidPrimary, InvalidEscapedNumber, @@ -114,7 +115,7 @@ impl<'a> Error<'a> { Self { context, kind } } - pub fn new_failure_from_kind(context: Span<'a>, kind: ErrorKind<'a>) -> nom::Err { + pub fn failure_from_kind(context: Span<'a>, kind: ErrorKind<'a>) -> nom::Err { nom::Err::Failure(Self::new_from_kind(context, kind)) } @@ -155,6 +156,20 @@ impl Display for Error<'_> { // first line being the diagnostic and the second line being the incriminated filter. let escaped_input = input.escape_debug(); + fn key_suggestion<'a>(key: &str, keys: &[&'a str]) -> Option<&'a str> { + let typos = + levenshtein_automata::LevenshteinAutomatonBuilder::new(2, true).build_dfa(key); + for key in keys.iter() { + match typos.eval(key) { + levenshtein_automata::Distance::Exact(_) => { + return Some(key); + } + levenshtein_automata::Distance::AtLeast(_) => continue, + } + } + None + } + match &self.kind { ErrorKind::ExpectedValue(_) if input.trim().is_empty() => { writeln!(f, "Was expecting a value but instead got nothing.")? @@ -199,6 +214,16 @@ impl Display for Error<'_> { ErrorKind::VectorFilterLeftover => { writeln!(f, "The vector filter has leftover tokens.")? } + ErrorKind::VectorFilterUnknownSuffix(value) if value.as_str() == "." => { + writeln!(f, "Was expecting one of `.fragments`, `.userProvided`, `.documentTemplate`, `.regenerate` or nothing, but instead found a point without a valid value.")?; + } + ErrorKind::VectorFilterUnknownSuffix(value) => { + if let Some(suggestion) = key_suggestion(value, &["fragments", "userProvided", "documentTemplate", "regenerate"]) { + writeln!(f, "Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `{value}`. Did you mean `{suggestion}`?")?; + } else { + writeln!(f, "Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `{value}`.")?; + } + } ErrorKind::VectorFilterInvalidFragment => { writeln!(f, "The vector filter's fragment is invalid.")? } diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 2c8dfac80..75cbecc26 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -437,7 +437,7 @@ fn parse_geo_bounding_box(input: Span) -> IResult { let (input, args) = parsed?; if args.len() != 2 || args[0].len() != 2 || args[1].len() != 2 { - return Err(Error::new_failure_from_kind(input, ErrorKind::GeoBoundingBox)); + return Err(Error::failure_from_kind(input, ErrorKind::GeoBoundingBox)); } let res = FilterCondition::GeoBoundingBox { @@ -458,7 +458,7 @@ fn parse_geo_point(input: Span) -> IResult { ))(input) .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))))?; // if we succeeded we still return a `Failure` because geoPoints are not allowed - Err(Error::new_failure_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))) + Err(Error::failure_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))) } /// geoPoint = WS* "_geoDistance(float WS* "," WS* float WS* "," WS* float) @@ -472,7 +472,7 @@ fn parse_geo_distance(input: Span) -> IResult { ))(input) .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))))?; // if we succeeded we still return a `Failure` because `geoDistance` filters are not allowed - Err(Error::new_failure_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))) + Err(Error::failure_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))) } /// geo = WS* "_geo(float WS* "," WS* float WS* "," WS* float) @@ -486,7 +486,7 @@ fn parse_geo(input: Span) -> IResult { ))(input) .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geo"))))?; // if we succeeded we still return a `Failure` because `_geo` filter is not allowed - Err(Error::new_failure_from_kind(input, ErrorKind::ReservedGeo("_geo"))) + Err(Error::failure_from_kind(input, ErrorKind::ReservedGeo("_geo"))) } fn parse_error_reserved_keyword(input: Span) -> IResult { @@ -1014,8 +1014,8 @@ pub mod tests { 10:30 _vectors .embedderName EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName. EXISTS"#), @r" - The vector filter has leftover tokens. - 22:30 _vectors.embedderName. EXISTS + Was expecting one of `.fragments`, `.userProvided`, `.documentTemplate`, `.regenerate` or nothing, but instead found a point without a valid value. + 22:23 _vectors.embedderName. EXISTS "); insta::assert_snapshot!(p(r#"_vectors."embedderName EXISTS"#), @r#" The vector filter's embedder is invalid. @@ -1026,8 +1026,8 @@ pub mod tests { 23:31 _vectors."embedderNam"e EXISTS "#); insta::assert_snapshot!(p(r#"_vectors.embedderName.documentTemplate. EXISTS"#), @r" - The vector filter has leftover tokens. - 39:47 _vectors.embedderName.documentTemplate. EXISTS + Was expecting one of `.fragments`, `.userProvided`, `.documentTemplate`, `.regenerate` or nothing, but instead found a point without a valid value. + 39:40 _vectors.embedderName.documentTemplate. EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments EXISTS"#), @r" The vector filter is missing a fragment name. @@ -1053,6 +1053,10 @@ pub mod tests { Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter. 23:45 _vectors.embedderName .fragments.test EXISTS "); + insta::assert_snapshot!(p(r#"_vectors.embedderName.fargments.test EXISTS"#), @r" + Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `fargments`. Did you mean `fragments`? + 23:32 _vectors.embedderName.fargments.test EXISTS + "); insta::assert_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###" Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes. diff --git a/crates/filter-parser/src/value.rs b/crates/filter-parser/src/value.rs index ac645799b..dac96b4f4 100644 --- a/crates/filter-parser/src/value.rs +++ b/crates/filter-parser/src/value.rs @@ -132,21 +132,21 @@ pub fn parse_value(input: Span) -> IResult { } match parse_geo_radius(input) { - Ok(_) => return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoRadius)), + Ok(_) => return Err(Error::failure_from_kind(input, ErrorKind::MisusedGeoRadius)), // if we encountered a failure it means the user badly wrote a _geoRadius filter. // But instead of showing them how to fix his syntax we are going to tell them they should not use this filter as a value. Err(e) if e.is_failure() => { - return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoRadius)) + return Err(Error::failure_from_kind(input, ErrorKind::MisusedGeoRadius)) } _ => (), } match parse_geo_bounding_box(input) { - Ok(_) => return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)), + Ok(_) => return Err(Error::failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)), // if we encountered a failure it means the user badly wrote a _geoBoundingBox filter. // But instead of showing them how to fix his syntax we are going to tell them they should not use this filter as a value. Err(e) if e.is_failure() => { - return Err(Error::new_failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)) + return Err(Error::failure_from_kind(input, ErrorKind::MisusedGeoBoundingBox)) } _ => (), } From b80869f2beaab82705911a10afe8bcfeea2324ab Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 13:16:25 +0200 Subject: [PATCH 309/312] Add two other "did you mean" messages --- crates/filter-parser/src/condition.rs | 6 ++--- crates/meilisearch/tests/search/filters.rs | 4 +-- crates/milli/src/error.rs | 26 +++++++++++++++++++ .../milli/src/search/facet/filter_vector.rs | 8 +++--- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index e4795d048..bdc5038e8 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -159,10 +159,8 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> if let Ok((input, point)) = tag::<_, _, ()>(".")(input) { let opt_value = parse_vector_value(input).ok().map(|(_, v)| v); - let value = opt_value - .as_ref() - .map(|v| v.original_span().to_string()) - .unwrap_or_else(|| point.to_string()); + let value = + opt_value.as_ref().map(|v| v.value().to_owned()).unwrap_or_else(|| point.to_string()); let context = opt_value.map(|v| v.original_span()).unwrap_or(point); return Err(Error::failure_from_kind(context, ErrorKind::VectorFilterUnknownSuffix(value))); } diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 8e3ee9249..9cd6575ac 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -952,13 +952,13 @@ async fn vector_filter_non_existant_fragment() { let (value, _code) = index .search_post(json!({ - "filter": "_vectors.rest.fragments.other EXISTS", + "filter": "_vectors.rest.fragments.withBred EXISTS", "attributesToRetrieve": ["name"] })) .await; snapshot!(value, @r#" { - "message": "Index `[uuid]`: The fragment `other` does not exist on embedder `rest`. Available fragments on this embedder are: `basic`, `withBreed`.\n25:30 _vectors.rest.fragments.other EXISTS", + "message": "Index `[uuid]`: The fragment `withBred` does not exist on embedder `rest`. Available fragments on this embedder are: `basic`, `withBreed`. Did you mean `withBreed`?\n25:33 _vectors.rest.fragments.withBred EXISTS", "code": "invalid_search_filter", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_search_filter" diff --git a/crates/milli/src/error.rs b/crates/milli/src/error.rs index 9ad9d0511..76ad3fda0 100644 --- a/crates/milli/src/error.rs +++ b/crates/milli/src/error.rs @@ -639,3 +639,29 @@ fn conditionally_lookup_for_error_message() { assert_eq!(err.to_string(), format!("{} {}", prefix, suffix)); } } + +pub struct DidYouMean<'a>(Option<&'a str>); + +impl<'a> DidYouMean<'a> { + pub fn new(key: &str, keys: &'a [String]) -> DidYouMean<'a> { + let typos = levenshtein_automata::LevenshteinAutomatonBuilder::new(2, true).build_dfa(key); + for key in keys.iter() { + match typos.eval(key) { + levenshtein_automata::Distance::Exact(_) => { + return DidYouMean(Some(key)); + } + levenshtein_automata::Distance::AtLeast(_) => continue, + } + } + DidYouMean(None) + } +} + +impl std::fmt::Display for DidYouMean<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(suggestion) = self.0 { + write!(f, " Did you mean `{suggestion}`?")?; + } + Ok(()) + } +} diff --git a/crates/milli/src/search/facet/filter_vector.rs b/crates/milli/src/search/facet/filter_vector.rs index 625bd5dde..1ef4b8e3d 100644 --- a/crates/milli/src/search/facet/filter_vector.rs +++ b/crates/milli/src/search/facet/filter_vector.rs @@ -1,7 +1,7 @@ use filter_parser::{Token, VectorFilter}; use roaring::{MultiOps, RoaringBitmap}; -use crate::error::Error; +use crate::error::{DidYouMean, Error}; use crate::vector::db::IndexEmbeddingConfig; use crate::vector::{ArroyStats, ArroyWrapper}; use crate::Index; @@ -14,7 +14,8 @@ pub enum VectorFilterError<'a> { } else { let mut available = available.clone(); available.sort_unstable(); - format!("Available embedders are: {}.", available.iter().map(|e| format!("`{e}`")).collect::>().join(", ")) + let did_you_mean = DidYouMean::new(embedder.value(), &available); + format!("Available embedders are: {}.{did_you_mean}", available.iter().map(|e| format!("`{e}`")).collect::>().join(", ")) } })] EmbedderDoesNotExist { embedder: &'a Token<'a>, available: Vec }, @@ -25,7 +26,8 @@ pub enum VectorFilterError<'a> { } else { let mut available = available.clone(); available.sort_unstable(); - format!("Available fragments on this embedder are: {}.", available.iter().map(|f| format!("`{f}`")).collect::>().join(", ")) + let did_you_mean = DidYouMean::new(fragment.value(), &available); + format!("Available fragments on this embedder are: {}.{did_you_mean}", available.iter().map(|f| format!("`{f}`")).collect::>().join(", ")) } })] FragmentDoesNotExist { From 8529e2161acc5c3894a1123c320ef02d60b4d380 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 13:37:19 +0200 Subject: [PATCH 310/312] Clarify more errors --- crates/filter-parser/src/condition.rs | 12 +++++++++++- crates/filter-parser/src/error.rs | 15 +++++++++++---- crates/filter-parser/src/lib.rs | 10 +++++++++- 3 files changed, 31 insertions(+), 6 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index bdc5038e8..c407a1e45 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -162,7 +162,17 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> let value = opt_value.as_ref().map(|v| v.value().to_owned()).unwrap_or_else(|| point.to_string()); let context = opt_value.map(|v| v.original_span()).unwrap_or(point); - return Err(Error::failure_from_kind(context, ErrorKind::VectorFilterUnknownSuffix(value))); + let previous_kind = match filter { + VectorFilter::Fragment(_) => Some("fragments"), + VectorFilter::DocumentTemplate => Some("documentTemplate"), + VectorFilter::UserProvided => Some("userProvided"), + VectorFilter::Regenerate => Some("regenerate"), + VectorFilter::None => None, + }; + return Err(Error::failure_from_kind( + context, + ErrorKind::VectorFilterUnknownSuffix(previous_kind, value), + )); } let (input, _) = multispace1(input).map_cut(ErrorKind::VectorFilterLeftover)?; diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index 91ae2e33c..05aaf8c17 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -83,7 +83,7 @@ pub enum ErrorKind<'a> { VectorFilterInvalidEmbedder, VectorFilterMissingFragment, VectorFilterInvalidFragment, - VectorFilterUnknownSuffix(String), + VectorFilterUnknownSuffix(Option<&'static str>, String), VectorFilterOperation, InvalidPrimary, InvalidEscapedNumber, @@ -214,16 +214,23 @@ impl Display for Error<'_> { ErrorKind::VectorFilterLeftover => { writeln!(f, "The vector filter has leftover tokens.")? } - ErrorKind::VectorFilterUnknownSuffix(value) if value.as_str() == "." => { + ErrorKind::VectorFilterUnknownSuffix(_, value) if value.as_str() == "." => { writeln!(f, "Was expecting one of `.fragments`, `.userProvided`, `.documentTemplate`, `.regenerate` or nothing, but instead found a point without a valid value.")?; } - ErrorKind::VectorFilterUnknownSuffix(value) => { + ErrorKind::VectorFilterUnknownSuffix(None, value) if ["fragments", "userProvided", "documentTemplate", "regenerate"].contains(&value.as_str()) => { + // This will happen with "_vectors.rest.\"userProvided\"" for instance + writeln!(f, "Was expecting this part to be unquoted.")? + } + ErrorKind::VectorFilterUnknownSuffix(None, value) => { if let Some(suggestion) = key_suggestion(value, &["fragments", "userProvided", "documentTemplate", "regenerate"]) { writeln!(f, "Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `{value}`. Did you mean `{suggestion}`?")?; } else { writeln!(f, "Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `{value}`.")?; } } + ErrorKind::VectorFilterUnknownSuffix(Some(previous_filter_kind), value) => { + writeln!(f, "Vector filter can only accept one of `fragments`, `userProvided`, `documentTemplate` or `regenerate`, but found both `{previous_filter_kind}` and `{value}`.")? + }, ErrorKind::VectorFilterInvalidFragment => { writeln!(f, "The vector filter's fragment is invalid.")? } @@ -234,7 +241,7 @@ impl Display for Error<'_> { writeln!(f, "Was expecting embedder name but found nothing.")? } ErrorKind::VectorFilterInvalidEmbedder => { - writeln!(f, "The vector filter's embedder is invalid.")? + writeln!(f, "The vector filter's embedder name is invalid.")? } ErrorKind::VectorFilterOperation => { writeln!(f, "Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter.")? diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 75cbecc26..8f6f02691 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -1018,7 +1018,7 @@ pub mod tests { 22:23 _vectors.embedderName. EXISTS "); insta::assert_snapshot!(p(r#"_vectors."embedderName EXISTS"#), @r#" - The vector filter's embedder is invalid. + The vector filter's embedder name is invalid. 30:30 _vectors."embedderName EXISTS "#); insta::assert_snapshot!(p(r#"_vectors."embedderNam"e EXISTS"#), @r#" @@ -1057,6 +1057,14 @@ pub mod tests { Was expecting one of `fragments`, `userProvided`, `documentTemplate`, `regenerate` or nothing, but instead found `fargments`. Did you mean `fragments`? 23:32 _vectors.embedderName.fargments.test EXISTS "); + insta::assert_snapshot!(p(r#"_vectors.embedderName."userProvided" EXISTS"#), @r#" + Was expecting this part to be unquoted. + 24:36 _vectors.embedderName."userProvided" EXISTS + "#); + insta::assert_snapshot!(p(r#"_vectors.embedderName.userProvided.fragments.test EXISTS"#), @r" + Vector filter can only accept one of `fragments`, `userProvided`, `documentTemplate` or `regenerate`, but found both `userProvided` and `fragments`. + 36:45 _vectors.embedderName.userProvided.fragments.test EXISTS + "); insta::assert_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###" Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes. From cdeca595872bb8fa668f0c2fc2f9e714f3da5294 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 17:14:36 +0200 Subject: [PATCH 311/312] Add error message for quoting errors --- crates/filter-parser/src/condition.rs | 5 +++-- crates/filter-parser/src/error.rs | 6 +++++- crates/filter-parser/src/lib.rs | 8 ++++---- crates/filter-parser/src/value.rs | 12 ++++++++++++ 4 files changed, 24 insertions(+), 7 deletions(-) diff --git a/crates/filter-parser/src/condition.rs b/crates/filter-parser/src/condition.rs index c407a1e45..8e3c04040 100644 --- a/crates/filter-parser/src/condition.rs +++ b/crates/filter-parser/src/condition.rs @@ -19,6 +19,7 @@ use Condition::*; use crate::error::IResultExt; use crate::value::parse_vector_value; +use crate::value::parse_vector_value_cut; use crate::Error; use crate::ErrorKind; use crate::VectorFilter; @@ -141,13 +142,13 @@ fn parse_vectors(input: Span) -> IResult<(Token, Option, VectorFilter<'_> } let (input, embedder_name) = - parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidEmbedder)?; + parse_vector_value_cut(input, ErrorKind::VectorFilterInvalidEmbedder)?; let (input, filter) = alt(( map( preceded(tag(".fragments"), |input| { let (input, _) = tag(".")(input).map_cut(ErrorKind::VectorFilterMissingFragment)?; - parse_vector_value(input).map_cut(ErrorKind::VectorFilterInvalidFragment) + parse_vector_value_cut(input, ErrorKind::VectorFilterInvalidFragment) }), VectorFilter::Fragment, ), diff --git a/crates/filter-parser/src/error.rs b/crates/filter-parser/src/error.rs index 05aaf8c17..e381f45e2 100644 --- a/crates/filter-parser/src/error.rs +++ b/crates/filter-parser/src/error.rs @@ -79,6 +79,7 @@ pub enum ErrorKind<'a> { MisusedGeoRadius, MisusedGeoBoundingBox, VectorFilterLeftover, + VectorFilterInvalidQuotes, VectorFilterMissingEmbedder, VectorFilterInvalidEmbedder, VectorFilterMissingFragment, @@ -232,7 +233,7 @@ impl Display for Error<'_> { writeln!(f, "Vector filter can only accept one of `fragments`, `userProvided`, `documentTemplate` or `regenerate`, but found both `{previous_filter_kind}` and `{value}`.")? }, ErrorKind::VectorFilterInvalidFragment => { - writeln!(f, "The vector filter's fragment is invalid.")? + writeln!(f, "The vector filter's fragment name is invalid.")? } ErrorKind::VectorFilterMissingFragment => { writeln!(f, "The vector filter is missing a fragment name.")? @@ -246,6 +247,9 @@ impl Display for Error<'_> { ErrorKind::VectorFilterOperation => { writeln!(f, "Was expecting an operation like `EXISTS` or `NOT EXISTS` after the vector filter.")? } + ErrorKind::VectorFilterInvalidQuotes => { + writeln!(f, "The quotes in one of the values are inconsistent.")? + } ErrorKind::ReservedKeyword(word) => { writeln!(f, "`{word}` is a reserved keyword and thus cannot be used as a field name unless it is put inside quotes. Use \"{word}\" or \'{word}\' instead.")? } diff --git a/crates/filter-parser/src/lib.rs b/crates/filter-parser/src/lib.rs index 8f6f02691..c761c583b 100644 --- a/crates/filter-parser/src/lib.rs +++ b/crates/filter-parser/src/lib.rs @@ -1018,8 +1018,8 @@ pub mod tests { 22:23 _vectors.embedderName. EXISTS "); insta::assert_snapshot!(p(r#"_vectors."embedderName EXISTS"#), @r#" - The vector filter's embedder name is invalid. - 30:30 _vectors."embedderName EXISTS + The quotes in one of the values are inconsistent. + 10:30 _vectors."embedderName EXISTS "#); insta::assert_snapshot!(p(r#"_vectors."embedderNam"e EXISTS"#), @r#" The vector filter has leftover tokens. @@ -1034,7 +1034,7 @@ pub mod tests { 32:39 _vectors.embedderName.fragments EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments. EXISTS"#), @r" - The vector filter's fragment is invalid. + The vector filter's fragment name is invalid. 33:40 _vectors.embedderName.fragments. EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments.test test EXISTS"#), @r" @@ -1042,7 +1042,7 @@ pub mod tests { 38:49 _vectors.embedderName.fragments.test test EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName.fragments. test EXISTS"#), @r" - The vector filter's fragment is invalid. + The vector filter's fragment name is invalid. 33:45 _vectors.embedderName.fragments. test EXISTS "); insta::assert_snapshot!(p(r#"_vectors.embedderName .fragments. test EXISTS"#), @r" diff --git a/crates/filter-parser/src/value.rs b/crates/filter-parser/src/value.rs index dac96b4f4..35a5c0ab4 100644 --- a/crates/filter-parser/src/value.rs +++ b/crates/filter-parser/src/value.rs @@ -113,6 +113,18 @@ pub fn parse_vector_value(input: Span) -> IResult { } } +pub fn parse_vector_value_cut<'a>(input: Span<'a>, kind: ErrorKind<'a>) -> IResult<'a, Token<'a>> { + parse_vector_value(input).map_err(|e| match e { + nom::Err::Failure(e) => match e.kind() { + ErrorKind::Char(c) if *c == '"' || *c == '\'' => { + crate::Error::failure_from_kind(input, ErrorKind::VectorFilterInvalidQuotes) + } + _ => crate::Error::failure_from_kind(input, kind), + }, + _ => crate::Error::failure_from_kind(input, kind), + }) +} + /// value = WS* ( word | singleQuoted | doubleQuoted) WS+ pub fn parse_value(input: Span) -> IResult { // to get better diagnostic message we are going to strip the left whitespaces from the input right now From 307ea38c2a8e3c08e5da6734f280a583388a653a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 13 Aug 2025 17:19:37 +0200 Subject: [PATCH 312/312] Remove old irrelevant tests --- crates/meilisearch/tests/search/filters.rs | 35 ---------------------- 1 file changed, 35 deletions(-) diff --git a/crates/meilisearch/tests/search/filters.rs b/crates/meilisearch/tests/search/filters.rs index 9cd6575ac..ef562bf4f 100644 --- a/crates/meilisearch/tests/search/filters.rs +++ b/crates/meilisearch/tests/search/filters.rs @@ -966,26 +966,6 @@ async fn vector_filter_non_existant_fragment() { "#); } -#[actix_rt::test] -async fn vector_filter_specific_fragment_user_provided() { - let index = shared_index_for_fragments().await; - - let (value, _code) = index - .search_post(json!({ - "filter": "_vectors.rest.fragments.other.userProvided EXISTS", - "attributesToRetrieve": ["name"] - })) - .await; - snapshot!(value, @r#" - { - "message": "The vector filter has leftover tokens.\n30:50 _vectors.rest.fragments.other.userProvided EXISTS", - "code": "invalid_search_filter", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_search_filter" - } - "#); -} - #[actix_rt::test] async fn vector_filter_document_template_but_fragments_used() { let index = shared_index_for_fragments().await; @@ -1179,19 +1159,4 @@ async fn vector_filter_regenerate() { "estimatedTotalHits": 3 } "#); - - let (value, _code) = index - .search_post(json!({ - "filter": format!("_vectors.rest.fragments.basic.regenerate EXISTS"), - "attributesToRetrieve": ["name"] - })) - .await; - snapshot!(value, @r#" - { - "message": "The vector filter has leftover tokens.\n30:48 _vectors.rest.fragments.basic.regenerate EXISTS", - "code": "invalid_search_filter", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_search_filter" - } - "#); }
+👇👇👇 +- [ ] Define the version of the release (`vX.Y.Z`) +- [ ] Manually test `--experimental-dumpless-upgrade` on a DB of the previous Meilisearch minor version
+- [ ] Check recent automated tests on `main`
+ - [ ] Scheduled test suite
+ - [ ] Scheduled SDK tests
+ - [ ] Scheduled flaky tests
+ - [ ] Scheduled fuzzer tests
+ - [ ] Scheduled Docker CI (dry run)
+ - [ ] Scheduled GitHub binary release (dry run)
+- [ ] Create the PR updating the versionand merge it. +