From 5e867f7ce068c2ca8a94cb9e22945a70dc9f6667 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 16:47:20 +0200 Subject: [PATCH 01/53] Add webhooks api key action --- crates/meilisearch-types/src/keys.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index aec3199a3..2eddb9547 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -365,6 +365,12 @@ pub enum Action { #[serde(rename = "*.get")] #[deserr(rename = "*.get")] AllGet, + #[serde(rename = "webhooks.get")] + #[deserr(rename = "webhooks.get")] + WebhooksGet, + #[serde(rename = "webhooks.update")] + #[deserr(rename = "webhooks.update")] + WebhooksUpdate, } impl Action { @@ -416,6 +422,8 @@ impl Action { NETWORK_GET => Some(Self::NetworkGet), NETWORK_UPDATE => Some(Self::NetworkUpdate), ALL_GET => Some(Self::AllGet), + WEBHOOKS_GET => Some(Self::WebhooksGet), + WEBHOOKS_UPDATE => Some(Self::WebhooksUpdate), _otherwise => None, } } @@ -463,6 +471,8 @@ impl Action { ChatsDelete => false, ChatsSettingsGet => true, ChatsSettingsUpdate => false, + WebhooksGet => true, + WebhooksUpdate => false, } } @@ -522,6 +532,9 @@ pub mod actions { pub const CHATS_SETTINGS_ALL: u8 = ChatsSettingsAll.repr(); pub const CHATS_SETTINGS_GET: u8 = ChatsSettingsGet.repr(); pub const CHATS_SETTINGS_UPDATE: u8 = ChatsSettingsUpdate.repr(); + + pub const WEBHOOKS_GET: u8 = WebhooksGet.repr(); + pub const WEBHOOKS_UPDATE: u8 = WebhooksUpdate.repr(); } #[cfg(test)] @@ -577,6 +590,8 @@ pub(crate) mod test { assert!(ChatsSettingsGet.repr() == 42 && CHATS_SETTINGS_GET == 42); assert!(ChatsSettingsUpdate.repr() == 43 && CHATS_SETTINGS_UPDATE == 43); assert!(AllGet.repr() == 44 && ALL_GET == 44); + assert!(WebhooksGet.repr() == 45 && WEBHOOKS_GET == 45); + assert!(WebhooksUpdate.repr() == 46 && WEBHOOKS_UPDATE == 46); } #[test] From 5567653c96cfefe1779e95c54bde730bd3758f7a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 29 Jul 2025 16:47:28 +0200 Subject: [PATCH 02/53] Fix network documentation --- crates/meilisearch/src/routes/network.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/network.rs b/crates/meilisearch/src/routes/network.rs index 7e58df113..6ee68ea33 100644 --- a/crates/meilisearch/src/routes/network.rs +++ b/crates/meilisearch/src/routes/network.rs @@ -51,7 +51,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { get, path = "", tag = "Network", - security(("Bearer" = ["network.get", "network.*", "*"])), + security(("Bearer" = ["network.get", "*"])), responses( (status = OK, description = "Known nodes are returned", body = Network, content_type = "application/json", example = json!( { From cc37eb870f7cb3835a49bae2785fadaff4689166 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 12:01:40 +0200 Subject: [PATCH 03/53] Initial implementation --- crates/index-scheduler/src/features.rs | 1 + crates/index-scheduler/src/insta_snapshot.rs | 20 +- crates/index-scheduler/src/lib.rs | 190 +++++++++------ crates/meilisearch-types/src/error.rs | 6 +- crates/meilisearch-types/src/lib.rs | 1 + crates/meilisearch-types/src/webhooks.rs | 18 ++ crates/meilisearch/src/routes/mod.rs | 5 +- crates/meilisearch/src/routes/network.rs | 2 +- crates/meilisearch/src/routes/webhooks.rs | 239 +++++++++++++++++++ 9 files changed, 409 insertions(+), 73 deletions(-) create mode 100644 crates/meilisearch-types/src/webhooks.rs create mode 100644 crates/meilisearch/src/routes/webhooks.rs diff --git a/crates/index-scheduler/src/features.rs b/crates/index-scheduler/src/features.rs index b52a659a6..dacac1a9c 100644 --- a/crates/index-scheduler/src/features.rs +++ b/crates/index-scheduler/src/features.rs @@ -182,6 +182,7 @@ impl FeatureData { ..persisted_features })); + // Once this is stabilized, network should be stored along with webhooks in index-scheduler's persisted database let network_db = runtime_features_db.remap_data_type::>(); let network: Network = network_db.get(wtxn, db_keys::NETWORK)?.unwrap_or_default(); diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index 32ce131b5..21626fb2e 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -26,11 +26,11 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { version, queue, scheduler, + persisted, index_mapper, features: _, - webhook_url: _, - webhook_authorization_header: _, + cached_webhooks: _, test_breakpoint_sdr: _, planned_failures: _, run_loop_iteration: _, @@ -62,6 +62,10 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { } snap.push_str("\n----------------------------------------------------------------------\n"); + snap.push_str("### Persisted:\n"); + snap.push_str(&snapshot_persisted_db(&rtxn, persisted)); + snap.push_str("----------------------------------------------------------------------\n"); + snap.push_str("### All Tasks:\n"); snap.push_str(&snapshot_all_tasks(&rtxn, queue.tasks.all_tasks)); snap.push_str("----------------------------------------------------------------------\n"); @@ -200,6 +204,16 @@ pub fn snapshot_date_db(rtxn: &RoTxn, db: Database) -> String { + let mut snap = String::new(); + let iter = db.iter(rtxn).unwrap(); + for next in iter { + let (key, value) = next.unwrap(); + snap.push_str(&format!("{key}: {value}\n")); + } + snap +} + pub fn snapshot_task(task: &Task) -> String { let mut snap = String::new(); let Task { @@ -311,6 +325,7 @@ pub fn snapshot_status( } snap } + pub fn snapshot_kind(rtxn: &RoTxn, db: Database, RoaringBitmapCodec>) -> String { let mut snap = String::new(); let iter = db.iter(rtxn).unwrap(); @@ -331,6 +346,7 @@ pub fn snapshot_index_tasks(rtxn: &RoTxn, db: Database) } snap } + pub fn snapshot_canceled_by(rtxn: &RoTxn, db: Database) -> String { let mut snap = String::new(); let iter = db.iter(rtxn).unwrap(); diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 46566b9ba..9e1d8d1a8 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -65,6 +65,7 @@ use meilisearch_types::milli::vector::{ use meilisearch_types::milli::{self, Index}; use meilisearch_types::task_view::TaskView; use meilisearch_types::tasks::{KindWithContent, Task}; +use meilisearch_types::webhooks::{Webhook, Webhooks}; use milli::vector::db::IndexEmbeddingConfig; use processing::ProcessingTasks; pub use queue::Query; @@ -80,7 +81,15 @@ use crate::utils::clamp_to_page_size; pub(crate) type BEI128 = I128; const TASK_SCHEDULER_SIZE_THRESHOLD_PERCENT_INT: u64 = 40; -const CHAT_SETTINGS_DB_NAME: &str = "chat-settings"; + +mod db_name { + pub const CHAT_SETTINGS: &str = "chat-settings"; + pub const PERSISTED: &str = "persisted"; +} + +mod db_keys { + pub const WEBHOOKS: &str = "webhooks"; +} #[derive(Debug)] pub struct IndexSchedulerOptions { @@ -171,10 +180,11 @@ pub struct IndexScheduler { /// Whether we should use the old document indexer or the new one. pub(crate) experimental_no_edition_2024_for_dumps: bool, - /// The webhook url we should send tasks to after processing every batches. - pub(crate) webhook_url: Option, - /// The Authorization header to send to the webhook URL. - pub(crate) webhook_authorization_header: Option, + /// A database to store single-keyed data that is persisted across restarts. + persisted: Database, + + /// Webhook + cached_webhooks: Arc>, /// A map to retrieve the runtime representation of an embedder depending on its configuration. /// @@ -214,8 +224,8 @@ impl IndexScheduler { index_mapper: self.index_mapper.clone(), cleanup_enabled: self.cleanup_enabled, experimental_no_edition_2024_for_dumps: self.experimental_no_edition_2024_for_dumps, - webhook_url: self.webhook_url.clone(), - webhook_authorization_header: self.webhook_authorization_header.clone(), + persisted: self.persisted, + cached_webhooks: self.cached_webhooks.clone(), embedders: self.embedders.clone(), #[cfg(test)] test_breakpoint_sdr: self.test_breakpoint_sdr.clone(), @@ -284,10 +294,16 @@ impl IndexScheduler { let version = versioning::Versioning::new(&env, from_db_version)?; let mut wtxn = env.write_txn()?; + let features = features::FeatureData::new(&env, &mut wtxn, options.instance_features)?; let queue = Queue::new(&env, &mut wtxn, &options)?; let index_mapper = IndexMapper::new(&env, &mut wtxn, &options, budget)?; - let chat_settings = env.create_database(&mut wtxn, Some(CHAT_SETTINGS_DB_NAME))?; + let chat_settings = env.create_database(&mut wtxn, Some(db_name::CHAT_SETTINGS))?; + + let persisted = env.create_database(&mut wtxn, Some(db_name::PERSISTED))?; + let webhooks_db = persisted.remap_data_type::>(); + let webhooks = webhooks_db.get(&wtxn, db_keys::WEBHOOKS)?.unwrap_or_default(); + wtxn.commit()?; // allow unreachable_code to get rids of the warning in the case of a test build. @@ -303,8 +319,9 @@ impl IndexScheduler { experimental_no_edition_2024_for_dumps: options .indexer_config .experimental_no_edition_2024_for_dumps, - webhook_url: options.webhook_url, - webhook_authorization_header: options.webhook_authorization_header, + persisted, + cached_webhooks: Arc::new(RwLock::new(webhooks)), + embedders: Default::default(), #[cfg(test)] @@ -754,80 +771,103 @@ impl IndexScheduler { /// Once the tasks changes have been committed we must send all the tasks that were updated to our webhook if there is one. fn notify_webhook(&self, updated: &RoaringBitmap) -> Result<()> { - if let Some(ref url) = self.webhook_url { - struct TaskReader<'a, 'b> { - rtxn: &'a RoTxn<'a>, - index_scheduler: &'a IndexScheduler, - tasks: &'b mut roaring::bitmap::Iter<'b>, - buffer: Vec, - written: usize, - } + let webhooks = self.cached_webhooks.read().unwrap_or_else(|poisoned| poisoned.into_inner()); + if webhooks.webhooks.is_empty() { + return Ok(()); + } + let webhooks = Webhooks::clone(&*webhooks); - impl Read for TaskReader<'_, '_> { - fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result { - if self.buffer.is_empty() { - match self.tasks.next() { - None => return Ok(0), - Some(task_id) => { - let task = self - .index_scheduler - .queue - .tasks - .get_task(self.rtxn, task_id) - .map_err(|err| io::Error::new(io::ErrorKind::Other, err))? - .ok_or_else(|| { - io::Error::new( - io::ErrorKind::Other, - Error::CorruptedTaskQueue, - ) - })?; + struct TaskReader<'a, 'b> { + rtxn: &'a RoTxn<'a>, + index_scheduler: &'a IndexScheduler, + tasks: &'b mut roaring::bitmap::Iter<'b>, + buffer: Vec, + written: usize, + } - serde_json::to_writer( - &mut self.buffer, - &TaskView::from_task(&task), - )?; - self.buffer.push(b'\n'); - } + impl Read for TaskReader<'_, '_> { + fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result { + if self.buffer.is_empty() { + match self.tasks.next() { + None => return Ok(0), + Some(task_id) => { + let task = self + .index_scheduler + .queue + .tasks + .get_task(self.rtxn, task_id) + .map_err(|err| io::Error::new(io::ErrorKind::Other, err))? + .ok_or_else(|| { + io::Error::new(io::ErrorKind::Other, Error::CorruptedTaskQueue) + })?; + + serde_json::to_writer(&mut self.buffer, &TaskView::from_task(&task))?; + self.buffer.push(b'\n'); } } + } - let mut to_write = &self.buffer[self.written..]; - let wrote = io::copy(&mut to_write, &mut buf)?; - self.written += wrote as usize; + let mut to_write = &self.buffer[self.written..]; + let wrote = io::copy(&mut to_write, &mut buf)?; + self.written += wrote as usize; - // we wrote everything and must refresh our buffer on the next call - if self.written == self.buffer.len() { - self.written = 0; - self.buffer.clear(); - } + // we wrote everything and must refresh our buffer on the next call + if self.written == self.buffer.len() { + self.written = 0; + self.buffer.clear(); + } - Ok(wrote as usize) + Ok(wrote as usize) + } + } + + let rtxn = self.env.read_txn()?; + + let task_reader = TaskReader { + rtxn: &rtxn, + index_scheduler: self, + tasks: &mut updated.into_iter(), + buffer: Vec::with_capacity(800), // on average a task is around ~600 bytes + written: 0, + }; + + enum EitherRead { + Other(T), + Data(Vec), + } + + impl Read for &mut EitherRead { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + match self { + EitherRead::Other(reader) => reader.read(buf), + EitherRead::Data(data) => data.as_slice().read(buf), } } + } - let rtxn = self.env.read_txn()?; + let mut reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); - let task_reader = TaskReader { - rtxn: &rtxn, - index_scheduler: self, - tasks: &mut updated.into_iter(), - buffer: Vec::with_capacity(50), // on average a task is around ~100 bytes - written: 0, - }; + // When there is more than one webhook, cache the data in memory + let mut reader = match webhooks.webhooks.len() { + 1 => EitherRead::Other(reader), + _ => { + let mut data = Vec::new(); + reader.read_to_end(&mut data)?; + EitherRead::Data(data) + } + }; - // let reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); - let reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); - let request = ureq::post(url) + for (name, Webhook { url, headers }) in webhooks.webhooks.iter() { + let mut request = ureq::post(url) .timeout(Duration::from_secs(30)) .set("Content-Encoding", "gzip") .set("Content-Type", "application/x-ndjson"); - let request = match &self.webhook_authorization_header { - Some(header) => request.set("Authorization", header), - None => request, - }; + for (header_name, header_value) in headers.iter() { + request = request.set(header_name, header_value); + } - if let Err(e) = request.send(reader) { - tracing::error!("While sending data to the webhook: {e}"); + if let Err(e) = request.send(&mut reader) { + tracing::error!("While sending data to the webhook {name}: {e}"); } } @@ -862,6 +902,20 @@ impl IndexScheduler { self.features.network() } + pub fn put_webhooks(&self, webhooks: Webhooks) -> Result<()> { + let mut wtxn = self.env.write_txn()?; + let webhooks_db = self.persisted.remap_data_type::>(); + webhooks_db.put(&mut wtxn, db_keys::WEBHOOKS, &webhooks)?; + wtxn.commit()?; + *self.cached_webhooks.write().unwrap() = webhooks; + Ok(()) + } + + pub fn webhooks(&self) -> Webhooks { + let webhooks = self.cached_webhooks.read().unwrap_or_else(|poisoned| poisoned.into_inner()); + Webhooks::clone(&*webhooks) + } + pub fn embedders( &self, index_uid: String, diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 458034c00..92425d386 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -418,7 +418,11 @@ InvalidChatCompletionSearchDescriptionPrompt , InvalidRequest , BAD_REQU InvalidChatCompletionSearchQueryParamPrompt , InvalidRequest , BAD_REQUEST ; InvalidChatCompletionSearchFilterParamPrompt , InvalidRequest , BAD_REQUEST ; InvalidChatCompletionSearchIndexUidParamPrompt , InvalidRequest , BAD_REQUEST ; -InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQUEST +InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQUEST ; +// Webhooks +InvalidWebhooks , InvalidRequest , BAD_REQUEST ; +InvalidWebhooksUrl , InvalidRequest , BAD_REQUEST ; +InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST } impl ErrorCode for JoinError { diff --git a/crates/meilisearch-types/src/lib.rs b/crates/meilisearch-types/src/lib.rs index fe69da526..9857bfb29 100644 --- a/crates/meilisearch-types/src/lib.rs +++ b/crates/meilisearch-types/src/lib.rs @@ -15,6 +15,7 @@ pub mod star_or; pub mod task_view; pub mod tasks; pub mod versioning; +pub mod webhooks; pub use milli::{heed, Index}; use uuid::Uuid; pub use versioning::VERSION_FILE_NAME; diff --git a/crates/meilisearch-types/src/webhooks.rs b/crates/meilisearch-types/src/webhooks.rs new file mode 100644 index 000000000..c30d32bc6 --- /dev/null +++ b/crates/meilisearch-types/src/webhooks.rs @@ -0,0 +1,18 @@ +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Webhook { + pub url: String, + #[serde(default)] + pub headers: BTreeMap, +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Webhooks { + #[serde(default)] + pub webhooks: BTreeMap, +} diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 260d973a1..4ae72b0bd 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -70,6 +70,7 @@ mod swap_indexes; pub mod tasks; #[cfg(test)] mod tasks_test; +mod webhooks; #[derive(OpenApi)] #[openapi( @@ -89,6 +90,7 @@ mod tasks_test; (path = "/experimental-features", api = features::ExperimentalFeaturesApi), (path = "/export", api = export::ExportApi), (path = "/network", api = network::NetworkApi), + (path = "/webhooks", api = webhooks::WebhooksApi), ), paths(get_health, get_version, get_stats), tags( @@ -120,7 +122,8 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service(web::scope("/experimental-features").configure(features::configure)) .service(web::scope("/network").configure(network::configure)) .service(web::scope("/export").configure(export::configure)) - .service(web::scope("/chats").configure(chats::configure)); + .service(web::scope("/chats").configure(chats::configure)) + .service(web::scope("/webhooks").configure(webhooks::configure)); #[cfg(feature = "swagger")] { diff --git a/crates/meilisearch/src/routes/network.rs b/crates/meilisearch/src/routes/network.rs index 6ee68ea33..4afa32c09 100644 --- a/crates/meilisearch/src/routes/network.rs +++ b/crates/meilisearch/src/routes/network.rs @@ -168,7 +168,7 @@ impl Aggregate for PatchNetworkAnalytics { path = "", tag = "Network", request_body = Network, - security(("Bearer" = ["network.update", "network.*", "*"])), + security(("Bearer" = ["network.update", "*"])), responses( (status = OK, description = "New network state is returned", body = Network, content_type = "application/json", example = json!( { diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs new file mode 100644 index 000000000..d05c16672 --- /dev/null +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -0,0 +1,239 @@ +use std::collections::BTreeMap; + +use actix_web::web::{self, Data}; +use actix_web::{HttpRequest, HttpResponse}; +use deserr::actix_web::AwebJson; +use deserr::Deserr; +use index_scheduler::IndexScheduler; +use meilisearch_types::deserr::DeserrJsonError; +use meilisearch_types::error::deserr_codes::{ + InvalidWebhooks, InvalidWebhooksHeaders, InvalidWebhooksUrl, +}; +use meilisearch_types::error::{ErrorCode, ResponseError}; +use meilisearch_types::keys::actions; +use meilisearch_types::milli::update::Setting; +use meilisearch_types::webhooks::{Webhook, Webhooks}; +use serde::Serialize; +use tracing::debug; +use utoipa::{OpenApi, ToSchema}; + +use crate::analytics::{Aggregate, Analytics}; +use crate::extractors::authentication::policies::ActionPolicy; +use crate::extractors::authentication::GuardedData; +use crate::extractors::sequential_extractor::SeqHandler; + +#[derive(OpenApi)] +#[openapi( + paths(get_webhooks, patch_webhooks), + tags(( + name = "Webhooks", + description = "The `/webhooks` route allows you to register endpoints to be called once tasks are processed.", + external_docs(url = "https://www.meilisearch.com/docs/reference/api/webhooks"), + )), +)] +pub struct WebhooksApi; + +pub fn configure(cfg: &mut web::ServiceConfig) { + cfg.service( + web::resource("") + .route(web::get().to(get_webhooks)) + .route(web::patch().to(SeqHandler(patch_webhooks))), + ); +} + +#[utoipa::path( + get, + path = "", + tag = "Webhooks", + security(("Bearer" = ["webhooks.get", "*.get", "*"])), + responses( + (status = OK, description = "Webhooks are returned", body = WebhooksSettings, content_type = "application/json", example = json!({ + "webhooks": { + "name": { + "url": "http://example.com/webhook", + }, + "anotherName": { + "url": "https://your.site/on-tasks-completed", + "headers": { + "Authorization": "Bearer a-secret-token" + } + } + } + })), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] +async fn get_webhooks( + index_scheduler: GuardedData, Data>, +) -> Result { + let webhooks = index_scheduler.webhooks(); + debug!(returns = ?webhooks, "Get webhooks"); + Ok(HttpResponse::Ok().json(webhooks)) +} + +#[derive(Debug, Deserr, ToSchema)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] +struct WebhookSettings { + #[schema(value_type = Option)] + #[deserr(default, error = DeserrJsonError)] + #[serde(default)] + url: Setting, + #[schema(value_type = Option>, example = json!({"Authorization":"Bearer a-secret-token"}))] + #[deserr(default, error = DeserrJsonError)] + #[serde(default)] + headers: Setting>>, +} + +#[derive(Debug, Deserr, ToSchema)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] +struct WebhooksSettings { + #[schema(value_type = Option>)] + #[deserr(default, error = DeserrJsonError)] + #[serde(default)] + webhooks: Setting>>, +} + +#[derive(Serialize)] +pub struct PatchWebhooksAnalytics; + +impl Aggregate for PatchWebhooksAnalytics { + fn event_name(&self) -> &'static str { + "Webhooks Updated" + } + + fn aggregate(self: Box, _new: Box) -> Box { + self + } + + fn into_event(self: Box) -> serde_json::Value { + serde_json::to_value(*self).unwrap_or_default() + } +} + +#[derive(Debug, thiserror::Error)] +enum WebhooksError { + #[error("The URL for the webhook `{0}` is missing.")] + MissingUrl(String), +} + +impl ErrorCode for WebhooksError { + fn error_code(&self) -> meilisearch_types::error::Code { + match self { + WebhooksError::MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhooksUrl, + } + } +} + +#[utoipa::path( + patch, + path = "", + tag = "Webhooks", + request_body = WebhooksSettings, + security(("Bearer" = ["webhooks.update", "*"])), + responses( + (status = 200, description = "Returns the updated webhooks", body = WebhooksSettings, content_type = "application/json", example = json!({ + "webhooks": { + "name": { + "url": "http://example.com/webhook", + }, + "anotherName": { + "url": "https://your.site/on-tasks-completed", + "headers": { + "Authorization": "Bearer a-secret-token" + } + } + } + })), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!({ + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + })), + ) +)] +async fn patch_webhooks( + index_scheduler: GuardedData, Data>, + new_webhooks: AwebJson, + req: HttpRequest, + analytics: Data, +) -> Result { + let WebhooksSettings { webhooks: new_webhooks } = new_webhooks.0; + let Webhooks { mut webhooks } = index_scheduler.webhooks(); + debug!(parameters = ?new_webhooks, "Patch webhooks"); + + fn merge_webhook( + name: &str, + old_webhook: Option, + new_webhook: WebhookSettings, + ) -> Result { + let (old_url, mut headers) = + old_webhook.map(|w| (Some(w.url), w.headers)).unwrap_or((None, BTreeMap::new())); + + let url = match new_webhook.url { + Setting::Set(url) => url, + Setting::NotSet => old_url.ok_or_else(|| WebhooksError::MissingUrl(name.to_owned()))?, + Setting::Reset => return Err(WebhooksError::MissingUrl(name.to_owned())), + }; + + let headers = match new_webhook.headers { + Setting::Set(new_headers) => { + for (name, value) in new_headers { + match value { + Setting::Set(value) => { + headers.insert(name, value); + } + Setting::NotSet => continue, + Setting::Reset => { + headers.remove(&name); + continue; + } + } + } + headers + } + Setting::NotSet => headers, + Setting::Reset => BTreeMap::new(), + }; + + Ok(Webhook { url, headers }) + } + + match new_webhooks { + Setting::Set(new_webhooks) => { + for (name, new_webhook) in new_webhooks { + match new_webhook { + Setting::Set(new_webhook) => { + let old_webhook = webhooks.remove(&name); + let webhook = merge_webhook(&name, old_webhook, new_webhook)?; + webhooks.insert(name.clone(), webhook); + } + Setting::Reset => { + webhooks.remove(&name); + } + Setting::NotSet => (), + } + } + } + Setting::Reset => webhooks.clear(), + Setting::NotSet => (), + }; + + analytics.publish(PatchWebhooksAnalytics, &req); + + let webhooks = Webhooks { webhooks }; + index_scheduler.put_webhooks(webhooks.clone())?; + debug!(returns = ?webhooks, "Patch webhooks"); + Ok(HttpResponse::Ok().json(webhooks)) +} From 466e1a7aac704a4d93355aef76902cb08581160a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 12:25:59 +0200 Subject: [PATCH 04/53] Support legacy cli arguments --- crates/index-scheduler/src/lib.rs | 4 ---- crates/index-scheduler/src/test_utils.rs | 2 -- crates/meilisearch-types/src/webhooks.rs | 2 +- crates/meilisearch/src/lib.rs | 28 ++++++++++++++++++++++-- crates/meilisearch/src/option.rs | 2 ++ 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 9e1d8d1a8..dfe8138f3 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -107,10 +107,6 @@ pub struct IndexSchedulerOptions { pub snapshots_path: PathBuf, /// The path to the folder containing the dumps. pub dumps_path: PathBuf, - /// The URL on which we must send the tasks statuses - pub webhook_url: Option, - /// The value we will send into the Authorization HTTP header on the webhook URL - pub webhook_authorization_header: Option, /// The maximum size, in bytes, of the task index. pub task_db_size: usize, /// The size, in bytes, with which a meilisearch index is opened the first time of each meilisearch index. diff --git a/crates/index-scheduler/src/test_utils.rs b/crates/index-scheduler/src/test_utils.rs index bfed7f53a..b7d69b5b3 100644 --- a/crates/index-scheduler/src/test_utils.rs +++ b/crates/index-scheduler/src/test_utils.rs @@ -98,8 +98,6 @@ impl IndexScheduler { indexes_path: tempdir.path().join("indexes"), snapshots_path: tempdir.path().join("snapshots"), dumps_path: tempdir.path().join("dumps"), - webhook_url: None, - webhook_authorization_header: None, task_db_size: 1000 * 1000 * 10, // 10 MB, we don't use MiB on purpose. index_base_map_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. enable_mdb_writemap: false, diff --git a/crates/meilisearch-types/src/webhooks.rs b/crates/meilisearch-types/src/webhooks.rs index c30d32bc6..9d371bd5f 100644 --- a/crates/meilisearch-types/src/webhooks.rs +++ b/crates/meilisearch-types/src/webhooks.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; -#[derive(Debug, Serialize, Deserialize, Clone)] +#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] #[serde(rename_all = "camelCase")] pub struct Webhook { pub url: String, diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 0fb93b65a..24741d22d 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -13,6 +13,7 @@ pub mod routes; pub mod search; pub mod search_queue; +use std::collections::BTreeMap; use std::fs::File; use std::io::{BufReader, BufWriter}; use std::path::Path; @@ -48,6 +49,7 @@ use meilisearch_types::tasks::KindWithContent; use meilisearch_types::versioning::{ create_current_version_file, get_version, VersionFileError, VERSION_MINOR, VERSION_PATCH, }; +use meilisearch_types::webhooks::Webhook; use meilisearch_types::{compression, heed, milli, VERSION_FILE_NAME}; pub use option::Opt; use option::ScheduleSnapshot; @@ -223,8 +225,6 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< indexes_path: opt.db_path.join("indexes"), snapshots_path: opt.snapshot_dir.clone(), dumps_path: opt.dump_dir.clone(), - webhook_url: opt.task_webhook_url.as_ref().map(|url| url.to_string()), - webhook_authorization_header: opt.task_webhook_authorization_header.clone(), task_db_size: opt.max_task_db_size.as_u64() as usize, index_base_map_size: opt.max_index_size.as_u64() as usize, enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage, @@ -327,6 +327,30 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< .unwrap(); } + // We set the webhook url + let cli_webhook = opt.task_webhook_url.as_ref().map(|u| Webhook { + url: u.to_string(), + headers: { + let mut headers = BTreeMap::new(); + if let Some(value) = &opt.task_webhook_authorization_header { + headers.insert(String::from("Authorization"), value.to_string()); + } + headers + }, + }); + let mut webhooks = index_scheduler.webhooks(); + if webhooks.webhooks.get("_cli") != cli_webhook.as_ref() { + match cli_webhook { + Some(webhook) => { + webhooks.webhooks.insert("_cli".to_string(), webhook); + } + None => { + webhooks.webhooks.remove("_cli"); + } + } + index_scheduler.put_webhooks(webhooks)?; + } + Ok((index_scheduler, auth_controller)) } diff --git a/crates/meilisearch/src/option.rs b/crates/meilisearch/src/option.rs index dd77a1222..e27fa08cd 100644 --- a/crates/meilisearch/src/option.rs +++ b/crates/meilisearch/src/option.rs @@ -206,11 +206,13 @@ pub struct Opt { pub env: String, /// Called whenever a task finishes so a third party can be notified. + /// See also the dedicated API `/webhooks`. #[clap(long, env = MEILI_TASK_WEBHOOK_URL)] pub task_webhook_url: Option, /// The Authorization header to send on the webhook URL whenever /// a task finishes so a third party can be notified. + /// See also the dedicated API `/webhooks`. #[clap(long, env = MEILI_TASK_WEBHOOK_AUTHORIZATION_HEADER)] pub task_webhook_authorization_header: Option, From 93f8b31eecc91615505a927372213ff0693871e5 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 12:52:01 +0200 Subject: [PATCH 05/53] Fix tests --- crates/index-scheduler/src/insta_snapshot.rs | 9 ++++++--- crates/index-scheduler/src/lib.rs | 1 + crates/meilisearch/tests/auth/api_keys.rs | 2 +- crates/meilisearch/tests/auth/errors.rs | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index 21626fb2e..f3431dd33 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -62,9 +62,12 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { } snap.push_str("\n----------------------------------------------------------------------\n"); - snap.push_str("### Persisted:\n"); - snap.push_str(&snapshot_persisted_db(&rtxn, persisted)); - snap.push_str("----------------------------------------------------------------------\n"); + let persisted_db_snapshot = snapshot_persisted_db(&rtxn, persisted); + if !persisted_db_snapshot.is_empty() { + snap.push_str("### Persisted:\n"); + snap.push_str(&persisted_db_snapshot); + snap.push_str("----------------------------------------------------------------------\n"); + } snap.push_str("### All Tasks:\n"); snap.push_str(&snapshot_all_tasks(&rtxn, queue.tasks.all_tasks)); diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index dfe8138f3..ce8791a63 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -240,6 +240,7 @@ impl IndexScheduler { + IndexMapper::nb_db() + features::FeatureData::nb_db() + 1 // chat-prompts + + 1 // persisted } /// Create an index scheduler and start its run loop. diff --git a/crates/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs index 6dc3f429b..f16789add 100644 --- a/crates/meilisearch/tests/auth/api_keys.rs +++ b/crates/meilisearch/tests/auth/api_keys.rs @@ -421,7 +421,7 @@ async fn error_add_api_key_invalid_parameters_actions() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r#" { - "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`", + "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`, `webhooks.get`, `webhooks.update`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" diff --git a/crates/meilisearch/tests/auth/errors.rs b/crates/meilisearch/tests/auth/errors.rs index b16ccb2f5..6d3369144 100644 --- a/crates/meilisearch/tests/auth/errors.rs +++ b/crates/meilisearch/tests/auth/errors.rs @@ -93,7 +93,7 @@ async fn create_api_key_bad_actions() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r#" { - "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`", + "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`, `webhooks.get`, `webhooks.update`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" From 064d9d5ff81be5d342a3c44fc7fbea48b41acfd3 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:06:37 +0200 Subject: [PATCH 06/53] Add dump support --- crates/dump/src/reader/compat/v5_to_v6.rs | 4 +++ crates/dump/src/reader/mod.rs | 7 +++++ crates/dump/src/reader/v6/mod.rs | 25 +++++++++++---- crates/dump/src/writer.rs | 11 +++++++ crates/index-scheduler/src/processing.rs | 1 + .../src/scheduler/process_dump_creation.rs | 5 +++ crates/meilisearch-types/src/webhooks.rs | 2 +- crates/meilisearch/src/lib.rs | 31 +++++++++++-------- 8 files changed, 66 insertions(+), 20 deletions(-) diff --git a/crates/dump/src/reader/compat/v5_to_v6.rs b/crates/dump/src/reader/compat/v5_to_v6.rs index f173bb6bd..3a0c8ef0d 100644 --- a/crates/dump/src/reader/compat/v5_to_v6.rs +++ b/crates/dump/src/reader/compat/v5_to_v6.rs @@ -202,6 +202,10 @@ impl CompatV5ToV6 { pub fn network(&self) -> Result> { Ok(None) } + + pub fn webhooks(&self) -> Option<&v6::Webhooks> { + None + } } pub enum CompatIndexV5ToV6 { diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index c894c255f..328a01f60 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -138,6 +138,13 @@ impl DumpReader { DumpReader::Compat(compat) => compat.network(), } } + + pub fn webhooks(&self) -> Option<&v6::Webhooks> { + match self { + DumpReader::Current(current) => current.webhooks(), + DumpReader::Compat(compat) => compat.webhooks(), + } + } } impl From for DumpReader { diff --git a/crates/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs index 08d4700e5..d8ce430f9 100644 --- a/crates/dump/src/reader/v6/mod.rs +++ b/crates/dump/src/reader/v6/mod.rs @@ -25,6 +25,7 @@ pub type Key = meilisearch_types::keys::Key; pub type ChatCompletionSettings = meilisearch_types::features::ChatCompletionSettings; pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures; pub type Network = meilisearch_types::features::Network; +pub type Webhooks = meilisearch_types::webhooks::Webhooks; // ===== Other types to clarify the code of the compat module // everything related to the tasks @@ -59,6 +60,7 @@ pub struct V6Reader { keys: BufReader, features: Option, network: Option, + webhooks: Option, } impl V6Reader { @@ -93,8 +95,8 @@ impl V6Reader { Err(e) => return Err(e.into()), }; - let network_file = match fs::read(dump.path().join("network.json")) { - Ok(network_file) => Some(network_file), + let network = match fs::read(dump.path().join("network.json")) { + Ok(network_file) => Some(serde_json::from_reader(&*network_file)?), Err(error) => match error.kind() { // Allows the file to be missing, this will only result in all experimental features disabled. ErrorKind::NotFound => { @@ -104,10 +106,16 @@ impl V6Reader { _ => return Err(error.into()), }, }; - let network = if let Some(network_file) = network_file { - Some(serde_json::from_reader(&*network_file)?) - } else { - None + + let webhooks = match fs::read(dump.path().join("webhooks.json")) { + Ok(webhooks_file) => Some(serde_json::from_reader(&*webhooks_file)?), + Err(error) => match error.kind() { + ErrorKind::NotFound => { + debug!("`webhooks.json` not found in dump"); + None + } + _ => return Err(error.into()), + }, }; Ok(V6Reader { @@ -119,6 +127,7 @@ impl V6Reader { features, network, dump, + webhooks, }) } @@ -229,6 +238,10 @@ impl V6Reader { pub fn network(&self) -> Option<&Network> { self.network.as_ref() } + + pub fn webhooks(&self) -> Option<&Webhooks> { + self.webhooks.as_ref() + } } pub struct UpdateFile { diff --git a/crates/dump/src/writer.rs b/crates/dump/src/writer.rs index 9f828595a..84a76e483 100644 --- a/crates/dump/src/writer.rs +++ b/crates/dump/src/writer.rs @@ -8,6 +8,7 @@ use meilisearch_types::batches::Batch; use meilisearch_types::features::{ChatCompletionSettings, Network, RuntimeTogglableFeatures}; use meilisearch_types::keys::Key; use meilisearch_types::settings::{Checked, Settings}; +use meilisearch_types::webhooks::Webhooks; use serde_json::{Map, Value}; use tempfile::TempDir; use time::OffsetDateTime; @@ -74,6 +75,16 @@ impl DumpWriter { Ok(std::fs::write(self.dir.path().join("network.json"), serde_json::to_string(&network)?)?) } + pub fn create_webhooks(&self, webhooks: Webhooks) -> Result<()> { + if webhooks == Webhooks::default() { + return Ok(()); + } + Ok(std::fs::write( + self.dir.path().join("webhooks.json"), + serde_json::to_string(&webhooks)?, + )?) + } + pub fn persist_to(self, mut writer: impl Write) -> Result<()> { let gz_encoder = GzEncoder::new(&mut writer, Compression::default()); let mut tar_encoder = tar::Builder::new(gz_encoder); diff --git a/crates/index-scheduler/src/processing.rs b/crates/index-scheduler/src/processing.rs index fdd8e42ef..3da81f143 100644 --- a/crates/index-scheduler/src/processing.rs +++ b/crates/index-scheduler/src/processing.rs @@ -108,6 +108,7 @@ make_enum_progress! { DumpTheBatches, DumpTheIndexes, DumpTheExperimentalFeatures, + DumpTheWebhooks, CompressTheDump, } } diff --git a/crates/index-scheduler/src/scheduler/process_dump_creation.rs b/crates/index-scheduler/src/scheduler/process_dump_creation.rs index b14f23d0b..8f47cbd0c 100644 --- a/crates/index-scheduler/src/scheduler/process_dump_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_dump_creation.rs @@ -270,6 +270,11 @@ impl IndexScheduler { let network = self.network(); dump.create_network(network)?; + // 7. Dump the webhooks + progress.update_progress(DumpCreationProgress::DumpTheWebhooks); + let webhooks = self.webhooks(); + dump.create_webhooks(webhooks)?; + let dump_uid = started_at.format(format_description!( "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]" )).unwrap(); diff --git a/crates/meilisearch-types/src/webhooks.rs b/crates/meilisearch-types/src/webhooks.rs index 9d371bd5f..8849182ac 100644 --- a/crates/meilisearch-types/src/webhooks.rs +++ b/crates/meilisearch-types/src/webhooks.rs @@ -10,7 +10,7 @@ pub struct Webhook { pub headers: BTreeMap, } -#[derive(Debug, Serialize, Deserialize, Default, Clone)] +#[derive(Debug, Serialize, Deserialize, Default, Clone, PartialEq)] #[serde(rename_all = "camelCase")] pub struct Webhooks { #[serde(default)] diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 24741d22d..fcc71f04d 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -515,7 +515,12 @@ fn import_dump( let _ = std::fs::write(db_path.join("instance-uid"), instance_uid.to_string().as_bytes()); }; - // 2. Import the `Key`s. + // 2. Import the webhooks + if let Some(webhooks) = dump_reader.webhooks() { + index_scheduler.put_webhooks(webhooks.clone())?; + } + + // 3. Import the `Key`s. let mut keys = Vec::new(); auth.raw_delete_all_keys()?; for key in dump_reader.keys()? { @@ -524,20 +529,20 @@ fn import_dump( keys.push(key); } - // 3. Import the `ChatCompletionSettings`s. + // 4. Import the `ChatCompletionSettings`s. for result in dump_reader.chat_completions_settings()? { let (name, settings) = result?; index_scheduler.put_chat_settings(&name, &settings)?; } - // 4. Import the runtime features and network + // 5. Import the runtime features and network let features = dump_reader.features()?.unwrap_or_default(); index_scheduler.put_runtime_features(features)?; let network = dump_reader.network()?.cloned().unwrap_or_default(); index_scheduler.put_network(network)?; - // 4.1 Use all cpus to process dump if `max_indexing_threads` not configured + // 5.1 Use all cpus to process dump if `max_indexing_threads` not configured let backup_config; let base_config = index_scheduler.indexer_config(); @@ -554,7 +559,7 @@ fn import_dump( // /!\ The tasks must be imported AFTER importing the indexes or else the scheduler might // try to process tasks while we're trying to import the indexes. - // 5. Import the indexes. + // 6. Import the indexes. for index_reader in dump_reader.indexes()? { let mut index_reader = index_reader?; let metadata = index_reader.metadata(); @@ -567,12 +572,12 @@ fn import_dump( let mut wtxn = index.write_txn()?; let mut builder = milli::update::Settings::new(&mut wtxn, &index, indexer_config); - // 5.1 Import the primary key if there is one. + // 6.1 Import the primary key if there is one. if let Some(ref primary_key) = metadata.primary_key { builder.set_primary_key(primary_key.to_string()); } - // 5.2 Import the settings. + // 6.2 Import the settings. tracing::info!("Importing the settings."); let settings = index_reader.settings()?; apply_settings_to_builder(&settings, &mut builder); @@ -584,8 +589,8 @@ fn import_dump( let rtxn = index.read_txn()?; if index_scheduler.no_edition_2024_for_dumps() { - // 5.3 Import the documents. - // 5.3.1 We need to recreate the grenad+obkv format accepted by the index. + // 6.3 Import the documents. + // 6.3.1 We need to recreate the grenad+obkv format accepted by the index. tracing::info!("Importing the documents."); let file = tempfile::tempfile()?; let mut builder = DocumentsBatchBuilder::new(BufWriter::new(file)); @@ -596,7 +601,7 @@ fn import_dump( // This flush the content of the batch builder. let file = builder.into_inner()?.into_inner()?; - // 5.3.2 We feed it to the milli index. + // 6.3.2 We feed it to the milli index. let reader = BufReader::new(file); let reader = DocumentsBatchReader::from_reader(reader)?; @@ -675,15 +680,15 @@ fn import_dump( index_scheduler.refresh_index_stats(&uid)?; } - // 6. Import the queue + // 7. Import the queue let mut index_scheduler_dump = index_scheduler.register_dumped_task()?; - // 6.1. Import the batches + // 7.1. Import the batches for ret in dump_reader.batches()? { let batch = ret?; index_scheduler_dump.register_dumped_batch(batch)?; } - // 6.2. Import the tasks + // 7.2. Import the tasks for ret in dump_reader.tasks()? { let (task, file) = ret?; index_scheduler_dump.register_dumped_task(task, file)?; From dc7af47371274f2367d739b8f5b0c857034782bd Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:18:43 +0200 Subject: [PATCH 07/53] Add new errors --- crates/meilisearch/src/routes/webhooks.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index d05c16672..99306fa1e 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -125,12 +125,20 @@ impl Aggregate for PatchWebhooksAnalytics { enum WebhooksError { #[error("The URL for the webhook `{0}` is missing.")] MissingUrl(String), + #[error("Defining too many webhooks would crush the server. Please limit the number of webhooks to 20. You may use a third-party proxy server to dispatch events to more than 20 endpoints.")] + TooManyWebhooks, + #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] + TooManyHeaders(String), } impl ErrorCode for WebhooksError { fn error_code(&self) -> meilisearch_types::error::Code { match self { WebhooksError::MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhooksUrl, + WebhooksError::TooManyWebhooks => meilisearch_types::error::Code::InvalidWebhooks, + WebhooksError::TooManyHeaders(_) => { + meilisearch_types::error::Code::InvalidWebhooksHeaders + } } } } @@ -217,6 +225,9 @@ async fn patch_webhooks( Setting::Set(new_webhook) => { let old_webhook = webhooks.remove(&name); let webhook = merge_webhook(&name, old_webhook, new_webhook)?; + if webhook.headers.len() > 200 { + return Err(WebhooksError::TooManyHeaders(name).into()); + } webhooks.insert(name.clone(), webhook); } Setting::Reset => { @@ -230,6 +241,10 @@ async fn patch_webhooks( Setting::NotSet => (), }; + if webhooks.len() > 20 { + return Err(WebhooksError::TooManyWebhooks.into()); + } + analytics.publish(PatchWebhooksAnalytics, &req); let webhooks = Webhooks { webhooks }; From 3e77c1d8c84d0af236ee344b4bb64d7aff46d418 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:23:06 +0200 Subject: [PATCH 08/53] Add reserved webhook --- crates/meilisearch-types/src/error.rs | 3 ++- crates/meilisearch/src/routes/webhooks.rs | 14 +++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 92425d386..56590e79d 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -422,7 +422,8 @@ InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQU // Webhooks InvalidWebhooks , InvalidRequest , BAD_REQUEST ; InvalidWebhooksUrl , InvalidRequest , BAD_REQUEST ; -InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST +InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST ; +ReservedWebhook , InvalidRequest , BAD_REQUEST } impl ErrorCode for JoinError { diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 99306fa1e..631dd822d 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -129,6 +129,8 @@ enum WebhooksError { TooManyWebhooks, #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] TooManyHeaders(String), + #[error("Cannot edit webhook `{0}`. Webhooks prefixed with an underscore are special and may not be modified using the API.")] + ReservedWebhook(String), } impl ErrorCode for WebhooksError { @@ -139,6 +141,7 @@ impl ErrorCode for WebhooksError { WebhooksError::TooManyHeaders(_) => { meilisearch_types::error::Code::InvalidWebhooksHeaders } + WebhooksError::ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, } } } @@ -186,6 +189,10 @@ async fn patch_webhooks( old_webhook: Option, new_webhook: WebhookSettings, ) -> Result { + if name.starts_with('_') { + return Err(WebhooksError::ReservedWebhook(name.to_owned())); + } + let (old_url, mut headers) = old_webhook.map(|w| (Some(w.url), w.headers)).unwrap_or((None, BTreeMap::new())); @@ -215,6 +222,10 @@ async fn patch_webhooks( Setting::Reset => BTreeMap::new(), }; + if headers.len() > 200 { + return Err(WebhooksError::TooManyHeaders(name.to_owned())); + } + Ok(Webhook { url, headers }) } @@ -225,9 +236,6 @@ async fn patch_webhooks( Setting::Set(new_webhook) => { let old_webhook = webhooks.remove(&name); let webhook = merge_webhook(&name, old_webhook, new_webhook)?; - if webhook.headers.len() > 200 { - return Err(WebhooksError::TooManyHeaders(name).into()); - } webhooks.insert(name.clone(), webhook); } Setting::Reset => { From b565ec1497d8978ca31064fa075fbb01d9248b8b Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:44:42 +0200 Subject: [PATCH 09/53] Test cli behavior --- crates/meilisearch/tests/common/server.rs | 8 ++++++++ crates/meilisearch/tests/tasks/webhook.rs | 18 +++++++++++++++++- 2 files changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index ad0678122..1dfe2e593 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -182,6 +182,10 @@ impl Server { self.service.patch("/network", value).await } + pub async fn set_webhooks(&self, value: Value) -> (Value, StatusCode) { + self.service.patch("/webhooks", value).await + } + pub async fn get_metrics(&self) -> (Value, StatusCode) { self.service.get("/metrics").await } @@ -447,6 +451,10 @@ impl Server { pub async fn get_network(&self) -> (Value, StatusCode) { self.service.get("/network").await } + + pub async fn get_webhooks(&self) -> (Value, StatusCode) { + self.service.get("/webhooks").await + } } pub fn default_settings(dir: impl AsRef) -> Opt { diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index b18002eb7..984cfc23e 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -68,12 +68,13 @@ async fn create_webhook_server() -> WebhookHandle { } #[actix_web::test] -async fn test_basic_webhook() { +async fn test_cli_webhook() { let WebhookHandle { server_handle, url, mut receiver } = create_webhook_server().await; let db_path = tempfile::tempdir().unwrap(); let server = Server::new_with_options(Opt { task_webhook_url: Some(Url::parse(&url).unwrap()), + task_webhook_authorization_header: Some(String::from("Bearer a-secret-token")), ..default_settings(db_path.path()) }) .await @@ -125,5 +126,20 @@ async fn test_basic_webhook() { assert!(nb_tasks == 5, "We should have received the 5 tasks but only received {nb_tasks}"); + let (webhooks, code) = server.get_webhooks().await; + snapshot!(code, @"200 OK"); + snapshot!(webhooks, @r#" + { + "webhooks": { + "_cli": { + "url": "http://127.0.0.1:51503/", + "headers": { + "Authorization": "Bearer a-secret-token" + } + } + } + } + "#); + server_handle.abort(); } From e88480c7c4d3f37d98fa80004ac0515048ead489 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:44:51 +0200 Subject: [PATCH 10/53] Fix reserved name check --- crates/meilisearch/src/routes/webhooks.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 631dd822d..9cf57c585 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -189,10 +189,6 @@ async fn patch_webhooks( old_webhook: Option, new_webhook: WebhookSettings, ) -> Result { - if name.starts_with('_') { - return Err(WebhooksError::ReservedWebhook(name.to_owned())); - } - let (old_url, mut headers) = old_webhook.map(|w| (Some(w.url), w.headers)).unwrap_or((None, BTreeMap::new())); @@ -232,6 +228,10 @@ async fn patch_webhooks( match new_webhooks { Setting::Set(new_webhooks) => { for (name, new_webhook) in new_webhooks { + if name.starts_with('_') { + return Err(WebhooksError::ReservedWebhook(name).into()); + } + match new_webhook { Setting::Set(new_webhook) => { let old_webhook = webhooks.remove(&name); From c70ae91d3442ab81fbbbf50f03756e135b8f20a4 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:52:24 +0200 Subject: [PATCH 11/53] Add test for reserved webhooks --- crates/meilisearch/src/routes/webhooks.rs | 2 +- crates/meilisearch/tests/tasks/webhook.rs | 35 +++++++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 9cf57c585..adca710a0 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -129,7 +129,7 @@ enum WebhooksError { TooManyWebhooks, #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] TooManyHeaders(String), - #[error("Cannot edit webhook `{0}`. Webhooks prefixed with an underscore are special and may not be modified using the API.")] + #[error("Cannot edit webhook `{0}`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.")] ReservedWebhook(String), } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 984cfc23e..5b77394f8 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -8,7 +8,7 @@ use actix_http::body::MessageBody; use actix_web::dev::{ServiceFactory, ServiceResponse}; use actix_web::web::{Bytes, Data}; use actix_web::{post, App, HttpRequest, HttpResponse, HttpServer}; -use meili_snap::snapshot; +use meili_snap::{json_string, snapshot}; use meilisearch::Opt; use tokio::sync::mpsc; use url::Url; @@ -128,11 +128,11 @@ async fn test_cli_webhook() { let (webhooks, code) = server.get_webhooks().await; snapshot!(code, @"200 OK"); - snapshot!(webhooks, @r#" + snapshot!(json_string!(webhooks, { ".webhooks._cli.url" => "[ignored]" }), @r#" { "webhooks": { "_cli": { - "url": "http://127.0.0.1:51503/", + "url": "[ignored]", "headers": { "Authorization": "Bearer a-secret-token" } @@ -143,3 +143,32 @@ async fn test_cli_webhook() { server_handle.abort(); } + +#[actix_web::test] +async fn reserved_names() { + let server = Server::new().await; + + let (value, code) = server + .set_webhooks(json!({ "webhooks": { "_cli": { "url": "http://localhost:8080" } } })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Cannot edit webhook `_cli`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", + "code": "reserved_webhook", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#reserved_webhook" + } + "#); + + let (value, code) = server.set_webhooks(json!({ "webhooks": { "_cli": null } })).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Cannot edit webhook `_cli`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", + "code": "reserved_webhook", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#reserved_webhook" + } + "#); +} From a75b327b376f1f568d624be1eec2fe9733edd004 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 15:59:19 +0200 Subject: [PATCH 12/53] Add test for webhooks over limits --- crates/meilisearch/tests/tasks/webhook.rs | 55 +++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 5b77394f8..c271446d8 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -172,3 +172,58 @@ async fn reserved_names() { } "#); } + +#[actix_web::test] +async fn over_limits() { + let server = Server::new().await; + + // Too many webhooks + for i in 0..20 { + let (_value, code) = server + .set_webhooks(json!({ "webhooks": { format!("webhook_{i}"): { "url": "http://localhost:8080" } } })) + .await; + snapshot!(code, @"200 OK"); + } + let (value, code) = server + .set_webhooks(json!({ "webhooks": { "webhook_21": { "url": "http://localhost:8080" } } })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Defining too many webhooks would crush the server. Please limit the number of webhooks to 20. You may use a third-party proxy server to dispatch events to more than 20 endpoints.", + "code": "invalid_webhooks", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks" + } + "#); + + // Reset webhooks + let (value, code) = server.set_webhooks(json!({ "webhooks": null })).await; + snapshot!(code, @"200 OK"); + snapshot!(value, @r#" + { + "webhooks": {} + } + "#); + + // Test too many headers + for i in 0..200 { + let header_name = format!("header_{i}"); + let (_value, code) = server + .set_webhooks(json!({ "webhooks": { "webhook": { "url": "http://localhost:8080", "headers": { header_name: "value" } } } })) + .await; + snapshot!(code, @"200 OK"); + } + let (value, code) = server + .set_webhooks(json!({ "webhooks": { "webhook": { "url": "http://localhost:8080", "headers": { "header_201": "value" } } } })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Too many headers for the webhook `webhook`. Please limit the number of headers to 200.", + "code": "invalid_webhooks_headers", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" + } + "#); +} From fc4c5d2718731442517059f3eceff8dcacce42c2 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Wed, 30 Jul 2025 16:16:12 +0200 Subject: [PATCH 13/53] Add dump test --- crates/dump/src/reader/mod.rs | 39 ++++++++++++++++++ .../dump/tests/assets/v6-with-webhooks.dump | Bin 0 -> 1693 bytes 2 files changed, 39 insertions(+) create mode 100644 crates/dump/tests/assets/v6-with-webhooks.dump diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index 328a01f60..a34365905 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -372,6 +372,7 @@ pub(crate) mod test { assert_eq!(dump.features().unwrap().unwrap(), RuntimeTogglableFeatures::default()); assert_eq!(dump.network().unwrap(), None); + assert_eq!(dump.webhooks(), None); } #[test] @@ -442,6 +443,44 @@ pub(crate) mod test { insta::assert_snapshot!(network.remotes.get("ms-2").as_ref().unwrap().search_api_key.as_ref().unwrap(), @"foo"); } + #[test] + fn import_dump_v6_webhooks() { + let dump = File::open("tests/assets/v6-with-webhooks.dump").unwrap(); + let dump = DumpReader::open(dump).unwrap(); + + // top level infos + insta::assert_snapshot!(dump.date().unwrap(), @"2025-07-30 14:06:57.240882 +00:00:00"); + insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @r" + Some( + cb887dcc-34b3-48d1-addd-9815ae721a81, + ) + "); + + // webhooks + + let webhooks = dump.webhooks().unwrap(); + insta::assert_json_snapshot!(webhooks, @r#" + { + "webhooks": { + "exampleName": { + "url": "https://example.com/hook", + "headers": { + "authorization": "TOKEN" + } + }, + "otherName": { + "url": "https://example.com/authorization-less", + "headers": {} + }, + "third": { + "url": "https://third.com", + "headers": {} + } + } + } + "#); + } + #[test] fn import_dump_v5() { let dump = File::open("tests/assets/v5.dump").unwrap(); diff --git a/crates/dump/tests/assets/v6-with-webhooks.dump b/crates/dump/tests/assets/v6-with-webhooks.dump new file mode 100644 index 0000000000000000000000000000000000000000..89b1f61be5a0cb74449d0f372aa388fb7f5ac1f1 GIT binary patch literal 1693 zcmV;O24eXiiwFP!00000|Ls~|lbbdW_j5i4p)=`SGSkUS(n&9QXr_}0 zB+m3OxFF_o_1wqm7w9MJN?`lWcMeW!J4vJ8gE3+yV3*ZOzg_oabZ~ec@j^=B=y_q# z9X+3P#||SzJ>nA|6A!8O5QpA~j*ft~k*v0?(SODAMV7~J1bNBD2hofUiBGpb@&bg7 z{f~fz$#nV^+Mj})vH#)G+yCUxR{$4k^6Yuzc^%(4_yV2XrI6b3dh! zfVe&y?$IwCpa;d#PCdkjxv2A*DB+v2GO$=%>;5DP-6$IFeRcUn~*AKT@P1#9( zKda9}J)#@n2j$gRUR!U61@8^K&4gU*a&PPXZQp(m`d41V%CeGH zvi!t&*N>IgRs{yC^;oWb^Jrx7C6Jf!sg|jD2VuZ%;b*je48(s}9+J)Y@6!OA_F}929qIp%dQmk48&*fQ_w(2Wse!70vmT9-z?yvXDgXURYNKvnv zOChhS`f_!CA*9+UtVX(f6SZusnp}#fvqW$%>h%+;<@f1o^}TMNwS~-Q6=y}?bTcye zMD#}5v&Y}XVEo7arvIbB3yuFD0kBaQMP^URwkXDSEoPO}`}7;*OWy@x_Gg?)cE|}} z$p4Z5fd7Yq@&6;BBjnydxb66}Hvv!vJXtp&7YGthHGh7%+H9O9w>jD$J|q7_{J`Y@jsZo3A1Lj!!chJP(|h&@0?0S>{}BN5zi84r zpZEEb*9}nnQjyvXFwDUHW_2~wR66`Ar6@Usf|AWt7&sFdgDsf#YbW{Sm!GZY;u@~& zl#G2rc-?oNC%%@!EM~DPEf`)l>nE zqIqxEI%Tc;^MiF(X&HF-sv8DUN6ebHNs*@?QRfpr%gcM(WIPEnEU1$T>N|vD;jlP` z#9T%=4?Qo5aduamkN2iXiv1n+s3o_m5xy$88Hd~vD0HUPWwErb`ee=3v??HF1FO-5 zHDaqG!3vE^-|eVQ$Z2rrrAk{otk@_^upjuyNiY+j*fek==6en$ltf<4z?nA{>jISl zO?vKBfA^$k$n^MlH%0fNAJeo>TC$RVf+L@*E}QxxVH~3*2%Ruuu|q@_IShG$lai48 zC?uF==&pV?`I_;b9h>?X)7|vci<UvmG4@jd<*zALl;I|df^I;{>#C;deib_*w}2RPsq=2p$rE^L^qr&XQ*wJgLqfBF3B3!Mws zdEVh6?+oA+kfpzK12q!N{58Knif&e8Y;-Uj)-ZGp!|1U1occcuN3;JsJhc5!x00G(F71bTsQmusZYA(Q_3!=*krn5ppAMOrVPb}f875|!nDG_G4E4xEF`_Hj nzy$6$5F>iYIE)5kL|+9Gg%}&c-N3-W;H$^K8)Yw;07L))oiSHZ literal 0 HcmV?d00001 From f67043801b81b7266635e367f3833b6c7d34509c Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 09:35:16 +0200 Subject: [PATCH 14/53] Add a test for concurrent cli and dump --- crates/dump/src/reader/mod.rs | 8 ++- .../dump/tests/assets/v6-with-webhooks.dump | Bin 1693 -> 1913 bytes crates/meilisearch/tests/tasks/webhook.rs | 46 +++++++++++++++++- 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index a34365905..85e5df432 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -449,7 +449,7 @@ pub(crate) mod test { let dump = DumpReader::open(dump).unwrap(); // top level infos - insta::assert_snapshot!(dump.date().unwrap(), @"2025-07-30 14:06:57.240882 +00:00:00"); + insta::assert_snapshot!(dump.date().unwrap(), @"2025-07-31 7:28:28.091553 +00:00:00"); insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @r" Some( cb887dcc-34b3-48d1-addd-9815ae721a81, @@ -462,6 +462,12 @@ pub(crate) mod test { insta::assert_json_snapshot!(webhooks, @r#" { "webhooks": { + "_cli": { + "url": "https://defined-in-dump.com/", + "headers": { + "Authorization": "Bearer defined in dump" + } + }, "exampleName": { "url": "https://example.com/hook", "headers": { diff --git a/crates/dump/tests/assets/v6-with-webhooks.dump b/crates/dump/tests/assets/v6-with-webhooks.dump index 89b1f61be5a0cb74449d0f372aa388fb7f5ac1f1..83989f25c4cc882dafeea5c60d917d594867f019 100644 GIT binary patch literal 1913 zcmV-<2Zs0`iwFP!00000|Lt4bj@vd9_Vxc1`gK`YS{`-qMS(Qi0!4weXt%v+QKYm; z$7*EBl_+esi+!xVK%cBbO19VDb>vNKH);4`v9hQkDa~kR$eB@iF*-RAVkqVujtIq3 zcSMAB#||UR5hD@ijN0~?Aw1%vGoWp>s6B7=zF0lT>iCf$FQs~;n$aoobopZx!F4D9 z(_<=hHva(S&*R9+{|xBmfAL4FS{44@RAn(bHb9&xVr-xM8H3+D@~4R7(OH=IiOc`i zZ*%ES(Abwsi@a$3$*nKj#qvkf>=Q&diYW>(L!8qn=KkH-2bO6|s4Fds7t<@zyoPh! zU1{y5rp+>?q=Hc}bkR3UwI2IiP^>rjB=fTAnR zCWucs4I|JbPJV{LW_g)6^S6y%)ezkQv-3JTv%*8~fnib*|@LECu z6_Z^*QQmkJ2&lGWh5p_DdP9T{FhmiL2~9t6L&QRko??h-$ll8k5t1BkQSQCeaG!i} zs2Kw4lq}xG2+?S9xDARzGEBZPKqMw1i{8TkAslk{Bm+bvuz7rMm&a@n3vsm122mEq z$ssm~aRkZhyV^$<0@Jp;f(ZE|tEn>euV@S4x}p`Ff=7 zNz9U|YI3cF!K|%K? zRw2dF@$e4$Uk0%zm;XHjO!WCgY2P&l>_4IWf&4LvsI&iP0N8)kWOcsm(-uEA5LwqI zn%qEG7Xq%cx?0#UHvA~HDkZprQY?%cxKuL%TQKj3j`G=eUwhxE8@O&WIlfF&G8Ksm zlt@ItaGC~bmhnK&NF)gtl#2w2Mb_`?e(}30{a^c3jIxcNnR5tJG^2_Knc^g1oT@;i8RWxeOv)Ic zX-a3?-2CiljO3IY$VW@LmyPgN!DJFrH^9(^WtS(iy6O`p*Ym1?bP23Rr=n4N83|Ho zOxEl`c0wXS2H%=o!Q+fgW>ZQcf`cg)3K*M3K_UnWIO8lqsQ_i(Wo!tc3}~{p)BHRV zpCQvr=wXcRN#Dk4n=2Hhe1alBH(hr5!{Ri>(*(n)&M~PN%*Y^%t+u|YWZ*! z+ib?z>G08q$!zh!=(Kp3`d^BP+y9*bR{j^ABYio`i&K<>zk~cq%w7HWInc@fl-1uu z{vY=G5A(l-N5uL6GhkEyZHuC8v1PXoUsXe{&Q}$c_X;YzYRZSjl(&9Y)cA{wF5HhU zj9ql$7}15Xi!Pigx-fRpg@d9CcIHhTr72oHpsddBT<#USU)nn1&X)GWJofp`-FEry zK=uFeKSa1+s_%S;^BK-(IG^Es#)s)MR$odD_>7&;G*0lz$l7TT=QNzta8AQH4d*mI zAg5t&dB2mmU*(rDMDSiGu~iR<5{mabi7is_Jk(`r9UL4S&WV2jsh^k*08Rh^55LRg literal 1693 zcmV;O24eXiiwFP!00000|Ls~|lbbdW_j5i4p)=`SGSkUS(n&9QXr_}0 zB+m3OxFF_o_1wqm7w9MJN?`lWcMeW!J4vJ8gE3+yV3*ZOzg_oabZ~ec@j^=B=y_q# z9X+3P#||SzJ>nA|6A!8O5QpA~j*ft~k*v0?(SODAMV7~J1bNBD2hofUiBGpb@&bg7 z{f~fz$#nV^+Mj})vH#)G+yCUxR{$4k^6Yuzc^%(4_yV2XrI6b3dh! zfVe&y?$IwCpa;d#PCdkjxv2A*DB+v2GO$=%>;5DP-6$IFeRcUn~*AKT@P1#9( zKda9}J)#@n2j$gRUR!U61@8^K&4gU*a&PPXZQp(m`d41V%CeGH zvi!t&*N>IgRs{yC^;oWb^Jrx7C6Jf!sg|jD2VuZ%;b*je48(s}9+J)Y@6!OA_F}929qIp%dQmk48&*fQ_w(2Wse!70vmT9-z?yvXDgXURYNKvnv zOChhS`f_!CA*9+UtVX(f6SZusnp}#fvqW$%>h%+;<@f1o^}TMNwS~-Q6=y}?bTcye zMD#}5v&Y}XVEo7arvIbB3yuFD0kBaQMP^URwkXDSEoPO}`}7;*OWy@x_Gg?)cE|}} z$p4Z5fd7Yq@&6;BBjnydxb66}Hvv!vJXtp&7YGthHGh7%+H9O9w>jD$J|q7_{J`Y@jsZo3A1Lj!!chJP(|h&@0?0S>{}BN5zi84r zpZEEb*9}nnQjyvXFwDUHW_2~wR66`Ar6@Usf|AWt7&sFdgDsf#YbW{Sm!GZY;u@~& zl#G2rc-?oNC%%@!EM~DPEf`)l>nE zqIqxEI%Tc;^MiF(X&HF-sv8DUN6ebHNs*@?QRfpr%gcM(WIPEnEU1$T>N|vD;jlP` z#9T%=4?Qo5aduamkN2iXiv1n+s3o_m5xy$88Hd~vD0HUPWwErb`ee=3v??HF1FO-5 zHDaqG!3vE^-|eVQ$Z2rrrAk{otk@_^upjuyNiY+j*fek==6en$ltf<4z?nA{>jISl zO?vKBfA^$k$n^MlH%0fNAJeo>TC$RVf+L@*E}QxxVH~3*2%Ruuu|q@_IShG$lai48 zC?uF==&pV?`I_;b9h>?X)7|vci<UvmG4@jd<*zALl;I|df^I;{>#C;deib_*w}2RPsq=2p$rE^L^qr&XQ*wJgLqfBF3B3!Mws zdEVh6?+oA+kfpzK12q!N{58Knif&e8Y;-Uj)-ZGp!|1U1occcuN3;JsJhc5!x00G(F71bTsQmusZYA(Q_3!=*krn5ppAMOrVPb}f875|!nDG_G4E4xEF`_Hj nzy$6$5F>iYIE)5kL|+9Gg%}&c-N3-W;H$^K8)Yw;07L))oiSHZ diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index c271446d8..12a8228fa 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -2,6 +2,7 @@ //! post requests. The webhook handle starts a server and forwards all the //! received requests into a channel for you to handle. +use std::path::PathBuf; use std::sync::Arc; use actix_http::body::MessageBody; @@ -68,7 +69,7 @@ async fn create_webhook_server() -> WebhookHandle { } #[actix_web::test] -async fn test_cli_webhook() { +async fn cli_only() { let WebhookHandle { server_handle, url, mut receiver } = create_webhook_server().await; let db_path = tempfile::tempdir().unwrap(); @@ -144,6 +145,49 @@ async fn test_cli_webhook() { server_handle.abort(); } + +#[actix_web::test] +async fn cli_with_dumps() { + let db_path = tempfile::tempdir().unwrap(); + let server = Server::new_with_options(Opt { + task_webhook_url: Some(Url::parse("http://defined-in-test-cli.com").unwrap()), + task_webhook_authorization_header: Some(String::from("Bearer a-secret-token-defined-in-test-cli")), + import_dump: Some(PathBuf::from("../dump/tests/assets/v6-with-webhooks.dump")), + ..default_settings(db_path.path()) + }) + .await + .unwrap(); + + let (webhooks, code) = server.get_webhooks().await; + snapshot!(code, @"200 OK"); + snapshot!(webhooks, @r#" + { + "webhooks": { + "_cli": { + "url": "http://defined-in-test-cli.com/", + "headers": { + "Authorization": "Bearer a-secret-token-defined-in-test-cli" + } + }, + "exampleName": { + "url": "https://example.com/hook", + "headers": { + "authorization": "TOKEN" + } + }, + "otherName": { + "url": "https://example.com/authorization-less", + "headers": {} + }, + "third": { + "url": "https://third.com", + "headers": {} + } + } + } + "#); +} + #[actix_web::test] async fn reserved_names() { let server = Server::new().await; From 446fce6c1679df3a94112804fe0b79285d1f53bd Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 10:01:25 +0200 Subject: [PATCH 15/53] Extract logic from route --- crates/meilisearch/src/routes/webhooks.rs | 25 ++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index adca710a0..710873d55 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -180,10 +180,17 @@ async fn patch_webhooks( req: HttpRequest, analytics: Data, ) -> Result { - let WebhooksSettings { webhooks: new_webhooks } = new_webhooks.0; - let Webhooks { mut webhooks } = index_scheduler.webhooks(); - debug!(parameters = ?new_webhooks, "Patch webhooks"); + let webhooks = patch_webhooks_inner(&index_scheduler, new_webhooks.0)?; + analytics.publish(PatchWebhooksAnalytics, &req); + + Ok(HttpResponse::Ok().json(webhooks)) +} + +fn patch_webhooks_inner( + index_scheduler: &GuardedData, Data>, + new_webhooks: WebhooksSettings, +) -> Result { fn merge_webhook( name: &str, old_webhook: Option, @@ -225,7 +232,11 @@ async fn patch_webhooks( Ok(Webhook { url, headers }) } - match new_webhooks { + debug!(parameters = ?new_webhooks, "Patch webhooks"); + + let Webhooks { mut webhooks } = index_scheduler.webhooks(); + + match new_webhooks.webhooks { Setting::Set(new_webhooks) => { for (name, new_webhook) in new_webhooks { if name.starts_with('_') { @@ -253,10 +264,10 @@ async fn patch_webhooks( return Err(WebhooksError::TooManyWebhooks.into()); } - analytics.publish(PatchWebhooksAnalytics, &req); - let webhooks = Webhooks { webhooks }; index_scheduler.put_webhooks(webhooks.clone())?; + debug!(returns = ?webhooks, "Patch webhooks"); - Ok(HttpResponse::Ok().json(webhooks)) + + Ok(webhooks) } From 7c2c17129f303aeaebea792069cf9d06b4311192 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 10:59:06 +0200 Subject: [PATCH 16/53] Add get webhook route --- crates/meilisearch/src/routes/webhooks.rs | 65 ++++++++++++++++++++--- 1 file changed, 57 insertions(+), 8 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 710873d55..6157b8efa 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -1,6 +1,6 @@ use std::collections::BTreeMap; -use actix_web::web::{self, Data}; +use actix_web::web::{self, Data, Path}; use actix_web::{HttpRequest, HttpResponse}; use deserr::actix_web::AwebJson; use deserr::Deserr; @@ -24,7 +24,7 @@ use crate::extractors::sequential_extractor::SeqHandler; #[derive(OpenApi)] #[openapi( - paths(get_webhooks, patch_webhooks), + paths(get_webhooks, patch_webhooks, get_webhook), tags(( name = "Webhooks", description = "The `/webhooks` route allows you to register endpoints to be called once tasks are processed.", @@ -37,7 +37,11 @@ pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") .route(web::get().to(get_webhooks)) - .route(web::patch().to(SeqHandler(patch_webhooks))), + .route(web::patch().to(SeqHandler(patch_webhooks))) + ) + .service( + web::resource("/{name}") + .route(web::get().to(get_webhook)) ); } @@ -104,16 +108,29 @@ struct WebhooksSettings { webhooks: Setting>>, } -#[derive(Serialize)] -pub struct PatchWebhooksAnalytics; +#[derive(Serialize, Default)] +pub struct PatchWebhooksAnalytics { + patch_webhooks_count: usize, +} + +impl PatchWebhooksAnalytics { + pub fn patch_webhooks() -> Self { + PatchWebhooksAnalytics { + patch_webhooks_count: 1, + ..Self::default() + } + } +} impl Aggregate for PatchWebhooksAnalytics { fn event_name(&self) -> &'static str { "Webhooks Updated" } - fn aggregate(self: Box, _new: Box) -> Box { - self + fn aggregate(self: Box, new: Box) -> Box { + Box::new(PatchWebhooksAnalytics { + patch_webhooks_count: self.patch_webhooks_count + new.patch_webhooks_count, + }) } fn into_event(self: Box) -> serde_json::Value { @@ -131,6 +148,8 @@ enum WebhooksError { TooManyHeaders(String), #[error("Cannot edit webhook `{0}`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.")] ReservedWebhook(String), + #[error("Webhook `{0}` not found.")] + WebhookNotFound(String), } impl ErrorCode for WebhooksError { @@ -142,6 +161,7 @@ impl ErrorCode for WebhooksError { meilisearch_types::error::Code::InvalidWebhooksHeaders } WebhooksError::ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, + WebhooksError::WebhookNotFound(_) => meilisearch_types::error::Code::InvalidWebhooks, } } } @@ -182,7 +202,7 @@ async fn patch_webhooks( ) -> Result { let webhooks = patch_webhooks_inner(&index_scheduler, new_webhooks.0)?; - analytics.publish(PatchWebhooksAnalytics, &req); + analytics.publish(PatchWebhooksAnalytics::patch_webhooks(), &req); Ok(HttpResponse::Ok().json(webhooks)) } @@ -271,3 +291,32 @@ fn patch_webhooks_inner( Ok(webhooks) } + +#[utoipa::path( + get, + path = "/{name}", + tag = "Webhooks", + security(("Bearer" = ["webhooks.get", "*.get", "*"])), + responses( + (status = 200, description = "Webhook found", body = WebhookSettings, content_type = "application/json"), + (status = 404, description = "Webhook not found", body = ResponseError, content_type = "application/json"), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json"), + ), + params( + ("name" = String, Path, description = "The name of the webhook") + ) +)] +async fn get_webhook( + index_scheduler: GuardedData, Data>, + name: Path, +) -> Result { + let webhook_name = name.into_inner(); + let webhooks = index_scheduler.webhooks(); + + if let Some(webhook) = webhooks.webhooks.get(&webhook_name) { + debug!(returns = ?webhook, "Get webhook {}", webhook_name); + Ok(HttpResponse::Ok().json(webhook)) + } else { + Err(WebhooksError::WebhookNotFound(webhook_name).into()) + } +} From 53397e28fce1a844fb8021ddfe30cc06f07afe54 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 11:19:46 +0200 Subject: [PATCH 17/53] Replace name by uuid --- crates/meilisearch-types/src/webhooks.rs | 3 +- crates/meilisearch/src/lib.rs | 7 +- crates/meilisearch/src/routes/webhooks.rs | 183 +++++++++++++--------- crates/meilisearch/tests/tasks/webhook.rs | 5 +- 4 files changed, 117 insertions(+), 81 deletions(-) diff --git a/crates/meilisearch-types/src/webhooks.rs b/crates/meilisearch-types/src/webhooks.rs index 8849182ac..0f0741d69 100644 --- a/crates/meilisearch-types/src/webhooks.rs +++ b/crates/meilisearch-types/src/webhooks.rs @@ -1,6 +1,7 @@ use std::collections::BTreeMap; use serde::{Deserialize, Serialize}; +use uuid::Uuid; #[derive(Debug, Serialize, Deserialize, Clone, PartialEq)] #[serde(rename_all = "camelCase")] @@ -14,5 +15,5 @@ pub struct Webhook { #[serde(rename_all = "camelCase")] pub struct Webhooks { #[serde(default)] - pub webhooks: BTreeMap, + pub webhooks: BTreeMap, } diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index fcc71f04d..613268936 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -56,6 +56,7 @@ use option::ScheduleSnapshot; use search_queue::SearchQueue; use tracing::{error, info_span}; use tracing_subscriber::filter::Targets; +use uuid::Uuid; use crate::error::MeilisearchHttpError; @@ -339,13 +340,13 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< }, }); let mut webhooks = index_scheduler.webhooks(); - if webhooks.webhooks.get("_cli") != cli_webhook.as_ref() { + if webhooks.webhooks.get(&Uuid::nil()) != cli_webhook.as_ref() { match cli_webhook { Some(webhook) => { - webhooks.webhooks.insert("_cli".to_string(), webhook); + webhooks.webhooks.insert(Uuid::nil(), webhook); } None => { - webhooks.webhooks.remove("_cli"); + webhooks.webhooks.remove(&Uuid::nil()); } } index_scheduler.put_webhooks(webhooks)?; diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 6157b8efa..a78c36b0c 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -16,6 +16,7 @@ use meilisearch_types::webhooks::{Webhook, Webhooks}; use serde::Serialize; use tracing::debug; use utoipa::{OpenApi, ToSchema}; +use uuid::Uuid; use crate::analytics::{Aggregate, Analytics}; use crate::extractors::authentication::policies::ActionPolicy; @@ -37,49 +38,9 @@ pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") .route(web::get().to(get_webhooks)) - .route(web::patch().to(SeqHandler(patch_webhooks))) + .route(web::patch().to(SeqHandler(patch_webhooks))), ) - .service( - web::resource("/{name}") - .route(web::get().to(get_webhook)) - ); -} - -#[utoipa::path( - get, - path = "", - tag = "Webhooks", - security(("Bearer" = ["webhooks.get", "*.get", "*"])), - responses( - (status = OK, description = "Webhooks are returned", body = WebhooksSettings, content_type = "application/json", example = json!({ - "webhooks": { - "name": { - "url": "http://example.com/webhook", - }, - "anotherName": { - "url": "https://your.site/on-tasks-completed", - "headers": { - "Authorization": "Bearer a-secret-token" - } - } - } - })), - (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( - { - "message": "The Authorization header is missing. It must use the bearer authorization method.", - "code": "missing_authorization_header", - "type": "auth", - "link": "https://docs.meilisearch.com/errors#missing_authorization_header" - } - )), - ) -)] -async fn get_webhooks( - index_scheduler: GuardedData, Data>, -) -> Result { - let webhooks = index_scheduler.webhooks(); - debug!(returns = ?webhooks, "Get webhooks"); - Ok(HttpResponse::Ok().json(webhooks)) + .service(web::resource("/{uuid}").route(web::get().to(get_webhook))); } #[derive(Debug, Deserr, ToSchema)] @@ -105,7 +66,73 @@ struct WebhooksSettings { #[schema(value_type = Option>)] #[deserr(default, error = DeserrJsonError)] #[serde(default)] - webhooks: Setting>>, + webhooks: Setting>>, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct WebhookWithMetadata { + uuid: Uuid, + is_editable: bool, + #[serde(flatten)] + webhook: Webhook, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct WebhookResults { + results: Vec, +} + +#[utoipa::path( + get, + path = "", + tag = "Webhooks", + security(("Bearer" = ["webhooks.get", "*.get", "*"])), + responses( + (status = OK, description = "Webhooks are returned", body = WebhooksSettings, content_type = "application/json", example = json!({ + "results": [ + { + "uuid": "550e8400-e29b-41d4-a716-446655440000", + "url": "https://your.site/on-tasks-completed", + "headers": { + "Authorization": "Bearer a-secret-token" + }, + "isEditable": true + }, + { + "uuid": "550e8400-e29b-41d4-a716-446655440001", + "url": "https://another.site/on-tasks-completed", + "isEditable": true + } + ] + })), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!( + { + "message": "The Authorization header is missing. It must use the bearer authorization method.", + "code": "missing_authorization_header", + "type": "auth", + "link": "https://docs.meilisearch.com/errors#missing_authorization_header" + } + )), + ) +)] +async fn get_webhooks( + index_scheduler: GuardedData, Data>, +) -> Result { + let webhooks = index_scheduler.webhooks(); + let results = webhooks + .webhooks + .into_iter() + .map(|(uuid, webhook)| WebhookWithMetadata { + uuid, + is_editable: uuid != Uuid::nil(), + webhook, + }) + .collect::>(); + let results = WebhookResults { results }; + debug!(returns = ?results, "Get webhooks"); + Ok(HttpResponse::Ok().json(results)) } #[derive(Serialize, Default)] @@ -115,10 +142,7 @@ pub struct PatchWebhooksAnalytics { impl PatchWebhooksAnalytics { pub fn patch_webhooks() -> Self { - PatchWebhooksAnalytics { - patch_webhooks_count: 1, - ..Self::default() - } + PatchWebhooksAnalytics { patch_webhooks_count: 1 } } } @@ -141,15 +165,15 @@ impl Aggregate for PatchWebhooksAnalytics { #[derive(Debug, thiserror::Error)] enum WebhooksError { #[error("The URL for the webhook `{0}` is missing.")] - MissingUrl(String), + MissingUrl(Uuid), #[error("Defining too many webhooks would crush the server. Please limit the number of webhooks to 20. You may use a third-party proxy server to dispatch events to more than 20 endpoints.")] TooManyWebhooks, #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] - TooManyHeaders(String), + TooManyHeaders(Uuid), #[error("Cannot edit webhook `{0}`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.")] - ReservedWebhook(String), + ReservedWebhook(Uuid), #[error("Webhook `{0}` not found.")] - WebhookNotFound(String), + WebhookNotFound(Uuid), } impl ErrorCode for WebhooksError { @@ -175,10 +199,10 @@ impl ErrorCode for WebhooksError { responses( (status = 200, description = "Returns the updated webhooks", body = WebhooksSettings, content_type = "application/json", example = json!({ "webhooks": { - "name": { + "550e8400-e29b-41d4-a716-446655440000": { "url": "http://example.com/webhook", }, - "anotherName": { + "550e8400-e29b-41d4-a716-446655440001": { "url": "https://your.site/on-tasks-completed", "headers": { "Authorization": "Bearer a-secret-token" @@ -212,7 +236,7 @@ fn patch_webhooks_inner( new_webhooks: WebhooksSettings, ) -> Result { fn merge_webhook( - name: &str, + uuid: &Uuid, old_webhook: Option, new_webhook: WebhookSettings, ) -> Result { @@ -221,8 +245,8 @@ fn patch_webhooks_inner( let url = match new_webhook.url { Setting::Set(url) => url, - Setting::NotSet => old_url.ok_or_else(|| WebhooksError::MissingUrl(name.to_owned()))?, - Setting::Reset => return Err(WebhooksError::MissingUrl(name.to_owned())), + Setting::NotSet => old_url.ok_or_else(|| WebhooksError::MissingUrl(uuid.to_owned()))?, + Setting::Reset => return Err(WebhooksError::MissingUrl(uuid.to_owned())), }; let headers = match new_webhook.headers { @@ -246,7 +270,7 @@ fn patch_webhooks_inner( }; if headers.len() > 200 { - return Err(WebhooksError::TooManyHeaders(name.to_owned())); + return Err(WebhooksError::TooManyHeaders(uuid.to_owned())); } Ok(Webhook { url, headers }) @@ -258,19 +282,19 @@ fn patch_webhooks_inner( match new_webhooks.webhooks { Setting::Set(new_webhooks) => { - for (name, new_webhook) in new_webhooks { - if name.starts_with('_') { - return Err(WebhooksError::ReservedWebhook(name).into()); + for (uuid, new_webhook) in new_webhooks { + if uuid.is_nil() { + return Err(WebhooksError::ReservedWebhook(uuid).into()); } match new_webhook { Setting::Set(new_webhook) => { - let old_webhook = webhooks.remove(&name); - let webhook = merge_webhook(&name, old_webhook, new_webhook)?; - webhooks.insert(name.clone(), webhook); + let old_webhook = webhooks.remove(&uuid); + let webhook = merge_webhook(&uuid, old_webhook, new_webhook)?; + webhooks.insert(uuid, webhook); } Setting::Reset => { - webhooks.remove(&name); + webhooks.remove(&uuid); } Setting::NotSet => (), } @@ -298,25 +322,34 @@ fn patch_webhooks_inner( tag = "Webhooks", security(("Bearer" = ["webhooks.get", "*.get", "*"])), responses( - (status = 200, description = "Webhook found", body = WebhookSettings, content_type = "application/json"), + (status = 200, description = "Webhook found", body = WebhookSettings, content_type = "application/json", example = json!({ + "uuid": "550e8400-e29b-41d4-a716-446655440000", + "url": "https://your.site/on-tasks-completed", + "headers": { + "Authorization": "Bearer a-secret" + }, + "isEditable": true + })), (status = 404, description = "Webhook not found", body = ResponseError, content_type = "application/json"), (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json"), ), params( - ("name" = String, Path, description = "The name of the webhook") + ("uuid" = Uuid, Path, description = "The universally unique identifier of the webhook") ) )] async fn get_webhook( index_scheduler: GuardedData, Data>, - name: Path, + uuid: Path, ) -> Result { - let webhook_name = name.into_inner(); - let webhooks = index_scheduler.webhooks(); + let uuid = uuid.into_inner(); + let mut webhooks = index_scheduler.webhooks(); - if let Some(webhook) = webhooks.webhooks.get(&webhook_name) { - debug!(returns = ?webhook, "Get webhook {}", webhook_name); - Ok(HttpResponse::Ok().json(webhook)) - } else { - Err(WebhooksError::WebhookNotFound(webhook_name).into()) - } + let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?; + + debug!(returns = ?webhook, "Get webhook {}", uuid); + Ok(HttpResponse::Ok().json(WebhookWithMetadata { + uuid, + is_editable: uuid != Uuid::nil(), + webhook, + })) } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 12a8228fa..7fa088eb5 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -145,13 +145,14 @@ async fn cli_only() { server_handle.abort(); } - #[actix_web::test] async fn cli_with_dumps() { let db_path = tempfile::tempdir().unwrap(); let server = Server::new_with_options(Opt { task_webhook_url: Some(Url::parse("http://defined-in-test-cli.com").unwrap()), - task_webhook_authorization_header: Some(String::from("Bearer a-secret-token-defined-in-test-cli")), + task_webhook_authorization_header: Some(String::from( + "Bearer a-secret-token-defined-in-test-cli", + )), import_dump: Some(PathBuf::from("../dump/tests/assets/v6-with-webhooks.dump")), ..default_settings(db_path.path()) }) From ca27bcaac72bfcc08252e9e0fe2207ac442a2ba4 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 11:34:47 +0200 Subject: [PATCH 18/53] Update tests --- crates/dump/src/reader/mod.rs | 16 ++--- .../dump/tests/assets/v6-with-webhooks.dump | Bin 1913 -> 1389 bytes crates/meilisearch/tests/tasks/webhook.rs | 58 +++++++++++------- 3 files changed, 43 insertions(+), 31 deletions(-) diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index 85e5df432..844aadc99 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -449,7 +449,7 @@ pub(crate) mod test { let dump = DumpReader::open(dump).unwrap(); // top level infos - insta::assert_snapshot!(dump.date().unwrap(), @"2025-07-31 7:28:28.091553 +00:00:00"); + insta::assert_snapshot!(dump.date().unwrap(), @"2025-07-31 9:21:30.479544 +00:00:00"); insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @r" Some( cb887dcc-34b3-48d1-addd-9815ae721a81, @@ -462,23 +462,23 @@ pub(crate) mod test { insta::assert_json_snapshot!(webhooks, @r#" { "webhooks": { - "_cli": { + "00000000-0000-0000-0000-000000000000": { "url": "https://defined-in-dump.com/", "headers": { "Authorization": "Bearer defined in dump" } }, - "exampleName": { + "627ea538-733d-4545-8d2d-03526eb381ce": { + "url": "https://example.com/authorization-less", + "headers": {} + }, + "771b0a28-ef28-4082-b984-536f82958c65": { "url": "https://example.com/hook", "headers": { "authorization": "TOKEN" } }, - "otherName": { - "url": "https://example.com/authorization-less", - "headers": {} - }, - "third": { + "f3583083-f8a7-4cbf-a5e7-fb3f1e28a7e9": { "url": "https://third.com", "headers": {} } diff --git a/crates/dump/tests/assets/v6-with-webhooks.dump b/crates/dump/tests/assets/v6-with-webhooks.dump index 83989f25c4cc882dafeea5c60d917d594867f019..c8f9649d88429ab8c16f3375f94a6c4474d3e79f 100644 GIT binary patch literal 1389 zcmV-z1(Nz7iwFP!00000|Ls~)Z`(Ey_H%!QrpLl`NQx5WyroGC3MQPdwcIj3zwl?SBAd%IV4f z(EhQEd;1>%OZy-F)j|uSKbxv72D=8DlO{qu#QqWW$H4*c=;J@Wf2rX^o10OKUxVG_ zUi;&{HT&~eM!o$H0G*|&G+L8LWDyamVMG~Y$T(%O0?C<5*`UV{3-~;PI-f%6R6#5# z*VfQ~Ff6K%&FiXlaMc1PcW0Gu z{VZvoUA$;Z?eeO;y00soJ(x&d94!^R&HAs;{|CW5T+IL^eZrMR z$XC8r@N#X9lxsL~Zq^(h9T~8B35Mh)@kQbY%ZVfO&IyLyj>esbC#{=Qb^flxf%;0(@3J-gI57FFPXz zG|i2{_)er`8C5(bm{L{?9=zDU$}R?xLf~x vqbZI3pnW!TcS@#0NH2+{SKdtS5>2@n%bq;x(W6I?9}@onHxdkx05|{u+j_X9 literal 1913 zcmV-<2Zs0`iwFP!00000|Lt4bj@vd9_Vxc1`gK`YS{`-qMS(Qi0!4weXt%v+QKYm; z$7*EBl_+esi+!xVK%cBbO19VDb>vNKH);4`v9hQkDa~kR$eB@iF*-RAVkqVujtIq3 zcSMAB#||UR5hD@ijN0~?Aw1%vGoWp>s6B7=zF0lT>iCf$FQs~;n$aoobopZx!F4D9 z(_<=hHva(S&*R9+{|xBmfAL4FS{44@RAn(bHb9&xVr-xM8H3+D@~4R7(OH=IiOc`i zZ*%ES(Abwsi@a$3$*nKj#qvkf>=Q&diYW>(L!8qn=KkH-2bO6|s4Fds7t<@zyoPh! zU1{y5rp+>?q=Hc}bkR3UwI2IiP^>rjB=fTAnR zCWucs4I|JbPJV{LW_g)6^S6y%)ezkQv-3JTv%*8~fnib*|@LECu z6_Z^*QQmkJ2&lGWh5p_DdP9T{FhmiL2~9t6L&QRko??h-$ll8k5t1BkQSQCeaG!i} zs2Kw4lq}xG2+?S9xDARzGEBZPKqMw1i{8TkAslk{Bm+bvuz7rMm&a@n3vsm122mEq z$ssm~aRkZhyV^$<0@Jp;f(ZE|tEn>euV@S4x}p`Ff=7 zNz9U|YI3cF!K|%K? zRw2dF@$e4$Uk0%zm;XHjO!WCgY2P&l>_4IWf&4LvsI&iP0N8)kWOcsm(-uEA5LwqI zn%qEG7Xq%cx?0#UHvA~HDkZprQY?%cxKuL%TQKj3j`G=eUwhxE8@O&WIlfF&G8Ksm zlt@ItaGC~bmhnK&NF)gtl#2w2Mb_`?e(}30{a^c3jIxcNnR5tJG^2_Knc^g1oT@;i8RWxeOv)Ic zX-a3?-2CiljO3IY$VW@LmyPgN!DJFrH^9(^WtS(iy6O`p*Ym1?bP23Rr=n4N83|Ho zOxEl`c0wXS2H%=o!Q+fgW>ZQcf`cg)3K*M3K_UnWIO8lqsQ_i(Wo!tc3}~{p)BHRV zpCQvr=wXcRN#Dk4n=2Hhe1alBH(hr5!{Ri>(*(n)&M~PN%*Y^%t+u|YWZ*! z+ib?z>G08q$!zh!=(Kp3`d^BP+y9*bR{j^ABYio`i&K<>zk~cq%w7HWInc@fl-1uu z{vY=G5A(l-N5uL6GhkEyZHuC8v1PXoUsXe{&Q}$c_X;YzYRZSjl(&9Y)cA{wF5HhU zj9ql$7}15Xi!Pigx-fRpg@d9CcIHhTr72oHpsddBT<#USU)nn1&X)GWJofp`-FEry zK=uFeKSa1+s_%S;^BK-(IG^Es#)s)MR$odD_>7&;G*0lz$l7TT=QNzta8AQH4d*mI zAg5t&dB2mmU*(rDMDSiGu~iR<5{mabi7is_Jk(`r9UL4S&WV2jsh^k*08Rh^55LRg diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 7fa088eb5..0990561e9 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -13,6 +13,7 @@ use meili_snap::{json_string, snapshot}; use meilisearch::Opt; use tokio::sync::mpsc; use url::Url; +use uuid::Uuid; use crate::common::{self, default_settings, Server}; use crate::json; @@ -129,16 +130,18 @@ async fn cli_only() { let (webhooks, code) = server.get_webhooks().await; snapshot!(code, @"200 OK"); - snapshot!(json_string!(webhooks, { ".webhooks._cli.url" => "[ignored]" }), @r#" + snapshot!(json_string!(webhooks, { ".results[].url" => "[ignored]" }), @r#" { - "webhooks": { - "_cli": { + "results": [ + { + "uuid": "00000000-0000-0000-0000-000000000000", + "isEditable": false, "url": "[ignored]", "headers": { "Authorization": "Bearer a-secret-token" } } - } + ] } "#); @@ -163,28 +166,36 @@ async fn cli_with_dumps() { snapshot!(code, @"200 OK"); snapshot!(webhooks, @r#" { - "webhooks": { - "_cli": { + "results": [ + { + "uuid": "00000000-0000-0000-0000-000000000000", + "isEditable": false, "url": "http://defined-in-test-cli.com/", "headers": { "Authorization": "Bearer a-secret-token-defined-in-test-cli" } }, - "exampleName": { + { + "uuid": "627ea538-733d-4545-8d2d-03526eb381ce", + "isEditable": true, + "url": "https://example.com/authorization-less", + "headers": {} + }, + { + "uuid": "771b0a28-ef28-4082-b984-536f82958c65", + "isEditable": true, "url": "https://example.com/hook", "headers": { "authorization": "TOKEN" } }, - "otherName": { - "url": "https://example.com/authorization-less", - "headers": {} - }, - "third": { + { + "uuid": "f3583083-f8a7-4cbf-a5e7-fb3f1e28a7e9", + "isEditable": true, "url": "https://third.com", "headers": {} } - } + ] } "#); } @@ -194,23 +205,23 @@ async fn reserved_names() { let server = Server::new().await; let (value, code) = server - .set_webhooks(json!({ "webhooks": { "_cli": { "url": "http://localhost:8080" } } })) + .set_webhooks(json!({ "webhooks": { Uuid::nil(): { "url": "http://localhost:8080" } } })) .await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Cannot edit webhook `_cli`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", + "message": "Cannot edit webhook `[uuid]`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", "code": "reserved_webhook", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#reserved_webhook" } "#); - let (value, code) = server.set_webhooks(json!({ "webhooks": { "_cli": null } })).await; + let (value, code) = server.set_webhooks(json!({ "webhooks": { Uuid::nil(): null } })).await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Cannot edit webhook `_cli`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", + "message": "Cannot edit webhook `[uuid]`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", "code": "reserved_webhook", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#reserved_webhook" @@ -223,14 +234,14 @@ async fn over_limits() { let server = Server::new().await; // Too many webhooks - for i in 0..20 { + for _ in 0..20 { let (_value, code) = server - .set_webhooks(json!({ "webhooks": { format!("webhook_{i}"): { "url": "http://localhost:8080" } } })) + .set_webhooks(json!({ "webhooks": { Uuid::new_v4(): { "url": "http://localhost:8080" } } })) .await; snapshot!(code, @"200 OK"); } let (value, code) = server - .set_webhooks(json!({ "webhooks": { "webhook_21": { "url": "http://localhost:8080" } } })) + .set_webhooks(json!({ "webhooks": { Uuid::new_v4(): { "url": "http://localhost:8080" } } })) .await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" @@ -252,20 +263,21 @@ async fn over_limits() { "#); // Test too many headers + let uuid = Uuid::new_v4(); for i in 0..200 { let header_name = format!("header_{i}"); let (_value, code) = server - .set_webhooks(json!({ "webhooks": { "webhook": { "url": "http://localhost:8080", "headers": { header_name: "value" } } } })) + .set_webhooks(json!({ "webhooks": { uuid: { "url": "http://localhost:8080", "headers": { header_name: "value" } } } })) .await; snapshot!(code, @"200 OK"); } let (value, code) = server - .set_webhooks(json!({ "webhooks": { "webhook": { "url": "http://localhost:8080", "headers": { "header_201": "value" } } } })) + .set_webhooks(json!({ "webhooks": { uuid: { "url": "http://localhost:8080", "headers": { "header_201": "value" } } } })) .await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Too many headers for the webhook `webhook`. Please limit the number of headers to 200.", + "message": "Too many headers for the webhook `[uuid]`. Please limit the number of headers to 200.", "code": "invalid_webhooks_headers", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" From 29fb4d5e2a155be419c3b439d024ad2e87d4d0db Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 12:27:12 +0200 Subject: [PATCH 19/53] Add post webhook route --- crates/meilisearch-types/src/error.rs | 3 +- crates/meilisearch/src/routes/webhooks.rs | 69 +++++++++++++++++++---- 2 files changed, 60 insertions(+), 12 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 56590e79d..3916012c1 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -423,7 +423,8 @@ InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQU InvalidWebhooks , InvalidRequest , BAD_REQUEST ; InvalidWebhooksUrl , InvalidRequest , BAD_REQUEST ; InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST ; -ReservedWebhook , InvalidRequest , BAD_REQUEST +ReservedWebhook , InvalidRequest , BAD_REQUEST ; +WebhookNotFound , InvalidRequest , NOT_FOUND } impl ErrorCode for JoinError { diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index a78c36b0c..7be6d0386 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -6,9 +6,7 @@ use deserr::actix_web::AwebJson; use deserr::Deserr; use index_scheduler::IndexScheduler; use meilisearch_types::deserr::DeserrJsonError; -use meilisearch_types::error::deserr_codes::{ - InvalidWebhooks, InvalidWebhooksHeaders, InvalidWebhooksUrl, -}; +use meilisearch_types::error::deserr_codes::{InvalidWebhooksHeaders, InvalidWebhooksUrl}; use meilisearch_types::error::{ErrorCode, ResponseError}; use meilisearch_types::keys::actions; use meilisearch_types::milli::update::Setting; @@ -25,7 +23,7 @@ use crate::extractors::sequential_extractor::SeqHandler; #[derive(OpenApi)] #[openapi( - paths(get_webhooks, patch_webhooks, get_webhook), + paths(get_webhooks, patch_webhooks, get_webhook, post_webhook), tags(( name = "Webhooks", description = "The `/webhooks` route allows you to register endpoints to be called once tasks are processed.", @@ -38,13 +36,14 @@ pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") .route(web::get().to(get_webhooks)) - .route(web::patch().to(SeqHandler(patch_webhooks))), + .route(web::patch().to(SeqHandler(patch_webhooks))) + .route(web::post().to(SeqHandler(post_webhook))), ) .service(web::resource("/{uuid}").route(web::get().to(get_webhook))); } #[derive(Debug, Deserr, ToSchema)] -#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] struct WebhookSettings { @@ -64,16 +63,17 @@ struct WebhookSettings { #[schema(rename_all = "camelCase")] struct WebhooksSettings { #[schema(value_type = Option>)] - #[deserr(default, error = DeserrJsonError)] #[serde(default)] webhooks: Setting>>, } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] struct WebhookWithMetadata { uuid: Uuid, is_editable: bool, + #[schema(value_type = WebhookSettings)] #[serde(flatten)] webhook: Webhook, } @@ -138,11 +138,16 @@ async fn get_webhooks( #[derive(Serialize, Default)] pub struct PatchWebhooksAnalytics { patch_webhooks_count: usize, + post_webhook_count: usize, } impl PatchWebhooksAnalytics { pub fn patch_webhooks() -> Self { - PatchWebhooksAnalytics { patch_webhooks_count: 1 } + PatchWebhooksAnalytics { patch_webhooks_count: 1, ..Default::default() } + } + + pub fn post_webhook() -> Self { + PatchWebhooksAnalytics { post_webhook_count: 1, ..Default::default() } } } @@ -154,6 +159,7 @@ impl Aggregate for PatchWebhooksAnalytics { fn aggregate(self: Box, new: Box) -> Box { Box::new(PatchWebhooksAnalytics { patch_webhooks_count: self.patch_webhooks_count + new.patch_webhooks_count, + post_webhook_count: self.post_webhook_count + new.post_webhook_count, }) } @@ -185,7 +191,7 @@ impl ErrorCode for WebhooksError { meilisearch_types::error::Code::InvalidWebhooksHeaders } WebhooksError::ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, - WebhooksError::WebhookNotFound(_) => meilisearch_types::error::Code::InvalidWebhooks, + WebhooksError::WebhookNotFound(_) => meilisearch_types::error::Code::WebhookNotFound, } } } @@ -318,7 +324,7 @@ fn patch_webhooks_inner( #[utoipa::path( get, - path = "/{name}", + path = "/{uuid}", tag = "Webhooks", security(("Bearer" = ["webhooks.get", "*.get", "*"])), responses( @@ -353,3 +359,44 @@ async fn get_webhook( webhook, })) } + +#[utoipa::path( + post, + path = "", + tag = "Webhooks", + request_body = WebhookSettings, + security(("Bearer" = ["webhooks.update", "*"])), + responses( + (status = 201, description = "Webhook created successfully", body = WebhookWithMetadata, content_type = "application/json", example = json!({ + "uuid": "550e8400-e29b-41d4-a716-446655440000", + "url": "https://your.site/on-tasks-completed", + "headers": { + "Authorization": "Bearer a-secret-token" + }, + "isEditable": true + })), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json"), + (status = 400, description = "Bad request", body = ResponseError, content_type = "application/json"), + ) +)] +async fn post_webhook( + index_scheduler: GuardedData, Data>, + webhook_settings: AwebJson, + req: HttpRequest, + analytics: Data, +) -> Result { + let uuid = Uuid::new_v4(); + + let webhooks = patch_webhooks_inner( + &index_scheduler, + WebhooksSettings { + webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Set(webhook_settings.0))])), + }, + )?; + let webhook = webhooks.webhooks.get(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?.clone(); + + analytics.publish(PatchWebhooksAnalytics::post_webhook(), &req); + + debug!(returns = ?webhook, "Created webhook {}", uuid); + Ok(HttpResponse::Created().json(WebhookWithMetadata { uuid, is_editable: true, webhook })) +} From ad68245186e80eb5e23da4853f5a2f7c4078ac93 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 12:33:34 +0200 Subject: [PATCH 20/53] Update tests --- crates/meilisearch/tests/common/server.rs | 9 +++++ crates/meilisearch/tests/tasks/webhook.rs | 41 ++++++++++++++++++++++- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 1dfe2e593..113dbc86f 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -186,6 +186,15 @@ impl Server { self.service.patch("/webhooks", value).await } + pub async fn create_webhook(&self, value: Value) -> (Value, StatusCode) { + self.service.post("/webhooks", value).await + } + + pub async fn get_webhook(&self, uuid: impl AsRef) -> (Value, StatusCode) { + let url = format!("/webhooks/{}", uuid.as_ref()); + self.service.get(url).await + } + pub async fn get_metrics(&self) -> (Value, StatusCode) { self.service.get("/metrics").await } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 0990561e9..8c2a59874 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -236,7 +236,9 @@ async fn over_limits() { // Too many webhooks for _ in 0..20 { let (_value, code) = server - .set_webhooks(json!({ "webhooks": { Uuid::new_v4(): { "url": "http://localhost:8080" } } })) + .set_webhooks( + json!({ "webhooks": { Uuid::new_v4(): { "url": "http://localhost:8080" } } }), + ) .await; snapshot!(code, @"200 OK"); } @@ -284,3 +286,40 @@ async fn over_limits() { } "#); } + +#[actix_web::test] +async fn post_and_get() { + let server = Server::new().await; + + let (value, code) = server + .create_webhook(json!({ + "url": "https://example.com/hook", + "headers": { "authorization": "TOKEN" } + })) + .await; + snapshot!(code, @"201 Created"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "https://example.com/hook", + "headers": { + "authorization": "TOKEN" + } + } + "#); + + let uuid = value.get("uuid").unwrap().as_str().unwrap(); + let (value, code) = server.get_webhook(uuid).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "https://example.com/hook", + "headers": { + "authorization": "TOKEN" + } + } + "#); +} From 94733a4a183e34c7fb461f889105f3d5ca764f4a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 12:38:14 +0200 Subject: [PATCH 21/53] Add delete endpoint --- crates/meilisearch/src/routes/webhooks.rs | 46 ++++++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 7be6d0386..7dd5b00d1 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -23,7 +23,7 @@ use crate::extractors::sequential_extractor::SeqHandler; #[derive(OpenApi)] #[openapi( - paths(get_webhooks, patch_webhooks, get_webhook, post_webhook), + paths(get_webhooks, patch_webhooks, get_webhook, post_webhook, delete_webhook), tags(( name = "Webhooks", description = "The `/webhooks` route allows you to register endpoints to be called once tasks are processed.", @@ -39,7 +39,11 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .route(web::patch().to(SeqHandler(patch_webhooks))) .route(web::post().to(SeqHandler(post_webhook))), ) - .service(web::resource("/{uuid}").route(web::get().to(get_webhook))); + .service( + web::resource("/{uuid}") + .route(web::get().to(get_webhook)) + .route(web::delete().to(SeqHandler(delete_webhook))), + ); } #[derive(Debug, Deserr, ToSchema)] @@ -400,3 +404,41 @@ async fn post_webhook( debug!(returns = ?webhook, "Created webhook {}", uuid); Ok(HttpResponse::Created().json(WebhookWithMetadata { uuid, is_editable: true, webhook })) } + +#[utoipa::path( + delete, + path = "/{uuid}", + tag = "Webhooks", + security(("Bearer" = ["webhooks.update", "*"])), + responses( + (status = 204, description = "Webhook deleted successfully"), + (status = 404, description = "Webhook not found", body = ResponseError, content_type = "application/json"), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json"), + ), + params( + ("uuid" = Uuid, Path, description = "The universally unique identifier of the webhook") + ) +)] +async fn delete_webhook( + index_scheduler: GuardedData, Data>, + uuid: Path, + req: HttpRequest, + analytics: Data, +) -> Result { + let uuid = uuid.into_inner(); + + let webhooks = index_scheduler.webhooks(); + if !webhooks.webhooks.contains_key(&uuid) { + return Err(WebhooksError::WebhookNotFound(uuid).into()); + } + + patch_webhooks_inner( + &index_scheduler, + WebhooksSettings { webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Reset)])) }, + )?; + + analytics.publish(PatchWebhooksAnalytics::patch_webhooks(), &req); + + debug!("Deleted webhook {}", uuid); + Ok(HttpResponse::NoContent().finish()) +} From 9e43f7b419b2f7955be045d01a9b4061f2164042 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 12:44:35 +0200 Subject: [PATCH 22/53] Update tests --- crates/meilisearch/src/routes/webhooks.rs | 11 +++++++---- crates/meilisearch/tests/common/server.rs | 5 +++++ crates/meilisearch/tests/tasks/webhook.rs | 8 +++++++- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 7dd5b00d1..15b3145d0 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -143,6 +143,7 @@ async fn get_webhooks( pub struct PatchWebhooksAnalytics { patch_webhooks_count: usize, post_webhook_count: usize, + delete_webhook_count: usize, } impl PatchWebhooksAnalytics { @@ -153,6 +154,10 @@ impl PatchWebhooksAnalytics { pub fn post_webhook() -> Self { PatchWebhooksAnalytics { post_webhook_count: 1, ..Default::default() } } + + pub fn delete_webhook() -> Self { + PatchWebhooksAnalytics { delete_webhook_count: 1, ..Default::default() } + } } impl Aggregate for PatchWebhooksAnalytics { @@ -164,6 +169,7 @@ impl Aggregate for PatchWebhooksAnalytics { Box::new(PatchWebhooksAnalytics { patch_webhooks_count: self.patch_webhooks_count + new.patch_webhooks_count, post_webhook_count: self.post_webhook_count + new.post_webhook_count, + delete_webhook_count: self.delete_webhook_count + new.delete_webhook_count, }) } @@ -356,7 +362,6 @@ async fn get_webhook( let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?; - debug!(returns = ?webhook, "Get webhook {}", uuid); Ok(HttpResponse::Ok().json(WebhookWithMetadata { uuid, is_editable: uuid != Uuid::nil(), @@ -401,7 +406,6 @@ async fn post_webhook( analytics.publish(PatchWebhooksAnalytics::post_webhook(), &req); - debug!(returns = ?webhook, "Created webhook {}", uuid); Ok(HttpResponse::Created().json(WebhookWithMetadata { uuid, is_editable: true, webhook })) } @@ -437,8 +441,7 @@ async fn delete_webhook( WebhooksSettings { webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Reset)])) }, )?; - analytics.publish(PatchWebhooksAnalytics::patch_webhooks(), &req); + analytics.publish(PatchWebhooksAnalytics::delete_webhook(), &req); - debug!("Deleted webhook {}", uuid); Ok(HttpResponse::NoContent().finish()) } diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 113dbc86f..dd690c3db 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -195,6 +195,11 @@ impl Server { self.service.get(url).await } + pub async fn delete_webhook(&self, uuid: impl AsRef) -> (Value, StatusCode) { + let url = format!("/webhooks/{}", uuid.as_ref()); + self.service.delete(url).await + } + pub async fn get_metrics(&self) -> (Value, StatusCode) { self.service.get("/metrics").await } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 8c2a59874..9d66800af 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -288,7 +288,7 @@ async fn over_limits() { } #[actix_web::test] -async fn post_and_get() { +async fn post_get_delete() { let server = Server::new().await; let (value, code) = server @@ -322,4 +322,10 @@ async fn post_and_get() { } } "#); + + let (_value, code) = server.delete_webhook(uuid).await; + snapshot!(code, @"204 No Content"); + + let (_value, code) = server.get_webhook(uuid).await; + snapshot!(code, @"404 Not Found"); } From 34590297c1fa31ef8fe1c5fb5628a02d110078e3 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 12:53:57 +0200 Subject: [PATCH 23/53] Add patch webhook endpoint --- crates/meilisearch/src/routes/webhooks.rs | 53 ++++++++++++++++++++++- 1 file changed, 52 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 15b3145d0..4b925c325 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -23,7 +23,7 @@ use crate::extractors::sequential_extractor::SeqHandler; #[derive(OpenApi)] #[openapi( - paths(get_webhooks, patch_webhooks, get_webhook, post_webhook, delete_webhook), + paths(get_webhooks, patch_webhooks, get_webhook, post_webhook, patch_webhook, delete_webhook), tags(( name = "Webhooks", description = "The `/webhooks` route allows you to register endpoints to be called once tasks are processed.", @@ -42,6 +42,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { .service( web::resource("/{uuid}") .route(web::get().to(get_webhook)) + .route(web::patch().to(SeqHandler(patch_webhook))) .route(web::delete().to(SeqHandler(delete_webhook))), ); } @@ -142,6 +143,7 @@ async fn get_webhooks( #[derive(Serialize, Default)] pub struct PatchWebhooksAnalytics { patch_webhooks_count: usize, + patch_webhook_count: usize, post_webhook_count: usize, delete_webhook_count: usize, } @@ -151,6 +153,10 @@ impl PatchWebhooksAnalytics { PatchWebhooksAnalytics { patch_webhooks_count: 1, ..Default::default() } } + pub fn patch_webhook() -> Self { + PatchWebhooksAnalytics { patch_webhook_count: 1, ..Default::default() } + } + pub fn post_webhook() -> Self { PatchWebhooksAnalytics { post_webhook_count: 1, ..Default::default() } } @@ -168,6 +174,7 @@ impl Aggregate for PatchWebhooksAnalytics { fn aggregate(self: Box, new: Box) -> Box { Box::new(PatchWebhooksAnalytics { patch_webhooks_count: self.patch_webhooks_count + new.patch_webhooks_count, + patch_webhook_count: self.patch_webhook_count + new.patch_webhook_count, post_webhook_count: self.post_webhook_count + new.post_webhook_count, delete_webhook_count: self.delete_webhook_count + new.delete_webhook_count, }) @@ -409,6 +416,50 @@ async fn post_webhook( Ok(HttpResponse::Created().json(WebhookWithMetadata { uuid, is_editable: true, webhook })) } +#[utoipa::path( + patch, + path = "/{uuid}", + tag = "Webhooks", + request_body = WebhookSettings, + security(("Bearer" = ["webhooks.update", "*"])), + responses( + (status = 200, description = "Webhook updated successfully", body = WebhookWithMetadata, content_type = "application/json", example = json!({ + "uuid": "550e8400-e29b-41d4-a716-446655440000", + "url": "https://your.site/on-tasks-completed", + "headers": { + "Authorization": "Bearer a-secret-token" + }, + "isEditable": true + })), + (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json"), + (status = 400, description = "Bad request", body = ResponseError, content_type = "application/json"), + ), + params( + ("uuid" = Uuid, Path, description = "The universally unique identifier of the webhook") + ) +)] +async fn patch_webhook( + index_scheduler: GuardedData, Data>, + uuid: Path, + webhook_settings: AwebJson, + req: HttpRequest, + analytics: Data, +) -> Result { + let uuid = uuid.into_inner(); + + let webhooks = patch_webhooks_inner( + &index_scheduler, + WebhooksSettings { + webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Set(webhook_settings.0))])), + }, + )?; + let webhook = webhooks.webhooks.get(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?.clone(); + + analytics.publish(PatchWebhooksAnalytics::patch_webhook(), &req); + + Ok(HttpResponse::Ok().json(WebhookWithMetadata { uuid, is_editable: uuid != Uuid::nil(), webhook })) +} + #[utoipa::path( delete, path = "/{uuid}", From ee80fc87c95a4c94190b3bb74066aa3c05fa80af Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 13:00:43 +0200 Subject: [PATCH 24/53] Add test for patch endpoint --- crates/meilisearch/src/routes/webhooks.rs | 6 +- crates/meilisearch/tests/common/server.rs | 5 ++ crates/meilisearch/tests/tasks/webhook.rs | 84 +++++++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 4b925c325..776667829 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -457,7 +457,11 @@ async fn patch_webhook( analytics.publish(PatchWebhooksAnalytics::patch_webhook(), &req); - Ok(HttpResponse::Ok().json(WebhookWithMetadata { uuid, is_editable: uuid != Uuid::nil(), webhook })) + Ok(HttpResponse::Ok().json(WebhookWithMetadata { + uuid, + is_editable: uuid != Uuid::nil(), + webhook, + })) } #[utoipa::path( diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index dd690c3db..0b57ca37a 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -200,6 +200,11 @@ impl Server { self.service.delete(url).await } + pub async fn patch_webhook(&self, uuid: impl AsRef, value: Value) -> (Value, StatusCode) { + let url = format!("/webhooks/{}", uuid.as_ref()); + self.service.patch(url, value).await + } + pub async fn get_metrics(&self) -> (Value, StatusCode) { self.service.get("/metrics").await } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 9d66800af..3d27d6be6 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -329,3 +329,87 @@ async fn post_get_delete() { let (_value, code) = server.get_webhook(uuid).await; snapshot!(code, @"404 Not Found"); } + +#[actix_web::test] +async fn patch() { + let server = Server::new().await; + + let uuid = Uuid::new_v4().to_string(); + let (value, code) = + server.patch_webhook(&uuid, json!({ "headers": { "authorization": "TOKEN" } })).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "The URL for the webhook `[uuid]` is missing.", + "code": "invalid_webhooks_url", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + } + "#); + + let (value, code) = + server.patch_webhook(&uuid, json!({ "url": "https://example.com/hook" })).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "https://example.com/hook", + "headers": {} + } + "#); + + let (value, code) = + server.patch_webhook(&uuid, json!({ "headers": { "authorization": "TOKEN" } })).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "https://example.com/hook", + "headers": { + "authorization": "TOKEN" + } + } + "#); + + let (value, code) = + server.patch_webhook(&uuid, json!({ "headers": { "authorization2": "TOKEN" } })).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "https://example.com/hook", + "headers": { + "authorization": "TOKEN", + "authorization2": "TOKEN" + } + } + "#); + + let (value, code) = + server.patch_webhook(&uuid, json!({ "headers": { "authorization": null } })).await; + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "https://example.com/hook", + "headers": { + "authorization2": "TOKEN" + } + } + "#); + + let (value, code) = server.patch_webhook(&uuid, json!({ "url": null })).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "The URL for the webhook `[uuid]` is missing.", + "code": "invalid_webhooks_url", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + } + "#); +} From 35537e0b0b2065243650af43727bfc5ac016c377 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 14:12:09 +0200 Subject: [PATCH 25/53] Add single_receives_data test --- crates/meilisearch/tests/tasks/webhook.rs | 56 +++++++++++++++-------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 3d27d6be6..14946e415 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -71,17 +71,50 @@ async fn create_webhook_server() -> WebhookHandle { #[actix_web::test] async fn cli_only() { - let WebhookHandle { server_handle, url, mut receiver } = create_webhook_server().await; - let db_path = tempfile::tempdir().unwrap(); let server = Server::new_with_options(Opt { - task_webhook_url: Some(Url::parse(&url).unwrap()), + task_webhook_url: Some(Url::parse("https://example-cli.com/").unwrap()), task_webhook_authorization_header: Some(String::from("Bearer a-secret-token")), ..default_settings(db_path.path()) }) .await .unwrap(); + let (webhooks, code) = server.get_webhooks().await; + snapshot!(code, @"200 OK"); + snapshot!(webhooks, @r#" + { + "results": [ + { + "uuid": "00000000-0000-0000-0000-000000000000", + "isEditable": false, + "url": "https://example-cli.com/", + "headers": { + "Authorization": "Bearer a-secret-token" + } + } + ] + } + "#); +} + +#[actix_web::test] +async fn single_receives_data() { + let WebhookHandle { server_handle, url, mut receiver } = create_webhook_server().await; + + let server = Server::new().await; + + let (value, code) = server.create_webhook(json!({ "url": url })).await; + snapshot!(code, @"201 Created"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]", ".url" => "[ignored]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "[ignored]", + "headers": {} + } + "#); + let index = server.index("tamo"); // May be flaky: we're relying on the fact that while the first document addition is processed, the other // operations will be received and will be batched together. If it doesn't happen it's not a problem @@ -128,23 +161,6 @@ async fn cli_only() { assert!(nb_tasks == 5, "We should have received the 5 tasks but only received {nb_tasks}"); - let (webhooks, code) = server.get_webhooks().await; - snapshot!(code, @"200 OK"); - snapshot!(json_string!(webhooks, { ".results[].url" => "[ignored]" }), @r#" - { - "results": [ - { - "uuid": "00000000-0000-0000-0000-000000000000", - "isEditable": false, - "url": "[ignored]", - "headers": { - "Authorization": "Bearer a-secret-token" - } - } - ] - } - "#); - server_handle.abort(); } From ed147f80ac782a0fade13f009b4365859b037ec0 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Thu, 31 Jul 2025 16:45:30 +0200 Subject: [PATCH 26/53] Add test and fix bug --- crates/index-scheduler/src/lib.rs | 41 +++++++++++++------ crates/meilisearch/tests/tasks/webhook.rs | 49 ++++++++++++++++++++++- 2 files changed, 77 insertions(+), 13 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index ce8791a63..8d7617b6c 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -828,16 +828,32 @@ impl IndexScheduler { written: 0, }; - enum EitherRead { - Other(T), - Data(Vec), + enum EitherRead<'a, T: Read> { + Other(Option), + Data(&'a [u8]), } - impl Read for &mut EitherRead { + impl EitherRead<'_, T> { + /// A clone that works only once for the Other variant. + fn clone(&mut self) -> Self { + match self { + Self::Other(r) => { + let r = r.take(); + Self::Other(r) + } + Self::Data(arg0) => Self::Data(arg0), + } + } + } + + impl Read for EitherRead<'_, T> { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { match self { - EitherRead::Other(reader) => reader.read(buf), - EitherRead::Data(data) => data.as_slice().read(buf), + EitherRead::Other(Some(reader)) => reader.read(buf), + EitherRead::Other(None) => { + Err(io::Error::new(io::ErrorKind::Other, "No reader available")) + } + EitherRead::Data(data) => data.read(buf), } } } @@ -845,16 +861,17 @@ impl IndexScheduler { let mut reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); // When there is more than one webhook, cache the data in memory + let mut data; let mut reader = match webhooks.webhooks.len() { - 1 => EitherRead::Other(reader), + 1 => EitherRead::Other(Some(reader)), _ => { - let mut data = Vec::new(); + data = Vec::new(); reader.read_to_end(&mut data)?; - EitherRead::Data(data) + EitherRead::Data(&data) } }; - for (name, Webhook { url, headers }) in webhooks.webhooks.iter() { + for (uuid, Webhook { url, headers }) in webhooks.webhooks.iter() { let mut request = ureq::post(url) .timeout(Duration::from_secs(30)) .set("Content-Encoding", "gzip") @@ -863,8 +880,8 @@ impl IndexScheduler { request = request.set(header_name, header_value); } - if let Err(e) = request.send(&mut reader) { - tracing::error!("While sending data to the webhook {name}: {e}"); + if let Err(e) = request.send(reader.clone()) { + tracing::error!("While sending data to the webhook {uuid}: {e}"); } } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 14946e415..4caa7df92 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -115,10 +115,10 @@ async fn single_receives_data() { } "#); - let index = server.index("tamo"); // May be flaky: we're relying on the fact that while the first document addition is processed, the other // operations will be received and will be batched together. If it doesn't happen it's not a problem // the rest of the test won't assume anything about the number of tasks per batch. + let index = server.index("tamo"); for i in 0..5 { let (_, _status) = index.add_documents(json!({ "id": i, "doggo": "bone" }), None).await; } @@ -164,6 +164,53 @@ async fn single_receives_data() { server_handle.abort(); } +#[actix_web::test] +async fn multiple_receive_data() { + let server = Server::new().await; + + let WebhookHandle { server_handle: handle1, url: url1, receiver: mut receiver1 } = + create_webhook_server().await; + let WebhookHandle { server_handle: handle2, url: url2, receiver: mut receiver2 } = + create_webhook_server().await; + let WebhookHandle { server_handle: handle3, url: url3, receiver: mut receiver3 } = + create_webhook_server().await; + + for url in [url1, url2, url3] { + let (value, code) = server.create_webhook(json!({ "url": url })).await; + snapshot!(code, @"201 Created"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]", ".url" => "[ignored]" }), @r#" + { + "uuid": "[uuid]", + "isEditable": true, + "url": "[ignored]", + "headers": {} + } + "#); + } + let index = server.index("tamo"); + let (_, status) = index.add_documents(json!({ "id": 1, "doggo": "bone" }), None).await; + snapshot!(status, @"202 Accepted"); + + let mut count1 = 0; + let mut count2 = 0; + let mut count3 = 0; + while count1 == 0 || count2 == 0 || count3 == 0 { + tokio::select! { + msg = receiver1.recv() => { if msg.is_some() { count1 += 1; } }, + msg = receiver2.recv() => { if msg.is_some() { count2 += 1; } }, + msg = receiver3.recv() => { if msg.is_some() { count3 += 1; } }, + } + } + + assert_eq!(count1, 1); + assert_eq!(count2, 1); + assert_eq!(count3, 1); + + handle1.abort(); + handle2.abort(); + handle3.abort(); +} + #[actix_web::test] async fn cli_with_dumps() { let db_path = tempfile::tempdir().unwrap(); From e3a6d63b523f0477367c0abc4f2971d6bdd779df Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Fri, 1 Aug 2025 08:42:27 +0200 Subject: [PATCH 27/53] Add utoipa types --- crates/meilisearch/src/routes/mod.rs | 5 ++++- crates/meilisearch/src/routes/webhooks.rs | 12 ++++++------ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 4ae72b0bd..2a41ce021 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -41,6 +41,9 @@ use crate::routes::indexes::IndexView; use crate::routes::multi_search::SearchResults; use crate::routes::network::{Network, Remote}; use crate::routes::swap_indexes::SwapIndexesPayload; +use crate::routes::webhooks::{ + WebhookResults, WebhookSettings, WebhookWithMetadata, WebhooksSettings, +}; use crate::search::{ FederatedSearch, FederatedSearchResult, Federation, FederationOptions, MergeFacets, SearchQueryWithIndex, SearchResultWithIndex, SimilarQuery, SimilarResult, @@ -101,7 +104,7 @@ mod webhooks; url = "/", description = "Local server", )), - components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export)) + components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export, WebhookSettings, WebhooksSettings, WebhookResults, WebhookWithMetadata)) )] pub struct MeilisearchApi; diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 776667829..054279727 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -51,7 +51,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] -struct WebhookSettings { +pub(super) struct WebhookSettings { #[schema(value_type = Option)] #[deserr(default, error = DeserrJsonError)] #[serde(default)] @@ -66,7 +66,7 @@ struct WebhookSettings { #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] -struct WebhooksSettings { +pub(super) struct WebhooksSettings { #[schema(value_type = Option>)] #[serde(default)] webhooks: Setting>>, @@ -75,7 +75,7 @@ struct WebhooksSettings { #[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] -struct WebhookWithMetadata { +pub(super) struct WebhookWithMetadata { uuid: Uuid, is_editable: bool, #[schema(value_type = WebhookSettings)] @@ -83,9 +83,9 @@ struct WebhookWithMetadata { webhook: Webhook, } -#[derive(Debug, Serialize)] +#[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] -struct WebhookResults { +pub(super) struct WebhookResults { results: Vec, } @@ -95,7 +95,7 @@ struct WebhookResults { tag = "Webhooks", security(("Bearer" = ["webhooks.get", "*.get", "*"])), responses( - (status = OK, description = "Webhooks are returned", body = WebhooksSettings, content_type = "application/json", example = json!({ + (status = OK, description = "Webhooks are returned", body = WebhookResults, content_type = "application/json", example = json!({ "results": [ { "uuid": "550e8400-e29b-41d4-a716-446655440000", From 7acbb1e14086b41e5de3cc9813ea9cc827118fdc Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 14:49:27 +0200 Subject: [PATCH 28/53] Remove PATCH /webhooks --- crates/meilisearch/src/routes/mod.rs | 6 +- crates/meilisearch/src/routes/webhooks.rs | 259 ++++++++-------------- crates/meilisearch/tests/common/server.rs | 4 - crates/meilisearch/tests/tasks/webhook.rs | 43 ++-- 4 files changed, 114 insertions(+), 198 deletions(-) diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 2a41ce021..745ac5824 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -41,9 +41,7 @@ use crate::routes::indexes::IndexView; use crate::routes::multi_search::SearchResults; use crate::routes::network::{Network, Remote}; use crate::routes::swap_indexes::SwapIndexesPayload; -use crate::routes::webhooks::{ - WebhookResults, WebhookSettings, WebhookWithMetadata, WebhooksSettings, -}; +use crate::routes::webhooks::{WebhookResults, WebhookSettings, WebhookWithMetadata}; use crate::search::{ FederatedSearch, FederatedSearchResult, Federation, FederationOptions, MergeFacets, SearchQueryWithIndex, SearchResultWithIndex, SimilarQuery, SimilarResult, @@ -104,7 +102,7 @@ mod webhooks; url = "/", description = "Local server", )), - components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export, WebhookSettings, WebhooksSettings, WebhookResults, WebhookWithMetadata)) + components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export, WebhookSettings, WebhookResults, WebhookWithMetadata)) )] pub struct MeilisearchApi; diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 054279727..b6f85f7d0 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -10,7 +10,7 @@ use meilisearch_types::error::deserr_codes::{InvalidWebhooksHeaders, InvalidWebh use meilisearch_types::error::{ErrorCode, ResponseError}; use meilisearch_types::keys::actions; use meilisearch_types::milli::update::Setting; -use meilisearch_types::webhooks::{Webhook, Webhooks}; +use meilisearch_types::webhooks::Webhook; use serde::Serialize; use tracing::debug; use utoipa::{OpenApi, ToSchema}; @@ -23,7 +23,7 @@ use crate::extractors::sequential_extractor::SeqHandler; #[derive(OpenApi)] #[openapi( - paths(get_webhooks, patch_webhooks, get_webhook, post_webhook, patch_webhook, delete_webhook), + paths(get_webhooks, get_webhook, post_webhook, patch_webhook, delete_webhook), tags(( name = "Webhooks", description = "The `/webhooks` route allows you to register endpoints to be called once tasks are processed.", @@ -36,7 +36,6 @@ pub fn configure(cfg: &mut web::ServiceConfig) { cfg.service( web::resource("") .route(web::get().to(get_webhooks)) - .route(web::patch().to(SeqHandler(patch_webhooks))) .route(web::post().to(SeqHandler(post_webhook))), ) .service( @@ -62,16 +61,6 @@ pub(super) struct WebhookSettings { headers: Setting>>, } -#[derive(Debug, Deserr, ToSchema)] -#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -#[schema(rename_all = "camelCase")] -pub(super) struct WebhooksSettings { - #[schema(value_type = Option>)] - #[serde(default)] - webhooks: Setting>>, -} - #[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] @@ -83,6 +72,12 @@ pub(super) struct WebhookWithMetadata { webhook: Webhook, } +impl WebhookWithMetadata { + pub fn from(uuid: Uuid, webhook: Webhook) -> Self { + Self { uuid, is_editable: uuid != Uuid::nil(), webhook } + } +} + #[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] pub(super) struct WebhookResults { @@ -142,17 +137,12 @@ async fn get_webhooks( #[derive(Serialize, Default)] pub struct PatchWebhooksAnalytics { - patch_webhooks_count: usize, patch_webhook_count: usize, post_webhook_count: usize, delete_webhook_count: usize, } impl PatchWebhooksAnalytics { - pub fn patch_webhooks() -> Self { - PatchWebhooksAnalytics { patch_webhooks_count: 1, ..Default::default() } - } - pub fn patch_webhook() -> Self { PatchWebhooksAnalytics { patch_webhook_count: 1, ..Default::default() } } @@ -173,7 +163,6 @@ impl Aggregate for PatchWebhooksAnalytics { fn aggregate(self: Box, new: Box) -> Box { Box::new(PatchWebhooksAnalytics { - patch_webhooks_count: self.patch_webhooks_count + new.patch_webhooks_count, patch_webhook_count: self.patch_webhook_count + new.patch_webhook_count, post_webhook_count: self.post_webhook_count + new.post_webhook_count, delete_webhook_count: self.delete_webhook_count + new.delete_webhook_count, @@ -213,130 +202,45 @@ impl ErrorCode for WebhooksError { } } -#[utoipa::path( - patch, - path = "", - tag = "Webhooks", - request_body = WebhooksSettings, - security(("Bearer" = ["webhooks.update", "*"])), - responses( - (status = 200, description = "Returns the updated webhooks", body = WebhooksSettings, content_type = "application/json", example = json!({ - "webhooks": { - "550e8400-e29b-41d4-a716-446655440000": { - "url": "http://example.com/webhook", - }, - "550e8400-e29b-41d4-a716-446655440001": { - "url": "https://your.site/on-tasks-completed", - "headers": { - "Authorization": "Bearer a-secret-token" - } - } - } - })), - (status = 401, description = "The authorization header is missing", body = ResponseError, content_type = "application/json", example = json!({ - "message": "The Authorization header is missing. It must use the bearer authorization method.", - "code": "missing_authorization_header", - "type": "auth", - "link": "https://docs.meilisearch.com/errors#missing_authorization_header" - })), - ) -)] -async fn patch_webhooks( - index_scheduler: GuardedData, Data>, - new_webhooks: AwebJson, - req: HttpRequest, - analytics: Data, -) -> Result { - let webhooks = patch_webhooks_inner(&index_scheduler, new_webhooks.0)?; +fn patch_webhook_inner( + uuid: &Uuid, + old_webhook: Option, + new_webhook: WebhookSettings, +) -> Result { + let (old_url, mut headers) = + old_webhook.map(|w| (Some(w.url), w.headers)).unwrap_or((None, BTreeMap::new())); - analytics.publish(PatchWebhooksAnalytics::patch_webhooks(), &req); - - Ok(HttpResponse::Ok().json(webhooks)) -} - -fn patch_webhooks_inner( - index_scheduler: &GuardedData, Data>, - new_webhooks: WebhooksSettings, -) -> Result { - fn merge_webhook( - uuid: &Uuid, - old_webhook: Option, - new_webhook: WebhookSettings, - ) -> Result { - let (old_url, mut headers) = - old_webhook.map(|w| (Some(w.url), w.headers)).unwrap_or((None, BTreeMap::new())); - - let url = match new_webhook.url { - Setting::Set(url) => url, - Setting::NotSet => old_url.ok_or_else(|| WebhooksError::MissingUrl(uuid.to_owned()))?, - Setting::Reset => return Err(WebhooksError::MissingUrl(uuid.to_owned())), - }; - - let headers = match new_webhook.headers { - Setting::Set(new_headers) => { - for (name, value) in new_headers { - match value { - Setting::Set(value) => { - headers.insert(name, value); - } - Setting::NotSet => continue, - Setting::Reset => { - headers.remove(&name); - continue; - } - } - } - headers - } - Setting::NotSet => headers, - Setting::Reset => BTreeMap::new(), - }; - - if headers.len() > 200 { - return Err(WebhooksError::TooManyHeaders(uuid.to_owned())); - } - - Ok(Webhook { url, headers }) - } - - debug!(parameters = ?new_webhooks, "Patch webhooks"); - - let Webhooks { mut webhooks } = index_scheduler.webhooks(); - - match new_webhooks.webhooks { - Setting::Set(new_webhooks) => { - for (uuid, new_webhook) in new_webhooks { - if uuid.is_nil() { - return Err(WebhooksError::ReservedWebhook(uuid).into()); - } - - match new_webhook { - Setting::Set(new_webhook) => { - let old_webhook = webhooks.remove(&uuid); - let webhook = merge_webhook(&uuid, old_webhook, new_webhook)?; - webhooks.insert(uuid, webhook); - } - Setting::Reset => { - webhooks.remove(&uuid); - } - Setting::NotSet => (), - } - } - } - Setting::Reset => webhooks.clear(), - Setting::NotSet => (), + let url = match new_webhook.url { + Setting::Set(url) => url, + Setting::NotSet => old_url.ok_or_else(|| WebhooksError::MissingUrl(uuid.to_owned()))?, + Setting::Reset => return Err(WebhooksError::MissingUrl(uuid.to_owned())), }; - if webhooks.len() > 20 { - return Err(WebhooksError::TooManyWebhooks.into()); + let headers = match new_webhook.headers { + Setting::Set(new_headers) => { + for (name, value) in new_headers { + match value { + Setting::Set(value) => { + headers.insert(name, value); + } + Setting::NotSet => continue, + Setting::Reset => { + headers.remove(&name); + continue; + } + } + } + headers + } + Setting::NotSet => headers, + Setting::Reset => BTreeMap::new(), + }; + + if headers.len() > 200 { + return Err(WebhooksError::TooManyHeaders(uuid.to_owned())); } - let webhooks = Webhooks { webhooks }; - index_scheduler.put_webhooks(webhooks.clone())?; - - debug!(returns = ?webhooks, "Patch webhooks"); - - Ok(webhooks) + Ok(Webhook { url, headers }) } #[utoipa::path( @@ -401,19 +305,35 @@ async fn post_webhook( req: HttpRequest, analytics: Data, ) -> Result { - let uuid = Uuid::new_v4(); + let webhook_settings = webhook_settings.into_inner(); + debug!(parameters = ?webhook_settings, "Post webhook"); - let webhooks = patch_webhooks_inner( - &index_scheduler, - WebhooksSettings { - webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Set(webhook_settings.0))])), - }, - )?; - let webhook = webhooks.webhooks.get(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?.clone(); + let uuid = Uuid::new_v4(); + if webhook_settings.headers.as_ref().set().is_some_and(|h| h.len() > 200) { + return Err(WebhooksError::TooManyHeaders(uuid).into()); + } + + let mut webhooks = index_scheduler.webhooks(); + if dbg!(webhooks.webhooks.len() >= 20) { + return Err(WebhooksError::TooManyWebhooks.into()); + } + + let webhook = Webhook { + url: webhook_settings.url.set().ok_or(WebhooksError::MissingUrl(uuid))?, + headers: webhook_settings + .headers + .set() + .map(|h| h.into_iter().map(|(k, v)| (k, v.set().unwrap_or_default())).collect()) + .unwrap_or_default(), + }; + webhooks.webhooks.insert(uuid, webhook.clone()); + index_scheduler.put_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::post_webhook(), &req); - Ok(HttpResponse::Created().json(WebhookWithMetadata { uuid, is_editable: true, webhook })) + let response = WebhookWithMetadata::from(uuid, webhook); + debug!(returns = ?response, "Post webhook"); + Ok(HttpResponse::Created().json(response)) } #[utoipa::path( @@ -446,22 +366,29 @@ async fn patch_webhook( analytics: Data, ) -> Result { let uuid = uuid.into_inner(); + let webhook_settings = webhook_settings.into_inner(); + debug!(parameters = ?(uuid, &webhook_settings), "Patch webhook"); - let webhooks = patch_webhooks_inner( - &index_scheduler, - WebhooksSettings { - webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Set(webhook_settings.0))])), - }, - )?; - let webhook = webhooks.webhooks.get(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?.clone(); + if uuid.is_nil() { + return Err(WebhooksError::ReservedWebhook(uuid).into()); + } + + let mut webhooks = index_scheduler.webhooks(); + let old_webhook = webhooks.webhooks.remove(&uuid); + let webhook = patch_webhook_inner(&uuid, old_webhook, webhook_settings)?; + + if webhook.headers.len() > 200 { + return Err(WebhooksError::TooManyHeaders(uuid).into()); + } + + webhooks.webhooks.insert(uuid, webhook.clone()); + index_scheduler.put_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::patch_webhook(), &req); - Ok(HttpResponse::Ok().json(WebhookWithMetadata { - uuid, - is_editable: uuid != Uuid::nil(), - webhook, - })) + let response = WebhookWithMetadata::from(uuid, webhook); + debug!(returns = ?response, "Patch webhook"); + Ok(HttpResponse::Ok().json(response)) } #[utoipa::path( @@ -485,18 +412,18 @@ async fn delete_webhook( analytics: Data, ) -> Result { let uuid = uuid.into_inner(); + debug!(parameters = ?uuid, "Delete webhook"); - let webhooks = index_scheduler.webhooks(); - if !webhooks.webhooks.contains_key(&uuid) { - return Err(WebhooksError::WebhookNotFound(uuid).into()); + if uuid.is_nil() { + return Err(WebhooksError::ReservedWebhook(uuid).into()); } - patch_webhooks_inner( - &index_scheduler, - WebhooksSettings { webhooks: Setting::Set(BTreeMap::from([(uuid, Setting::Reset)])) }, - )?; + let mut webhooks = index_scheduler.webhooks(); + webhooks.webhooks.remove(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?; + index_scheduler.put_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::delete_webhook(), &req); + debug!(returns = "No Content", "Delete webhook"); Ok(HttpResponse::NoContent().finish()) } diff --git a/crates/meilisearch/tests/common/server.rs b/crates/meilisearch/tests/common/server.rs index 0b57ca37a..4f1e93c88 100644 --- a/crates/meilisearch/tests/common/server.rs +++ b/crates/meilisearch/tests/common/server.rs @@ -182,10 +182,6 @@ impl Server { self.service.patch("/network", value).await } - pub async fn set_webhooks(&self, value: Value) -> (Value, StatusCode) { - self.service.patch("/webhooks", value).await - } - pub async fn create_webhook(&self, value: Value) -> (Value, StatusCode) { self.service.post("/webhooks", value).await } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 4caa7df92..f8fcd8ff9 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -99,6 +99,7 @@ async fn cli_only() { } #[actix_web::test] +#[ignore = "Broken"] async fn single_receives_data() { let WebhookHandle { server_handle, url, mut receiver } = create_webhook_server().await; @@ -165,6 +166,7 @@ async fn single_receives_data() { } #[actix_web::test] +#[ignore = "Broken"] async fn multiple_receive_data() { let server = Server::new().await; @@ -268,7 +270,7 @@ async fn reserved_names() { let server = Server::new().await; let (value, code) = server - .set_webhooks(json!({ "webhooks": { Uuid::nil(): { "url": "http://localhost:8080" } } })) + .patch_webhook(Uuid::nil().to_string(), json!({ "url": "http://localhost:8080" })) .await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" @@ -280,7 +282,7 @@ async fn reserved_names() { } "#); - let (value, code) = server.set_webhooks(json!({ "webhooks": { Uuid::nil(): null } })).await; + let (value, code) = server.delete_webhook(Uuid::nil().to_string()).await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { @@ -297,17 +299,13 @@ async fn over_limits() { let server = Server::new().await; // Too many webhooks + let mut uuids = Vec::new(); for _ in 0..20 { - let (_value, code) = server - .set_webhooks( - json!({ "webhooks": { Uuid::new_v4(): { "url": "http://localhost:8080" } } }), - ) - .await; - snapshot!(code, @"200 OK"); + let (value, code) = server.create_webhook(json!({ "url": "http://localhost:8080" } )).await; + snapshot!(code, @"201 Created"); + uuids.push(value.get("uuid").unwrap().as_str().unwrap().to_string()); } - let (value, code) = server - .set_webhooks(json!({ "webhooks": { Uuid::new_v4(): { "url": "http://localhost:8080" } } })) - .await; + let (value, code) = server.create_webhook(json!({ "url": "http://localhost:8080" })).await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { @@ -319,26 +317,23 @@ async fn over_limits() { "#); // Reset webhooks - let (value, code) = server.set_webhooks(json!({ "webhooks": null })).await; - snapshot!(code, @"200 OK"); - snapshot!(value, @r#" - { - "webhooks": {} + for uuid in uuids { + let (_value, code) = server.delete_webhook(&uuid).await; + snapshot!(code, @"204 No Content"); } - "#); // Test too many headers - let uuid = Uuid::new_v4(); + let (value, code) = server.create_webhook(json!({ "url": "http://localhost:8080" })).await; + snapshot!(code, @"201 Created"); + let uuid = value.get("uuid").unwrap().as_str().unwrap(); for i in 0..200 { let header_name = format!("header_{i}"); - let (_value, code) = server - .set_webhooks(json!({ "webhooks": { uuid: { "url": "http://localhost:8080", "headers": { header_name: "value" } } } })) - .await; + let (_value, code) = + server.patch_webhook(uuid, json!({ "headers": { header_name: "" } })).await; snapshot!(code, @"200 OK"); } - let (value, code) = server - .set_webhooks(json!({ "webhooks": { uuid: { "url": "http://localhost:8080", "headers": { "header_201": "value" } } } })) - .await; + let (value, code) = + server.patch_webhook(uuid, json!({ "headers": { "header_200": "" } })).await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { From c5caac95dd163e6dd8f868e096d148e38d120463 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 14:51:23 +0200 Subject: [PATCH 29/53] Format --- crates/meilisearch/src/routes/webhooks.rs | 35 +++++++++++------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index b6f85f7d0..07f19c498 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -20,6 +20,7 @@ use crate::analytics::{Aggregate, Analytics}; use crate::extractors::authentication::policies::ActionPolicy; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; +use WebhooksError::*; #[derive(OpenApi)] #[openapi( @@ -191,13 +192,11 @@ enum WebhooksError { impl ErrorCode for WebhooksError { fn error_code(&self) -> meilisearch_types::error::Code { match self { - WebhooksError::MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhooksUrl, - WebhooksError::TooManyWebhooks => meilisearch_types::error::Code::InvalidWebhooks, - WebhooksError::TooManyHeaders(_) => { - meilisearch_types::error::Code::InvalidWebhooksHeaders - } - WebhooksError::ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, - WebhooksError::WebhookNotFound(_) => meilisearch_types::error::Code::WebhookNotFound, + MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhooksUrl, + TooManyWebhooks => meilisearch_types::error::Code::InvalidWebhooks, + TooManyHeaders(_) => meilisearch_types::error::Code::InvalidWebhooksHeaders, + ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, + WebhookNotFound(_) => meilisearch_types::error::Code::WebhookNotFound, } } } @@ -212,8 +211,8 @@ fn patch_webhook_inner( let url = match new_webhook.url { Setting::Set(url) => url, - Setting::NotSet => old_url.ok_or_else(|| WebhooksError::MissingUrl(uuid.to_owned()))?, - Setting::Reset => return Err(WebhooksError::MissingUrl(uuid.to_owned())), + Setting::NotSet => old_url.ok_or_else(|| MissingUrl(uuid.to_owned()))?, + Setting::Reset => return Err(MissingUrl(uuid.to_owned())), }; let headers = match new_webhook.headers { @@ -237,7 +236,7 @@ fn patch_webhook_inner( }; if headers.len() > 200 { - return Err(WebhooksError::TooManyHeaders(uuid.to_owned())); + return Err(TooManyHeaders(uuid.to_owned())); } Ok(Webhook { url, headers }) @@ -271,7 +270,7 @@ async fn get_webhook( let uuid = uuid.into_inner(); let mut webhooks = index_scheduler.webhooks(); - let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?; + let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; Ok(HttpResponse::Ok().json(WebhookWithMetadata { uuid, @@ -310,16 +309,16 @@ async fn post_webhook( let uuid = Uuid::new_v4(); if webhook_settings.headers.as_ref().set().is_some_and(|h| h.len() > 200) { - return Err(WebhooksError::TooManyHeaders(uuid).into()); + return Err(TooManyHeaders(uuid).into()); } let mut webhooks = index_scheduler.webhooks(); if dbg!(webhooks.webhooks.len() >= 20) { - return Err(WebhooksError::TooManyWebhooks.into()); + return Err(TooManyWebhooks.into()); } let webhook = Webhook { - url: webhook_settings.url.set().ok_or(WebhooksError::MissingUrl(uuid))?, + url: webhook_settings.url.set().ok_or(MissingUrl(uuid))?, headers: webhook_settings .headers .set() @@ -370,7 +369,7 @@ async fn patch_webhook( debug!(parameters = ?(uuid, &webhook_settings), "Patch webhook"); if uuid.is_nil() { - return Err(WebhooksError::ReservedWebhook(uuid).into()); + return Err(ReservedWebhook(uuid).into()); } let mut webhooks = index_scheduler.webhooks(); @@ -378,7 +377,7 @@ async fn patch_webhook( let webhook = patch_webhook_inner(&uuid, old_webhook, webhook_settings)?; if webhook.headers.len() > 200 { - return Err(WebhooksError::TooManyHeaders(uuid).into()); + return Err(TooManyHeaders(uuid).into()); } webhooks.webhooks.insert(uuid, webhook.clone()); @@ -415,11 +414,11 @@ async fn delete_webhook( debug!(parameters = ?uuid, "Delete webhook"); if uuid.is_nil() { - return Err(WebhooksError::ReservedWebhook(uuid).into()); + return Err(ReservedWebhook(uuid).into()); } let mut webhooks = index_scheduler.webhooks(); - webhooks.webhooks.remove(&uuid).ok_or(WebhooksError::WebhookNotFound(uuid))?; + webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; index_scheduler.put_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::delete_webhook(), &req); From 4ec4710811a4b2c14470067430536823b02d1672 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 15:00:26 +0200 Subject: [PATCH 30/53] Improve logs --- crates/meilisearch/src/routes/webhooks.rs | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 07f19c498..a362b8bb1 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -125,13 +125,10 @@ async fn get_webhooks( let results = webhooks .webhooks .into_iter() - .map(|(uuid, webhook)| WebhookWithMetadata { - uuid, - is_editable: uuid != Uuid::nil(), - webhook, - }) + .map(|(uuid, webhook)| WebhookWithMetadata::from(uuid, webhook)) .collect::>(); let results = WebhookResults { results }; + debug!(returns = ?results, "Get webhooks"); Ok(HttpResponse::Ok().json(results)) } @@ -248,7 +245,7 @@ fn patch_webhook_inner( tag = "Webhooks", security(("Bearer" = ["webhooks.get", "*.get", "*"])), responses( - (status = 200, description = "Webhook found", body = WebhookSettings, content_type = "application/json", example = json!({ + (status = 200, description = "Webhook found", body = WebhookWithMetadata, content_type = "application/json", example = json!({ "uuid": "550e8400-e29b-41d4-a716-446655440000", "url": "https://your.site/on-tasks-completed", "headers": { @@ -271,12 +268,10 @@ async fn get_webhook( let mut webhooks = index_scheduler.webhooks(); let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; - - Ok(HttpResponse::Ok().json(WebhookWithMetadata { - uuid, - is_editable: uuid != Uuid::nil(), - webhook, - })) + let webhook = WebhookWithMetadata::from(uuid, webhook); + + debug!(returns = ?webhook, "Get webhook"); + Ok(HttpResponse::Ok().json(webhook)) } #[utoipa::path( From 737ad3ec1908e8c5f58baa26483d3b35ecf49f40 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 15:00:45 +0200 Subject: [PATCH 31/53] Add new api key actions --- crates/meilisearch-auth/src/store.rs | 8 ++++++++ crates/meilisearch-types/src/keys.rs | 21 ++++++++++++++++++++- crates/meilisearch/src/routes/webhooks.rs | 6 +++--- 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/crates/meilisearch-auth/src/store.rs b/crates/meilisearch-auth/src/store.rs index eb2170f08..470379e06 100644 --- a/crates/meilisearch-auth/src/store.rs +++ b/crates/meilisearch-auth/src/store.rs @@ -137,6 +137,14 @@ impl HeedAuthStore { Action::ChatsSettingsAll => { actions.extend([Action::ChatsSettingsGet, Action::ChatsSettingsUpdate]); } + Action::WebhooksAll => { + actions.extend([ + Action::WebhooksGet, + Action::WebhooksUpdate, + Action::WebhooksDelete, + Action::WebhooksCreate, + ]); + } other => { actions.insert(*other); } diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 2eddb9547..6763e2661 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -371,6 +371,15 @@ pub enum Action { #[serde(rename = "webhooks.update")] #[deserr(rename = "webhooks.update")] WebhooksUpdate, + #[serde(rename = "webhooks.delete")] + #[deserr(rename = "webhooks.delete")] + WebhooksDelete, + #[serde(rename = "webhooks.create")] + #[deserr(rename = "webhooks.create")] + WebhooksCreate, + #[serde(rename = "webhooks.*")] + #[deserr(rename = "webhooks.*")] + WebhooksAll, } impl Action { @@ -436,7 +445,9 @@ impl Action { match self { // Any action that expands to others must return false, as it wouldn't be able to expand recursively. All | AllGet | DocumentsAll | IndexesAll | ChatsAll | TasksAll | SettingsAll - | StatsAll | MetricsAll | DumpsAll | SnapshotsAll | ChatsSettingsAll => false, + | StatsAll | MetricsAll | DumpsAll | SnapshotsAll | ChatsSettingsAll | WebhooksAll => { + false + } Search => true, DocumentsAdd => false, @@ -473,6 +484,8 @@ impl Action { ChatsSettingsUpdate => false, WebhooksGet => true, WebhooksUpdate => false, + WebhooksDelete => false, + WebhooksCreate => false, } } @@ -535,6 +548,9 @@ pub mod actions { pub const WEBHOOKS_GET: u8 = WebhooksGet.repr(); pub const WEBHOOKS_UPDATE: u8 = WebhooksUpdate.repr(); + pub const WEBHOOKS_DELETE: u8 = WebhooksDelete.repr(); + pub const WEBHOOKS_CREATE: u8 = WebhooksCreate.repr(); + pub const WEBHOOKS_ALL: u8 = WebhooksAll.repr(); } #[cfg(test)] @@ -592,6 +608,9 @@ pub(crate) mod test { assert!(AllGet.repr() == 44 && ALL_GET == 44); assert!(WebhooksGet.repr() == 45 && WEBHOOKS_GET == 45); assert!(WebhooksUpdate.repr() == 46 && WEBHOOKS_UPDATE == 46); + assert!(WebhooksDelete.repr() == 47 && WEBHOOKS_DELETE == 47); + assert!(WebhooksCreate.repr() == 48 && WEBHOOKS_CREATE == 48); + assert!(WebhooksAll.repr() == 49 && WEBHOOKS_ALL == 49); } #[test] diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index a362b8bb1..67036e0b5 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -269,7 +269,7 @@ async fn get_webhook( let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; let webhook = WebhookWithMetadata::from(uuid, webhook); - + debug!(returns = ?webhook, "Get webhook"); Ok(HttpResponse::Ok().json(webhook)) } @@ -294,7 +294,7 @@ async fn get_webhook( ) )] async fn post_webhook( - index_scheduler: GuardedData, Data>, + index_scheduler: GuardedData, Data>, webhook_settings: AwebJson, req: HttpRequest, analytics: Data, @@ -400,7 +400,7 @@ async fn patch_webhook( ) )] async fn delete_webhook( - index_scheduler: GuardedData, Data>, + index_scheduler: GuardedData, Data>, uuid: Path, req: HttpRequest, analytics: Data, From 8dfebbb3e7b5b1f50ea3bc0e42ad6fc793c5ad40 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 15:37:12 +0200 Subject: [PATCH 32/53] Fix tests --- crates/index-scheduler/src/lib.rs | 63 ++++------------------- crates/meilisearch/tests/tasks/webhook.rs | 2 - 2 files changed, 11 insertions(+), 54 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 8d7617b6c..84cb0f752 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -820,58 +820,17 @@ impl IndexScheduler { let rtxn = self.env.read_txn()?; - let task_reader = TaskReader { - rtxn: &rtxn, - index_scheduler: self, - tasks: &mut updated.into_iter(), - buffer: Vec::with_capacity(800), // on average a task is around ~600 bytes - written: 0, - }; - - enum EitherRead<'a, T: Read> { - Other(Option), - Data(&'a [u8]), - } - - impl EitherRead<'_, T> { - /// A clone that works only once for the Other variant. - fn clone(&mut self) -> Self { - match self { - Self::Other(r) => { - let r = r.take(); - Self::Other(r) - } - Self::Data(arg0) => Self::Data(arg0), - } - } - } - - impl Read for EitherRead<'_, T> { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - match self { - EitherRead::Other(Some(reader)) => reader.read(buf), - EitherRead::Other(None) => { - Err(io::Error::new(io::ErrorKind::Other, "No reader available")) - } - EitherRead::Data(data) => data.read(buf), - } - } - } - - let mut reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); - - // When there is more than one webhook, cache the data in memory - let mut data; - let mut reader = match webhooks.webhooks.len() { - 1 => EitherRead::Other(Some(reader)), - _ => { - data = Vec::new(); - reader.read_to_end(&mut data)?; - EitherRead::Data(&data) - } - }; - for (uuid, Webhook { url, headers }) in webhooks.webhooks.iter() { + let task_reader = TaskReader { + rtxn: &rtxn, + index_scheduler: self, + tasks: &mut updated.into_iter(), + buffer: Vec::with_capacity(800), // on average a task is around ~600 bytes + written: 0, + }; + + let reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); + let mut request = ureq::post(url) .timeout(Duration::from_secs(30)) .set("Content-Encoding", "gzip") @@ -880,7 +839,7 @@ impl IndexScheduler { request = request.set(header_name, header_value); } - if let Err(e) = request.send(reader.clone()) { + if let Err(e) = request.send(reader) { tracing::error!("While sending data to the webhook {uuid}: {e}"); } } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index f8fcd8ff9..beef2f5c1 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -99,7 +99,6 @@ async fn cli_only() { } #[actix_web::test] -#[ignore = "Broken"] async fn single_receives_data() { let WebhookHandle { server_handle, url, mut receiver } = create_webhook_server().await; @@ -166,7 +165,6 @@ async fn single_receives_data() { } #[actix_web::test] -#[ignore = "Broken"] async fn multiple_receive_data() { let server = Server::new().await; From 69c59d3de3c709aafa66f751960e15fa5add6508 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 15:43:37 +0200 Subject: [PATCH 33/53] Update security in utoipa --- crates/meilisearch/src/routes/webhooks.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 67036e0b5..9dc448407 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -89,7 +89,7 @@ pub(super) struct WebhookResults { get, path = "", tag = "Webhooks", - security(("Bearer" = ["webhooks.get", "*.get", "*"])), + security(("Bearer" = ["webhooks.get", "webhooks.*", "*.get", "*"])), responses( (status = OK, description = "Webhooks are returned", body = WebhookResults, content_type = "application/json", example = json!({ "results": [ @@ -243,7 +243,7 @@ fn patch_webhook_inner( get, path = "/{uuid}", tag = "Webhooks", - security(("Bearer" = ["webhooks.get", "*.get", "*"])), + security(("Bearer" = ["webhooks.get", "webhooks.*", "*.get", "*"])), responses( (status = 200, description = "Webhook found", body = WebhookWithMetadata, content_type = "application/json", example = json!({ "uuid": "550e8400-e29b-41d4-a716-446655440000", @@ -279,7 +279,7 @@ async fn get_webhook( path = "", tag = "Webhooks", request_body = WebhookSettings, - security(("Bearer" = ["webhooks.update", "*"])), + security(("Bearer" = ["webhooks.create", "webhooks.*", "*"])), responses( (status = 201, description = "Webhook created successfully", body = WebhookWithMetadata, content_type = "application/json", example = json!({ "uuid": "550e8400-e29b-41d4-a716-446655440000", @@ -335,7 +335,7 @@ async fn post_webhook( path = "/{uuid}", tag = "Webhooks", request_body = WebhookSettings, - security(("Bearer" = ["webhooks.update", "*"])), + security(("Bearer" = ["webhooks.update", "webhooks.*", "*"])), responses( (status = 200, description = "Webhook updated successfully", body = WebhookWithMetadata, content_type = "application/json", example = json!({ "uuid": "550e8400-e29b-41d4-a716-446655440000", @@ -389,7 +389,7 @@ async fn patch_webhook( delete, path = "/{uuid}", tag = "Webhooks", - security(("Bearer" = ["webhooks.update", "*"])), + security(("Bearer" = ["webhooks.delete", "webhooks.*", "*"])), responses( (status = 204, description = "Webhook deleted successfully"), (status = 404, description = "Webhook not found", body = ResponseError, content_type = "application/json"), From 1754745c426ec8a1efe46ffd1f845c4d801cf3bb Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 16:26:20 +0200 Subject: [PATCH 34/53] Add URL and header validity checks --- crates/meilisearch/src/routes/webhooks.rs | 51 +++++++++++++++++++---- crates/meilisearch/tests/tasks/webhook.rs | 51 +++++++++++++++++++++++ 2 files changed, 94 insertions(+), 8 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 9dc448407..18edfb63c 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -1,5 +1,9 @@ use std::collections::BTreeMap; +use actix_http::header::{ + HeaderName, HeaderValue, InvalidHeaderName as ActixInvalidHeaderName, + InvalidHeaderValue as ActixInvalidHeaderValue, +}; use actix_web::web::{self, Data, Path}; use actix_web::{HttpRequest, HttpResponse}; use deserr::actix_web::AwebJson; @@ -13,6 +17,7 @@ use meilisearch_types::milli::update::Setting; use meilisearch_types::webhooks::Webhook; use serde::Serialize; use tracing::debug; +use url::Url; use utoipa::{OpenApi, ToSchema}; use uuid::Uuid; @@ -184,6 +189,12 @@ enum WebhooksError { ReservedWebhook(Uuid), #[error("Webhook `{0}` not found.")] WebhookNotFound(Uuid), + #[error("Invalid header name `{0}`: {1}")] + InvalidHeaderName(String, ActixInvalidHeaderName), + #[error("Invalid header value `{0}`: {1}")] + InvalidHeaderValue(String, ActixInvalidHeaderValue), + #[error("Invalid URL `{0}`: {1}")] + InvalidUrl(String, url::ParseError), } impl ErrorCode for WebhooksError { @@ -194,6 +205,9 @@ impl ErrorCode for WebhooksError { TooManyHeaders(_) => meilisearch_types::error::Code::InvalidWebhooksHeaders, ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, WebhookNotFound(_) => meilisearch_types::error::Code::WebhookNotFound, + InvalidHeaderName(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, + InvalidHeaderValue(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, + InvalidUrl(_, _) => meilisearch_types::error::Code::InvalidWebhooksUrl, } } } @@ -239,6 +253,32 @@ fn patch_webhook_inner( Ok(Webhook { url, headers }) } +fn check_changed(uuid: Uuid, webhook: &Webhook) -> Result<(), WebhooksError> { + if uuid.is_nil() { + return Err(ReservedWebhook(uuid)); + } + + if webhook.url.is_empty() { + return Err(MissingUrl(uuid)); + } + + if webhook.headers.len() > 200 { + return Err(TooManyHeaders(uuid)); + } + + for (header, value) in &webhook.headers { + HeaderName::from_bytes(header.as_bytes()) + .map_err(|e| InvalidHeaderName(header.to_owned(), e))?; + HeaderValue::from_str(value).map_err(|e| InvalidHeaderValue(header.to_owned(), e))?; + } + + if let Err(e) = Url::parse(&webhook.url) { + return Err(InvalidUrl(webhook.url.to_owned(), e)); + } + + Ok(()) +} + #[utoipa::path( get, path = "/{uuid}", @@ -320,6 +360,8 @@ async fn post_webhook( .map(|h| h.into_iter().map(|(k, v)| (k, v.set().unwrap_or_default())).collect()) .unwrap_or_default(), }; + + check_changed(uuid, &webhook)?; webhooks.webhooks.insert(uuid, webhook.clone()); index_scheduler.put_webhooks(webhooks)?; @@ -363,18 +405,11 @@ async fn patch_webhook( let webhook_settings = webhook_settings.into_inner(); debug!(parameters = ?(uuid, &webhook_settings), "Patch webhook"); - if uuid.is_nil() { - return Err(ReservedWebhook(uuid).into()); - } - let mut webhooks = index_scheduler.webhooks(); let old_webhook = webhooks.webhooks.remove(&uuid); let webhook = patch_webhook_inner(&uuid, old_webhook, webhook_settings)?; - if webhook.headers.len() > 200 { - return Err(TooManyHeaders(uuid).into()); - } - + check_changed(uuid, &webhook)?; webhooks.webhooks.insert(uuid, webhook.clone()); index_scheduler.put_webhooks(webhooks)?; diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index beef2f5c1..a1029da6d 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -469,3 +469,54 @@ async fn patch() { } "#); } + +#[actix_web::test] +async fn invalid_url_and_headers() { + let server = Server::new().await; + + // Test invalid URL format + let (value, code) = server.create_webhook(json!({ "url": "not-a-valid-url" })).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid URL `not-a-valid-url`: relative URL without a base", + "code": "invalid_webhooks_url", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + } + "#); + + // Test invalid header name (containing spaces) + let (value, code) = server + .create_webhook(json!({ + "url": "https://example.com/hook", + "headers": { "invalid header name": "value" } + })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid header name `invalid header name`: invalid HTTP header name", + "code": "invalid_webhooks_headers", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" + } + "#); + + // Test invalid header value (containing control characters) + let (value, code) = server + .create_webhook(json!({ + "url": "https://example.com/hook", + "headers": { "authorization": "token\nwith\nnewlines" } + })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid header value `authorization`: failed to parse header value", + "code": "invalid_webhooks_headers", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" + } + "#); +} From 3b0f576d56bb99c38ea03ce25c0b6b745a7dc2d9 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 16:38:00 +0200 Subject: [PATCH 35/53] Improve invalid uuid error message --- crates/meilisearch-types/src/error.rs | 1 + crates/meilisearch/src/routes/webhooks.rs | 16 +++++---- crates/meilisearch/tests/tasks/webhook.rs | 42 +++++++++++++++++++++++ 3 files changed, 53 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index 3916012c1..e14a06909 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -424,6 +424,7 @@ InvalidWebhooks , InvalidRequest , BAD_REQU InvalidWebhooksUrl , InvalidRequest , BAD_REQUEST ; InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST ; ReservedWebhook , InvalidRequest , BAD_REQUEST ; +InvalidWebhookUuid , InvalidRequest , BAD_REQUEST ; WebhookNotFound , InvalidRequest , NOT_FOUND } diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 18edfb63c..2454b624b 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::str::FromStr; use actix_http::header::{ HeaderName, HeaderValue, InvalidHeaderName as ActixInvalidHeaderName, @@ -195,6 +196,8 @@ enum WebhooksError { InvalidHeaderValue(String, ActixInvalidHeaderValue), #[error("Invalid URL `{0}`: {1}")] InvalidUrl(String, url::ParseError), + #[error("Invalid UUID: {0}")] + InvalidUuid(uuid::Error), } impl ErrorCode for WebhooksError { @@ -208,6 +211,7 @@ impl ErrorCode for WebhooksError { InvalidHeaderName(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, InvalidHeaderValue(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, InvalidUrl(_, _) => meilisearch_types::error::Code::InvalidWebhooksUrl, + InvalidUuid(_) => meilisearch_types::error::Code::InvalidWebhookUuid, } } } @@ -302,9 +306,9 @@ fn check_changed(uuid: Uuid, webhook: &Webhook) -> Result<(), WebhooksError> { )] async fn get_webhook( index_scheduler: GuardedData, Data>, - uuid: Path, + uuid: Path, ) -> Result { - let uuid = uuid.into_inner(); + let uuid = Uuid::from_str(&uuid.into_inner()).map_err(InvalidUuid)?; let mut webhooks = index_scheduler.webhooks(); let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; @@ -396,12 +400,12 @@ async fn post_webhook( )] async fn patch_webhook( index_scheduler: GuardedData, Data>, - uuid: Path, + uuid: Path, webhook_settings: AwebJson, req: HttpRequest, analytics: Data, ) -> Result { - let uuid = uuid.into_inner(); + let uuid = Uuid::from_str(&uuid.into_inner()).map_err(InvalidUuid)?; let webhook_settings = webhook_settings.into_inner(); debug!(parameters = ?(uuid, &webhook_settings), "Patch webhook"); @@ -436,11 +440,11 @@ async fn patch_webhook( )] async fn delete_webhook( index_scheduler: GuardedData, Data>, - uuid: Path, + uuid: Path, req: HttpRequest, analytics: Data, ) -> Result { - let uuid = uuid.into_inner(); + let uuid = Uuid::from_str(&uuid.into_inner()).map_err(InvalidUuid)?; debug!(parameters = ?uuid, "Delete webhook"); if uuid.is_nil() { diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index a1029da6d..155312b9d 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -520,3 +520,45 @@ async fn invalid_url_and_headers() { } "#); } + +#[actix_web::test] +async fn invalid_uuid() { + let server = Server::new().await; + + // Test get webhook with invalid UUID + let (value, code) = server.get_webhook("invalid-uuid").await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid UUID: invalid character: expected an optional prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `i` at 1", + "code": "invalid_webhook_uuid", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhook_uuid" + } + "#); + + // Test update webhook with invalid UUID + let (value, code) = + server.patch_webhook("invalid-uuid", json!({ "url": "https://example.com/hook" })).await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid UUID: invalid character: expected an optional prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `i` at 1", + "code": "invalid_webhook_uuid", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhook_uuid" + } + "#); + + // Test delete webhook with invalid UUID + let (value, code) = server.delete_webhook("invalid-uuid").await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Invalid UUID: invalid character: expected an optional prefix of `urn:uuid:` followed by [0-9a-fA-F-], found `i` at 1", + "code": "invalid_webhook_uuid", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhook_uuid" + } + "#); +} From 3b26d64a5d1504a324b5ca2606a227fe35409913 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 16:39:34 +0200 Subject: [PATCH 36/53] Edit reserved webhook message --- crates/meilisearch/src/routes/webhooks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 2454b624b..fc03f43d0 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -186,7 +186,7 @@ enum WebhooksError { TooManyWebhooks, #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] TooManyHeaders(Uuid), - #[error("Cannot edit webhook `{0}`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.")] + #[error("Cannot edit webhook `{0}`. The webhook defined from the command line cannot be modified using the API.")] ReservedWebhook(Uuid), #[error("Webhook `{0}` not found.")] WebhookNotFound(Uuid), From ddfcacbb621e6ba90484be4462c550f305c4b7e3 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 16:53:41 +0200 Subject: [PATCH 37/53] Add nice error message for users trying to set uuid or isEditable --- crates/meilisearch-types/src/error.rs | 4 +- crates/meilisearch/src/routes/webhooks.rs | 29 ++++++-- crates/meilisearch/tests/tasks/webhook.rs | 86 ++++++++++++++++++++++- 3 files changed, 111 insertions(+), 8 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index e14a06909..e25ce61a4 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -425,7 +425,9 @@ InvalidWebhooksUrl , InvalidRequest , BAD_REQU InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST ; ReservedWebhook , InvalidRequest , BAD_REQUEST ; InvalidWebhookUuid , InvalidRequest , BAD_REQUEST ; -WebhookNotFound , InvalidRequest , NOT_FOUND +WebhookNotFound , InvalidRequest , NOT_FOUND ; +ImmutableWebhookUuid , InvalidRequest , BAD_REQUEST ; +ImmutableWebhookIsEditable , InvalidRequest , BAD_REQUEST } impl ErrorCode for JoinError { diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index fc03f43d0..610e28271 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -7,12 +7,15 @@ use actix_http::header::{ }; use actix_web::web::{self, Data, Path}; use actix_web::{HttpRequest, HttpResponse}; +use core::convert::Infallible; use deserr::actix_web::AwebJson; -use deserr::Deserr; +use deserr::{DeserializeError, Deserr, ValuePointerRef}; use index_scheduler::IndexScheduler; -use meilisearch_types::deserr::DeserrJsonError; -use meilisearch_types::error::deserr_codes::{InvalidWebhooksHeaders, InvalidWebhooksUrl}; -use meilisearch_types::error::{ErrorCode, ResponseError}; +use meilisearch_types::deserr::{immutable_field_error, DeserrJsonError}; +use meilisearch_types::error::deserr_codes::{ + BadRequest, InvalidWebhooksHeaders, InvalidWebhooksUrl, +}; +use meilisearch_types::error::{Code, ErrorCode, ResponseError}; use meilisearch_types::keys::actions; use meilisearch_types::milli::update::Setting; use meilisearch_types::webhooks::Webhook; @@ -54,7 +57,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) { } #[derive(Debug, Deserr, ToSchema)] -#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields = deny_immutable_fields_webhook)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] pub(super) struct WebhookSettings { @@ -68,6 +71,22 @@ pub(super) struct WebhookSettings { headers: Setting>>, } +fn deny_immutable_fields_webhook( + field: &str, + accepted: &[&str], + location: ValuePointerRef, +) -> DeserrJsonError { + match field { + "uuid" => immutable_field_error(field, accepted, Code::ImmutableWebhookUuid), + "isEditable" => immutable_field_error(field, accepted, Code::ImmutableWebhookIsEditable), + _ => deserr::take_cf_content(DeserrJsonError::::error::( + None, + deserr::ErrorKind::UnknownKey { key: field, accepted }, + location, + )), + } +} + #[derive(Debug, Serialize, ToSchema)] #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 155312b9d..03f732f0d 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -273,7 +273,7 @@ async fn reserved_names() { snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Cannot edit webhook `[uuid]`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", + "message": "Cannot edit webhook `[uuid]`. The webhook defined from the command line cannot be modified using the API.", "code": "reserved_webhook", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#reserved_webhook" @@ -284,7 +284,7 @@ async fn reserved_names() { snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Cannot edit webhook `[uuid]`. Webhooks prefixed with an underscore are reserved and may not be modified using the API.", + "message": "Cannot edit webhook `[uuid]`. The webhook defined from the command line cannot be modified using the API.", "code": "reserved_webhook", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#reserved_webhook" @@ -562,3 +562,85 @@ async fn invalid_uuid() { } "#); } + +#[actix_web::test] +async fn forbidden_fields() { + let server = Server::new().await; + + // Test creating webhook with uuid field + let custom_uuid = Uuid::new_v4(); + let (value, code) = server + .create_webhook(json!({ + "url": "https://example.com/hook", + "uuid": custom_uuid.to_string(), + "headers": { "authorization": "TOKEN" } + })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Immutable field `uuid`: expected one of `url`, `headers`", + "code": "immutable_webhook_uuid", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#immutable_webhook_uuid" + } + "#); + + // Test creating webhook with isEditable field + let (value, code) = server + .create_webhook(json!({ + "url": "https://example.com/hook2", + "isEditable": false, + "headers": { "authorization": "TOKEN" } + })) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Immutable field `isEditable`: expected one of `url`, `headers`", + "code": "immutable_webhook_is_editable", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#immutable_webhook_is_editable" + } + "#); + + // Test patching webhook with uuid field + let (value, code) = server + .patch_webhook( + "uuid-whatever", + json!({ + "uuid": Uuid::new_v4(), + "headers": { "new-header": "value" } + }), + ) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(value, @r#" + { + "message": "Immutable field `uuid`: expected one of `url`, `headers`", + "code": "immutable_webhook_uuid", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#immutable_webhook_uuid" + } + "#); + + // Test patching webhook with isEditable field + let (value, code) = server + .patch_webhook( + "uuid-whatever", + json!({ + "isEditable": false, + "headers": { "another-header": "value" } + }), + ) + .await; + snapshot!(code, @"400 Bad Request"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" + { + "message": "Immutable field `isEditable`: expected one of `url`, `headers`", + "code": "immutable_webhook_is_editable", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#immutable_webhook_is_editable" + } + "#); +} From 7251cccd0310b17575786f695a9848600d6ebbaa Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Mon, 4 Aug 2025 17:13:05 +0200 Subject: [PATCH 38/53] Make notify_webhooks execute in its own thread --- crates/index-scheduler/src/lib.rs | 10 ++-------- crates/index-scheduler/src/scheduler/mod.rs | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 84cb0f752..9af74a3e5 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -766,14 +766,8 @@ impl IndexScheduler { Ok(()) } - /// Once the tasks changes have been committed we must send all the tasks that were updated to our webhook if there is one. - fn notify_webhook(&self, updated: &RoaringBitmap) -> Result<()> { - let webhooks = self.cached_webhooks.read().unwrap_or_else(|poisoned| poisoned.into_inner()); - if webhooks.webhooks.is_empty() { - return Ok(()); - } - let webhooks = Webhooks::clone(&*webhooks); - + /// Once the tasks changes have been committed we must send all the tasks that were updated to our webhooks + fn notify_webhooks(&self, webhooks: Webhooks, updated: &RoaringBitmap) -> Result<()> { struct TaskReader<'a, 'b> { rtxn: &'a RoTxn<'a>, index_scheduler: &'a IndexScheduler, diff --git a/crates/index-scheduler/src/scheduler/mod.rs b/crates/index-scheduler/src/scheduler/mod.rs index 5ac591143..b5acf7582 100644 --- a/crates/index-scheduler/src/scheduler/mod.rs +++ b/crates/index-scheduler/src/scheduler/mod.rs @@ -26,6 +26,7 @@ use meilisearch_types::error::ResponseError; use meilisearch_types::heed::{Env, WithoutTls}; use meilisearch_types::milli; use meilisearch_types::tasks::Status; +use meilisearch_types::webhooks::Webhooks; use process_batch::ProcessBatchInfo; use rayon::current_num_threads; use rayon::iter::{IntoParallelIterator, ParallelIterator}; @@ -446,8 +447,17 @@ impl IndexScheduler { Ok(()) })?; - // We shouldn't crash the tick function if we can't send data to the webhook. - let _ = self.notify_webhook(&ids); + // We shouldn't crash the tick function if we can't send data to the webhooks + let webhooks = self.cached_webhooks.read().unwrap_or_else(|p| p.into_inner()); + if !webhooks.webhooks.is_empty() { + let webhooks = Webhooks::clone(&*webhooks); + let cloned_index_scheduler = self.private_clone(); + std::thread::spawn(move || { + if let Err(e) = cloned_index_scheduler.notify_webhooks(webhooks, &ids) { + tracing::error!("Failure to notify webhooks: {e}"); + } + }); + } #[cfg(test)] self.breakpoint(crate::test_utils::Breakpoint::AfterProcessing); From d340013d8b28ab0d3bb289cc0e2c6d742481ed88 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:35:12 +0200 Subject: [PATCH 39/53] Change error name --- crates/meilisearch-types/src/error.rs | 2 +- crates/meilisearch/src/routes/webhooks.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index e25ce61a4..bb486726a 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -423,7 +423,7 @@ InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQU InvalidWebhooks , InvalidRequest , BAD_REQUEST ; InvalidWebhooksUrl , InvalidRequest , BAD_REQUEST ; InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST ; -ReservedWebhook , InvalidRequest , BAD_REQUEST ; +ImmutableWebhook , InvalidRequest , BAD_REQUEST ; InvalidWebhookUuid , InvalidRequest , BAD_REQUEST ; WebhookNotFound , InvalidRequest , NOT_FOUND ; ImmutableWebhookUuid , InvalidRequest , BAD_REQUEST ; diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 610e28271..65aff4179 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -205,8 +205,8 @@ enum WebhooksError { TooManyWebhooks, #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] TooManyHeaders(Uuid), - #[error("Cannot edit webhook `{0}`. The webhook defined from the command line cannot be modified using the API.")] - ReservedWebhook(Uuid), + #[error("Webhook `{0}` is immutable. The webhook defined from the command line cannot be modified using the API.")] + ImmutableWebhook(Uuid), #[error("Webhook `{0}` not found.")] WebhookNotFound(Uuid), #[error("Invalid header name `{0}`: {1}")] @@ -225,7 +225,7 @@ impl ErrorCode for WebhooksError { MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhooksUrl, TooManyWebhooks => meilisearch_types::error::Code::InvalidWebhooks, TooManyHeaders(_) => meilisearch_types::error::Code::InvalidWebhooksHeaders, - ReservedWebhook(_) => meilisearch_types::error::Code::ReservedWebhook, + ImmutableWebhook(_) => meilisearch_types::error::Code::ImmutableWebhook, WebhookNotFound(_) => meilisearch_types::error::Code::WebhookNotFound, InvalidHeaderName(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, InvalidHeaderValue(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, @@ -278,7 +278,7 @@ fn patch_webhook_inner( fn check_changed(uuid: Uuid, webhook: &Webhook) -> Result<(), WebhooksError> { if uuid.is_nil() { - return Err(ReservedWebhook(uuid)); + return Err(ImmutableWebhook(uuid)); } if webhook.url.is_empty() { @@ -467,7 +467,7 @@ async fn delete_webhook( debug!(parameters = ?uuid, "Delete webhook"); if uuid.is_nil() { - return Err(ReservedWebhook(uuid).into()); + return Err(ImmutableWebhook(uuid).into()); } let mut webhooks = index_scheduler.webhooks(); From 43c20bb3ed6903d709175b32f3639657a6d85c84 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:39:52 +0200 Subject: [PATCH 40/53] Add missing actions in from_repr Co-Authored-By: Thomas Campistron --- crates/meilisearch-types/src/keys.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/meilisearch-types/src/keys.rs b/crates/meilisearch-types/src/keys.rs index 6763e2661..06f621e70 100644 --- a/crates/meilisearch-types/src/keys.rs +++ b/crates/meilisearch-types/src/keys.rs @@ -433,6 +433,9 @@ impl Action { ALL_GET => Some(Self::AllGet), WEBHOOKS_GET => Some(Self::WebhooksGet), WEBHOOKS_UPDATE => Some(Self::WebhooksUpdate), + WEBHOOKS_DELETE => Some(Self::WebhooksDelete), + WEBHOOKS_CREATE => Some(Self::WebhooksCreate), + WEBHOOKS_ALL => Some(Self::WebhooksAll), _otherwise => None, } } From 84651ffd7d63e239e585a4986da6c11db1c73b72 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:41:28 +0200 Subject: [PATCH 41/53] Remove hardcoded buffer size Co-Authored-By: Thomas Campistron --- crates/index-scheduler/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 9af74a3e5..c18efeaf2 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -819,7 +819,7 @@ impl IndexScheduler { rtxn: &rtxn, index_scheduler: self, tasks: &mut updated.into_iter(), - buffer: Vec::with_capacity(800), // on average a task is around ~600 bytes + buffer: Vec::with_capacity(page_size::get()), written: 0, }; From 8ef1a50086b4848b3fe310eef9861c1dad954a56 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:42:39 +0200 Subject: [PATCH 42/53] Add hint Co-Authored-By: Thomas Campistron --- crates/meilisearch/src/routes/webhooks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 65aff4179..c97f2fc5f 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -203,7 +203,7 @@ enum WebhooksError { MissingUrl(Uuid), #[error("Defining too many webhooks would crush the server. Please limit the number of webhooks to 20. You may use a third-party proxy server to dispatch events to more than 20 endpoints.")] TooManyWebhooks, - #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200.")] + #[error("Too many headers for the webhook `{0}`. Please limit the number of headers to 200. Hint: To remove an already defined header set its value to `null`")] TooManyHeaders(Uuid), #[error("Webhook `{0}` is immutable. The webhook defined from the command line cannot be modified using the API.")] ImmutableWebhook(Uuid), From 386cf832856a6173544d2fec818cf1e085c9738f Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:48:39 +0200 Subject: [PATCH 43/53] Improve webhook settings --- crates/meilisearch/src/routes/webhooks.rs | 25 ++++++++--------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index c97f2fc5f..86847c4bc 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -61,10 +61,9 @@ pub fn configure(cfg: &mut web::ServiceConfig) { #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] pub(super) struct WebhookSettings { - #[schema(value_type = Option)] #[deserr(default, error = DeserrJsonError)] #[serde(default)] - url: Setting, + url: Option, #[schema(value_type = Option>, example = json!({"Authorization":"Bearer a-secret-token"}))] #[deserr(default, error = DeserrJsonError)] #[serde(default)] @@ -237,19 +236,14 @@ impl ErrorCode for WebhooksError { fn patch_webhook_inner( uuid: &Uuid, - old_webhook: Option, + old_webhook: Webhook, new_webhook: WebhookSettings, ) -> Result { - let (old_url, mut headers) = - old_webhook.map(|w| (Some(w.url), w.headers)).unwrap_or((None, BTreeMap::new())); + let Webhook { url: old_url, mut headers } = old_webhook; - let url = match new_webhook.url { - Setting::Set(url) => url, - Setting::NotSet => old_url.ok_or_else(|| MissingUrl(uuid.to_owned()))?, - Setting::Reset => return Err(MissingUrl(uuid.to_owned())), - }; + let url = new_webhook.url.unwrap_or(old_url); - let headers = match new_webhook.headers { + match new_webhook.headers { Setting::Set(new_headers) => { for (name, value) in new_headers { match value { @@ -263,10 +257,9 @@ fn patch_webhook_inner( } } } - headers } - Setting::NotSet => headers, - Setting::Reset => BTreeMap::new(), + Setting::Reset => headers.clear(), + Setting::NotSet => (), }; if headers.len() > 200 { @@ -376,7 +369,7 @@ async fn post_webhook( } let webhook = Webhook { - url: webhook_settings.url.set().ok_or(MissingUrl(uuid))?, + url: webhook_settings.url.ok_or(MissingUrl(uuid))?, headers: webhook_settings .headers .set() @@ -429,7 +422,7 @@ async fn patch_webhook( debug!(parameters = ?(uuid, &webhook_settings), "Patch webhook"); let mut webhooks = index_scheduler.webhooks(); - let old_webhook = webhooks.webhooks.remove(&uuid); + let old_webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; let webhook = patch_webhook_inner(&uuid, old_webhook, webhook_settings)?; check_changed(uuid, &webhook)?; From b2d157a74a7088ccd25679f5c78cdacc2b36dd5a Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 10:49:21 +0200 Subject: [PATCH 44/53] Remove dbg Co-Authored-By: Thomas Campistron --- crates/meilisearch/src/routes/webhooks.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 86847c4bc..996377240 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -364,7 +364,7 @@ async fn post_webhook( } let mut webhooks = index_scheduler.webhooks(); - if dbg!(webhooks.webhooks.len() >= 20) { + if webhooks.webhooks.len() >= 20 { return Err(TooManyWebhooks.into()); } From 6cb22966447b347a15439959f904858c9a6881c2 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 11:10:48 +0200 Subject: [PATCH 45/53] Update tests --- crates/meilisearch/tests/auth/api_keys.rs | 2 +- crates/meilisearch/tests/auth/errors.rs | 2 +- crates/meilisearch/tests/tasks/webhook.rs | 48 +++++++++++++---------- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/crates/meilisearch/tests/auth/api_keys.rs b/crates/meilisearch/tests/auth/api_keys.rs index f16789add..8dca24ac4 100644 --- a/crates/meilisearch/tests/auth/api_keys.rs +++ b/crates/meilisearch/tests/auth/api_keys.rs @@ -421,7 +421,7 @@ async fn error_add_api_key_invalid_parameters_actions() { meili_snap::snapshot!(code, @"400 Bad Request"); meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r#" { - "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`, `webhooks.get`, `webhooks.update`", + "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`, `webhooks.get`, `webhooks.update`, `webhooks.delete`, `webhooks.create`, `webhooks.*`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" diff --git a/crates/meilisearch/tests/auth/errors.rs b/crates/meilisearch/tests/auth/errors.rs index 6d3369144..2a40f4d2b 100644 --- a/crates/meilisearch/tests/auth/errors.rs +++ b/crates/meilisearch/tests/auth/errors.rs @@ -93,7 +93,7 @@ async fn create_api_key_bad_actions() { snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(response), @r#" { - "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`, `webhooks.get`, `webhooks.update`", + "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`, `export`, `network.get`, `network.update`, `chatCompletions`, `chats.*`, `chats.get`, `chats.delete`, `chatsSettings.*`, `chatsSettings.get`, `chatsSettings.update`, `*.get`, `webhooks.get`, `webhooks.update`, `webhooks.delete`, `webhooks.create`, `webhooks.*`", "code": "invalid_api_key_actions", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions" diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 03f732f0d..268a37e18 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -265,7 +265,14 @@ async fn cli_with_dumps() { #[actix_web::test] async fn reserved_names() { - let server = Server::new().await; + let db_path = tempfile::tempdir().unwrap(); + let server = Server::new_with_options(Opt { + task_webhook_url: Some(Url::parse("https://example-cli.com/").unwrap()), + task_webhook_authorization_header: Some(String::from("Bearer a-secret-token")), + ..default_settings(db_path.path()) + }) + .await + .unwrap(); let (value, code) = server .patch_webhook(Uuid::nil().to_string(), json!({ "url": "http://localhost:8080" })) @@ -273,10 +280,10 @@ async fn reserved_names() { snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Cannot edit webhook `[uuid]`. The webhook defined from the command line cannot be modified using the API.", - "code": "reserved_webhook", + "message": "Webhook `[uuid]` is immutable. The webhook defined from the command line cannot be modified using the API.", + "code": "immutable_webhook", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#reserved_webhook" + "link": "https://docs.meilisearch.com/errors#immutable_webhook" } "#); @@ -284,10 +291,10 @@ async fn reserved_names() { snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Cannot edit webhook `[uuid]`. The webhook defined from the command line cannot be modified using the API.", - "code": "reserved_webhook", + "message": "Webhook `[uuid]` is immutable. The webhook defined from the command line cannot be modified using the API.", + "code": "immutable_webhook", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#reserved_webhook" + "link": "https://docs.meilisearch.com/errors#immutable_webhook" } "#); } @@ -335,7 +342,7 @@ async fn over_limits() { snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { - "message": "Too many headers for the webhook `[uuid]`. Please limit the number of headers to 200.", + "message": "Too many headers for the webhook `[uuid]`. Please limit the number of headers to 200. Hint: To remove an already defined header set its value to `null`", "code": "invalid_webhooks_headers", "type": "invalid_request", "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" @@ -387,12 +394,11 @@ async fn post_get_delete() { } #[actix_web::test] -async fn patch() { +async fn create_and_patch() { let server = Server::new().await; - let uuid = Uuid::new_v4().to_string(); let (value, code) = - server.patch_webhook(&uuid, json!({ "headers": { "authorization": "TOKEN" } })).await; + server.create_webhook(json!({ "headers": { "authorization": "TOKEN" } })).await; snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { @@ -403,9 +409,8 @@ async fn patch() { } "#); - let (value, code) = - server.patch_webhook(&uuid, json!({ "url": "https://example.com/hook" })).await; - snapshot!(code, @"200 OK"); + let (value, code) = server.create_webhook(json!({ "url": "https://example.com/hook" })).await; + snapshot!(code, @"201 Created"); snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" { "uuid": "[uuid]", @@ -415,6 +420,7 @@ async fn patch() { } "#); + let uuid = value.get("uuid").unwrap().as_str().unwrap(); let (value, code) = server.patch_webhook(&uuid, json!({ "headers": { "authorization": "TOKEN" } })).await; snapshot!(code, @"200 OK"); @@ -459,13 +465,15 @@ async fn patch() { "#); let (value, code) = server.patch_webhook(&uuid, json!({ "url": null })).await; - snapshot!(code, @"400 Bad Request"); - snapshot!(value, @r#" + snapshot!(code, @"200 OK"); + snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" { - "message": "The URL for the webhook `[uuid]` is missing.", - "code": "invalid_webhooks_url", - "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + "uuid": "81ccb94c-74cf-4d40-8070-492055804693", + "isEditable": true, + "url": "https://example.com/hook", + "headers": { + "authorization2": "TOKEN" + } } "#); } From a9c924b433623165859bfac15b55b3c0ed1f8c79 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 11:16:34 +0200 Subject: [PATCH 46/53] Turn url back into a setting --- crates/meilisearch/src/routes/webhooks.rs | 11 ++++++++--- crates/meilisearch/tests/tasks/webhook.rs | 12 +++++------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 996377240..9adad3284 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -61,9 +61,10 @@ pub fn configure(cfg: &mut web::ServiceConfig) { #[serde(rename_all = "camelCase")] #[schema(rename_all = "camelCase")] pub(super) struct WebhookSettings { + #[schema(value_type = Option, example = "https://your.site/on-tasks-completed")] #[deserr(default, error = DeserrJsonError)] #[serde(default)] - url: Option, + url: Setting, #[schema(value_type = Option>, example = json!({"Authorization":"Bearer a-secret-token"}))] #[deserr(default, error = DeserrJsonError)] #[serde(default)] @@ -241,7 +242,11 @@ fn patch_webhook_inner( ) -> Result { let Webhook { url: old_url, mut headers } = old_webhook; - let url = new_webhook.url.unwrap_or(old_url); + let url = match new_webhook.url { + Setting::Set(url) => url, + Setting::NotSet => old_url, + Setting::Reset => return Err(MissingUrl(uuid.to_owned())), + }; match new_webhook.headers { Setting::Set(new_headers) => { @@ -369,7 +374,7 @@ async fn post_webhook( } let webhook = Webhook { - url: webhook_settings.url.ok_or(MissingUrl(uuid))?, + url: webhook_settings.url.set().ok_or(MissingUrl(uuid))?, headers: webhook_settings .headers .set() diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 268a37e18..f457fb697 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -465,15 +465,13 @@ async fn create_and_patch() { "#); let (value, code) = server.patch_webhook(&uuid, json!({ "url": null })).await; - snapshot!(code, @"200 OK"); + snapshot!(code, @"400 Bad Request"); snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" { - "uuid": "81ccb94c-74cf-4d40-8070-492055804693", - "isEditable": true, - "url": "https://example.com/hook", - "headers": { - "authorization2": "TOKEN" - } + "message": "The URL for the webhook `[uuid]` is missing.", + "code": "invalid_webhooks_url", + "type": "invalid_request", + "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" } "#); } From 8b27dec25c9e5d68b9e42c7d54e55195dc36e866 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 11:19:21 +0200 Subject: [PATCH 47/53] Test that the cli webhook receives data --- crates/meilisearch/tests/tasks/webhook.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index f457fb697..41362566a 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -166,8 +166,6 @@ async fn single_receives_data() { #[actix_web::test] async fn multiple_receive_data() { - let server = Server::new().await; - let WebhookHandle { server_handle: handle1, url: url1, receiver: mut receiver1 } = create_webhook_server().await; let WebhookHandle { server_handle: handle2, url: url2, receiver: mut receiver2 } = @@ -175,7 +173,15 @@ async fn multiple_receive_data() { let WebhookHandle { server_handle: handle3, url: url3, receiver: mut receiver3 } = create_webhook_server().await; - for url in [url1, url2, url3] { + let db_path = tempfile::tempdir().unwrap(); + let server = Server::new_with_options(Opt { + task_webhook_url: Some(Url::parse(&url3).unwrap()), + ..default_settings(db_path.path()) + }) + .await + .unwrap(); + + for url in [url1, url2] { let (value, code) = server.create_webhook(json!({ "url": url })).await; snapshot!(code, @"201 Created"); snapshot!(json_string!(value, { ".uuid" => "[uuid]", ".url" => "[ignored]" }), @r#" From 4f6a48c32779863bb84afaa01ec841846f726e32 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 11:44:53 +0200 Subject: [PATCH 48/53] Stop storing the cli webhook in the db --- crates/dump/src/reader/mod.rs | 5 +++ crates/index-scheduler/src/insta_snapshot.rs | 2 ++ crates/index-scheduler/src/lib.rs | 32 ++++++++++++++++++-- crates/index-scheduler/src/scheduler/mod.rs | 4 +-- crates/index-scheduler/src/test_utils.rs | 2 ++ crates/meilisearch/src/lib.rs | 29 ++---------------- 6 files changed, 41 insertions(+), 33 deletions(-) diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index 844aadc99..129b01f46 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -458,6 +458,11 @@ pub(crate) mod test { // webhooks + // Important note: You might be surprised to see the cli webhook in the dump, as it's not supposed to be saved. + // This is because the dump comes from a version that did save it. + // It's no longer the case, but that's not what this test is about. + // It's ok to see the cli webhook disappear when this test gets updated. + let webhooks = dump.webhooks().unwrap(); insta::assert_json_snapshot!(webhooks, @r#" { diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index f3431dd33..addd87be8 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -30,6 +30,8 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { index_mapper, features: _, + cli_webhook_url: _, + cli_webhook_authorization: _, cached_webhooks: _, test_breakpoint_sdr: _, planned_failures: _, diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index c18efeaf2..d04b8f9e2 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -73,6 +73,7 @@ use queue::Queue; use roaring::RoaringBitmap; use scheduler::Scheduler; use time::OffsetDateTime; +use uuid::Uuid; use versioning::Versioning; use crate::index_mapper::IndexMapper; @@ -107,6 +108,10 @@ pub struct IndexSchedulerOptions { pub snapshots_path: PathBuf, /// The path to the folder containing the dumps. pub dumps_path: PathBuf, + /// The webhook url that was set by the CLI. + pub cli_webhook_url: Option, + /// The Authorization header to send to the webhook URL that was set by the CLI. + pub cli_webhook_authorization: Option, /// The maximum size, in bytes, of the task index. pub task_db_size: usize, /// The size, in bytes, with which a meilisearch index is opened the first time of each meilisearch index. @@ -179,6 +184,10 @@ pub struct IndexScheduler { /// A database to store single-keyed data that is persisted across restarts. persisted: Database, + /// The webhook url that was set by the CLI. + cli_webhook_url: Option, + /// The Authorization header to send to the webhook URL that was set by the CLI. + cli_webhook_authorization: Option, /// Webhook cached_webhooks: Arc>, @@ -221,7 +230,11 @@ impl IndexScheduler { cleanup_enabled: self.cleanup_enabled, experimental_no_edition_2024_for_dumps: self.experimental_no_edition_2024_for_dumps, persisted: self.persisted, + + cli_webhook_url: self.cli_webhook_url.clone(), + cli_webhook_authorization: self.cli_webhook_authorization.clone(), cached_webhooks: self.cached_webhooks.clone(), + embedders: self.embedders.clone(), #[cfg(test)] test_breakpoint_sdr: self.test_breakpoint_sdr.clone(), @@ -299,7 +312,8 @@ impl IndexScheduler { let persisted = env.create_database(&mut wtxn, Some(db_name::PERSISTED))?; let webhooks_db = persisted.remap_data_type::>(); - let webhooks = webhooks_db.get(&wtxn, db_keys::WEBHOOKS)?.unwrap_or_default(); + let mut webhooks = webhooks_db.get(&wtxn, db_keys::WEBHOOKS)?.unwrap_or_default(); + webhooks.webhooks.remove(&Uuid::nil()); // remove the cli webhook if it was saved by mistake wtxn.commit()?; @@ -317,7 +331,10 @@ impl IndexScheduler { .indexer_config .experimental_no_edition_2024_for_dumps, persisted, + cached_webhooks: Arc::new(RwLock::new(webhooks)), + cli_webhook_url: options.cli_webhook_url, + cli_webhook_authorization: options.cli_webhook_authorization, embedders: Default::default(), @@ -869,9 +886,10 @@ impl IndexScheduler { self.features.network() } - pub fn put_webhooks(&self, webhooks: Webhooks) -> Result<()> { + pub fn put_webhooks(&self, mut webhooks: Webhooks) -> Result<()> { let mut wtxn = self.env.write_txn()?; let webhooks_db = self.persisted.remap_data_type::>(); + webhooks.webhooks.remove(&Uuid::nil()); // the cli webhook must not be saved webhooks_db.put(&mut wtxn, db_keys::WEBHOOKS, &webhooks)?; wtxn.commit()?; *self.cached_webhooks.write().unwrap() = webhooks; @@ -880,7 +898,15 @@ impl IndexScheduler { pub fn webhooks(&self) -> Webhooks { let webhooks = self.cached_webhooks.read().unwrap_or_else(|poisoned| poisoned.into_inner()); - Webhooks::clone(&*webhooks) + let mut webhooks = Webhooks::clone(&*webhooks); + if let Some(url) = self.cli_webhook_url.as_ref().cloned() { + let mut headers = BTreeMap::new(); + if let Some(auth) = self.cli_webhook_authorization.as_ref().cloned() { + headers.insert(String::from("Authorization"), auth); + } + webhooks.webhooks.insert(Uuid::nil(), Webhook { url, headers }); + } + webhooks } pub fn embedders( diff --git a/crates/index-scheduler/src/scheduler/mod.rs b/crates/index-scheduler/src/scheduler/mod.rs index b5acf7582..bbfc4e058 100644 --- a/crates/index-scheduler/src/scheduler/mod.rs +++ b/crates/index-scheduler/src/scheduler/mod.rs @@ -26,7 +26,6 @@ use meilisearch_types::error::ResponseError; use meilisearch_types::heed::{Env, WithoutTls}; use meilisearch_types::milli; use meilisearch_types::tasks::Status; -use meilisearch_types::webhooks::Webhooks; use process_batch::ProcessBatchInfo; use rayon::current_num_threads; use rayon::iter::{IntoParallelIterator, ParallelIterator}; @@ -448,9 +447,8 @@ impl IndexScheduler { })?; // We shouldn't crash the tick function if we can't send data to the webhooks - let webhooks = self.cached_webhooks.read().unwrap_or_else(|p| p.into_inner()); + let webhooks = self.webhooks(); if !webhooks.webhooks.is_empty() { - let webhooks = Webhooks::clone(&*webhooks); let cloned_index_scheduler = self.private_clone(); std::thread::spawn(move || { if let Err(e) = cloned_index_scheduler.notify_webhooks(webhooks, &ids) { diff --git a/crates/index-scheduler/src/test_utils.rs b/crates/index-scheduler/src/test_utils.rs index b7d69b5b3..36de0ed9e 100644 --- a/crates/index-scheduler/src/test_utils.rs +++ b/crates/index-scheduler/src/test_utils.rs @@ -98,6 +98,8 @@ impl IndexScheduler { indexes_path: tempdir.path().join("indexes"), snapshots_path: tempdir.path().join("snapshots"), dumps_path: tempdir.path().join("dumps"), + cli_webhook_url: None, + cli_webhook_authorization: None, task_db_size: 1000 * 1000 * 10, // 10 MB, we don't use MiB on purpose. index_base_map_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose. enable_mdb_writemap: false, diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 613268936..533f0327f 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -13,7 +13,6 @@ pub mod routes; pub mod search; pub mod search_queue; -use std::collections::BTreeMap; use std::fs::File; use std::io::{BufReader, BufWriter}; use std::path::Path; @@ -49,14 +48,12 @@ use meilisearch_types::tasks::KindWithContent; use meilisearch_types::versioning::{ create_current_version_file, get_version, VersionFileError, VERSION_MINOR, VERSION_PATCH, }; -use meilisearch_types::webhooks::Webhook; use meilisearch_types::{compression, heed, milli, VERSION_FILE_NAME}; pub use option::Opt; use option::ScheduleSnapshot; use search_queue::SearchQueue; use tracing::{error, info_span}; use tracing_subscriber::filter::Targets; -use uuid::Uuid; use crate::error::MeilisearchHttpError; @@ -226,6 +223,8 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< indexes_path: opt.db_path.join("indexes"), snapshots_path: opt.snapshot_dir.clone(), dumps_path: opt.dump_dir.clone(), + cli_webhook_url: opt.task_webhook_url.as_ref().map(|url| url.to_string()), + cli_webhook_authorization: opt.task_webhook_authorization_header.clone(), task_db_size: opt.max_task_db_size.as_u64() as usize, index_base_map_size: opt.max_index_size.as_u64() as usize, enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage, @@ -328,30 +327,6 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc, Arc< .unwrap(); } - // We set the webhook url - let cli_webhook = opt.task_webhook_url.as_ref().map(|u| Webhook { - url: u.to_string(), - headers: { - let mut headers = BTreeMap::new(); - if let Some(value) = &opt.task_webhook_authorization_header { - headers.insert(String::from("Authorization"), value.to_string()); - } - headers - }, - }); - let mut webhooks = index_scheduler.webhooks(); - if webhooks.webhooks.get(&Uuid::nil()) != cli_webhook.as_ref() { - match cli_webhook { - Some(webhook) => { - webhooks.webhooks.insert(Uuid::nil(), webhook); - } - None => { - webhooks.webhooks.remove(&Uuid::nil()); - } - } - index_scheduler.put_webhooks(webhooks)?; - } - Ok((index_scheduler, auth_controller)) } From 2b5b41790edfb72530bc8db00cf604eda889fef0 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 5 Aug 2025 16:21:14 +0200 Subject: [PATCH 49/53] update the dump so it doesn't contains the null-uuid webhook --- crates/dump/src/reader/mod.rs | 12 ------------ .../dump/tests/assets/v6-with-webhooks.dump | Bin 1389 -> 1391 bytes 2 files changed, 12 deletions(-) diff --git a/crates/dump/src/reader/mod.rs b/crates/dump/src/reader/mod.rs index 129b01f46..da55bb4a8 100644 --- a/crates/dump/src/reader/mod.rs +++ b/crates/dump/src/reader/mod.rs @@ -457,22 +457,10 @@ pub(crate) mod test { "); // webhooks - - // Important note: You might be surprised to see the cli webhook in the dump, as it's not supposed to be saved. - // This is because the dump comes from a version that did save it. - // It's no longer the case, but that's not what this test is about. - // It's ok to see the cli webhook disappear when this test gets updated. - let webhooks = dump.webhooks().unwrap(); insta::assert_json_snapshot!(webhooks, @r#" { "webhooks": { - "00000000-0000-0000-0000-000000000000": { - "url": "https://defined-in-dump.com/", - "headers": { - "Authorization": "Bearer defined in dump" - } - }, "627ea538-733d-4545-8d2d-03526eb381ce": { "url": "https://example.com/authorization-less", "headers": {} diff --git a/crates/dump/tests/assets/v6-with-webhooks.dump b/crates/dump/tests/assets/v6-with-webhooks.dump index c8f9649d88429ab8c16f3375f94a6c4474d3e79f..955c2a63dc79b51bfc121b30b0f5843480c75361 100644 GIT binary patch literal 1391 zcmV-#1(5n5iwFRO5t3*C1MOPfZ`(Ey&vX9@O^<~si(itQw=QXcp};nDZ68()1w|ff zEwbcE)Y{C*|GuMSCyldw0k*Ss`oYi#b<)vyygMFw(e!BVh~qewGU(6AaE_T6&T9lz z#zKhUI;M;!@+gq|@7mE(*E-c;SBeJytnqLg9g<={d3OQsQ*JCntq0SUe{-B^m|)Zd*OldNQAgu{~}>q_Ak%~-c zZeBOVU&;pJjqbp4g1dWTq2&Hxu9A zYo86TvBkIE{`C6gkD=#gxy*Q+abh!-5}|WTl!TPnoLdSE8)0^no0}Jn@w~Qre{u2b zsWrA+MAI`kA8fze7ms`FpK-bE{{>^|X#a=6Dm2jQrdX~G=Uody;R06&EvQD%1KZS# zz|Fy5R1PYG7E!4Mx`FT%tm;Y^{4!9!`1yz6HJsz~t}ysmq?#%!a$+r-pP-f!G;K|= zky)TBp~@n$(w;S%*N1Rm8U8j2i^{+k=o_E=d-$?M-LxN-3y-U=EGMCEKshjdOUPm- zNt_Z+-^R0J7UM6HX)n2duIeLZa6ADAQplE}tc7fW6sa`_U zJD(^xL1%;)-vy_2PlJ80htJd%zOd}_ve0*k^MzRy)eUJ{W6@vQ84($!L>Q(>o@-5% z5|-*D6?&Flm*(?b2{K7}CLSipHR-K_#Bqh}Oi)xP(nT+@K<_QM2p@AfO1BBFd%|ag;&}SR6!MsZEnTZKz$y40*6h# z@AQ|ulC+fUfr9QS?~=3j!&Ft~5r%x?U)h#Up7B|RZkeQzV*s_YgeVioB+X==YRi<8 z=DKt|=abq;>!E3~W5Z5poY4o0X(0X9qW10(O}(~=zET%y!8!DEyUPh5pZ}$tiZTB? z1fuBzobn#`>GD6y*!KJ{kz@XM5Mcc17R&dT-wg5NeS&_~R$iWN?uA4_&`Q)a3fjMijsEsPuO{33xG@>ag&Z$bnM@;vl4i7u$ zac}&`hu7_2N)eCte+cM2%Tl8?;UecmWQG!Dj3Kj(N(CvSDx*)`#}WHOG|iRMbJ%AE zz~k3{ctVqL{pS$y{=Z+5{g>%qjO%{~!Il00(Lo1$*$BQj{@>8QOw)1y_aNBRf9+5E z_v)X=Lf)AFjr;$H!D`s$izi|42j2OcVJ)bS3t!!%09=R3SJ-x10~qkTe(rqP7XL0056E(^dcg literal 1389 zcmV-z1(Nz7iwFP!00000|Ls~)Z`(Ey_H%!QrpLl`NQx5WyroGC3MQPdwcIj3zwl?SBAd%IV4f z(EhQEd;1>%OZy-F)j|uSKbxv72D=8DlO{qu#QqWW$H4*c=;J@Wf2rX^o10OKUxVG_ zUi;&{HT&~eM!o$H0G*|&G+L8LWDyamVMG~Y$T(%O0?C<5*`UV{3-~;PI-f%6R6#5# z*VfQ~Ff6K%&FiXlaMc1PcW0Gu z{VZvoUA$;Z?eeO;y00soJ(x&d94!^R&HAs;{|CW5T+IL^eZrMR z$XC8r@N#X9lxsL~Zq^(h9T~8B35Mh)@kQbY%ZVfO&IyLyj>esbC#{=Qb^flxf%;0(@3J-gI57FFPXz zG|i2{_)er`8C5(bm{L{?9=zDU$}R?xLf~x vqbZI3pnW!TcS@#0NH2+{SKdtS5>2@n%bq;x(W6I?9}@onHxdkx05|{u+j_X9 From 3f1e172c6f8adf9db2d782e043032c0848b63d45 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 5 Aug 2025 16:47:35 +0200 Subject: [PATCH 50/53] fix race condition: take the rtxn before entering the thread so we're sure we won't try to retrieve deleted tasks --- crates/index-scheduler/src/lib.rs | 65 ++++++++++++--------- crates/index-scheduler/src/scheduler/mod.rs | 11 +--- 2 files changed, 40 insertions(+), 36 deletions(-) diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index d04b8f9e2..419e6f21e 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -784,7 +784,7 @@ impl IndexScheduler { } /// Once the tasks changes have been committed we must send all the tasks that were updated to our webhooks - fn notify_webhooks(&self, webhooks: Webhooks, updated: &RoaringBitmap) -> Result<()> { + fn notify_webhooks(&self, updated: RoaringBitmap) { struct TaskReader<'a, 'b> { rtxn: &'a RoTxn<'a>, index_scheduler: &'a IndexScheduler, @@ -829,33 +829,46 @@ impl IndexScheduler { } } - let rtxn = self.env.read_txn()?; - - for (uuid, Webhook { url, headers }) in webhooks.webhooks.iter() { - let task_reader = TaskReader { - rtxn: &rtxn, - index_scheduler: self, - tasks: &mut updated.into_iter(), - buffer: Vec::with_capacity(page_size::get()), - written: 0, - }; - - let reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); - - let mut request = ureq::post(url) - .timeout(Duration::from_secs(30)) - .set("Content-Encoding", "gzip") - .set("Content-Type", "application/x-ndjson"); - for (header_name, header_value) in headers.iter() { - request = request.set(header_name, header_value); - } - - if let Err(e) = request.send(reader) { - tracing::error!("While sending data to the webhook {uuid}: {e}"); - } + let webhooks = self.webhooks(); + if webhooks.webhooks.is_empty() { + return; } + let this = self.private_clone(); + // We must take the RoTxn before entering the thread::spawn otherwise another batch may be + // processed before we had the time to take our txn. + let rtxn = match self.env.clone().static_read_txn() { + Ok(rtxn) => rtxn, + Err(e) => { + tracing::error!("Couldn't get an rtxn to notify the webhook: {e}"); + return; + } + }; - Ok(()) + std::thread::spawn(move || { + for (uuid, Webhook { url, headers }) in webhooks.webhooks.iter() { + let task_reader = TaskReader { + rtxn: &rtxn, + index_scheduler: &this, + tasks: &mut updated.iter(), + buffer: Vec::with_capacity(page_size::get()), + written: 0, + }; + + let reader = GzEncoder::new(BufReader::new(task_reader), Compression::default()); + + let mut request = ureq::post(url) + .timeout(Duration::from_secs(30)) + .set("Content-Encoding", "gzip") + .set("Content-Type", "application/x-ndjson"); + for (header_name, header_value) in headers.iter() { + request = request.set(header_name, header_value); + } + + if let Err(e) = request.send(reader) { + tracing::error!("While sending data to the webhook {uuid}: {e}"); + } + } + }); } pub fn index_stats(&self, index_uid: &str) -> Result { diff --git a/crates/index-scheduler/src/scheduler/mod.rs b/crates/index-scheduler/src/scheduler/mod.rs index bbfc4e058..b2bb90c0b 100644 --- a/crates/index-scheduler/src/scheduler/mod.rs +++ b/crates/index-scheduler/src/scheduler/mod.rs @@ -446,16 +446,7 @@ impl IndexScheduler { Ok(()) })?; - // We shouldn't crash the tick function if we can't send data to the webhooks - let webhooks = self.webhooks(); - if !webhooks.webhooks.is_empty() { - let cloned_index_scheduler = self.private_clone(); - std::thread::spawn(move || { - if let Err(e) = cloned_index_scheduler.notify_webhooks(webhooks, &ids) { - tracing::error!("Failure to notify webhooks: {e}"); - } - }); - } + self.notify_webhooks(ids); #[cfg(test)] self.breakpoint(crate::test_utils::Breakpoint::AfterProcessing); From b5158e1e8362dd5a6ea5bb0b751ec356e44f3f54 Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 16:49:54 +0200 Subject: [PATCH 51/53] Fix cli webhook getting stored in dumps --- crates/dump/src/writer.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crates/dump/src/writer.rs b/crates/dump/src/writer.rs index 84a76e483..448896e06 100644 --- a/crates/dump/src/writer.rs +++ b/crates/dump/src/writer.rs @@ -75,10 +75,11 @@ impl DumpWriter { Ok(std::fs::write(self.dir.path().join("network.json"), serde_json::to_string(&network)?)?) } - pub fn create_webhooks(&self, webhooks: Webhooks) -> Result<()> { + pub fn create_webhooks(&self, mut webhooks: Webhooks) -> Result<()> { if webhooks == Webhooks::default() { return Ok(()); } + webhooks.webhooks.remove(&Uuid::nil()); // Don't store the cli webhook Ok(std::fs::write( self.dir.path().join("webhooks.json"), serde_json::to_string(&webhooks)?, From 1ff6da63e8d1236e5332067b4729a2bd513f885f Mon Sep 17 00:00:00 2001 From: Mubelotix Date: Tue, 5 Aug 2025 16:58:21 +0200 Subject: [PATCH 52/53] Make errors singular --- crates/meilisearch-types/src/error.rs | 4 ++-- crates/meilisearch/src/routes/webhooks.rs | 16 +++++++-------- crates/meilisearch/tests/tasks/webhook.rs | 24 +++++++++++------------ 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/crates/meilisearch-types/src/error.rs b/crates/meilisearch-types/src/error.rs index bb486726a..4360d947b 100644 --- a/crates/meilisearch-types/src/error.rs +++ b/crates/meilisearch-types/src/error.rs @@ -421,8 +421,8 @@ InvalidChatCompletionSearchIndexUidParamPrompt , InvalidRequest , BAD_REQU InvalidChatCompletionPreQueryPrompt , InvalidRequest , BAD_REQUEST ; // Webhooks InvalidWebhooks , InvalidRequest , BAD_REQUEST ; -InvalidWebhooksUrl , InvalidRequest , BAD_REQUEST ; -InvalidWebhooksHeaders , InvalidRequest , BAD_REQUEST ; +InvalidWebhookUrl , InvalidRequest , BAD_REQUEST ; +InvalidWebhookHeaders , InvalidRequest , BAD_REQUEST ; ImmutableWebhook , InvalidRequest , BAD_REQUEST ; InvalidWebhookUuid , InvalidRequest , BAD_REQUEST ; WebhookNotFound , InvalidRequest , NOT_FOUND ; diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index 9adad3284..e3547996a 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -13,7 +13,7 @@ use deserr::{DeserializeError, Deserr, ValuePointerRef}; use index_scheduler::IndexScheduler; use meilisearch_types::deserr::{immutable_field_error, DeserrJsonError}; use meilisearch_types::error::deserr_codes::{ - BadRequest, InvalidWebhooksHeaders, InvalidWebhooksUrl, + BadRequest, InvalidWebhookHeaders, InvalidWebhookUrl, }; use meilisearch_types::error::{Code, ErrorCode, ResponseError}; use meilisearch_types::keys::actions; @@ -62,11 +62,11 @@ pub fn configure(cfg: &mut web::ServiceConfig) { #[schema(rename_all = "camelCase")] pub(super) struct WebhookSettings { #[schema(value_type = Option, example = "https://your.site/on-tasks-completed")] - #[deserr(default, error = DeserrJsonError)] + #[deserr(default, error = DeserrJsonError)] #[serde(default)] url: Setting, #[schema(value_type = Option>, example = json!({"Authorization":"Bearer a-secret-token"}))] - #[deserr(default, error = DeserrJsonError)] + #[deserr(default, error = DeserrJsonError)] #[serde(default)] headers: Setting>>, } @@ -222,14 +222,14 @@ enum WebhooksError { impl ErrorCode for WebhooksError { fn error_code(&self) -> meilisearch_types::error::Code { match self { - MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhooksUrl, + MissingUrl(_) => meilisearch_types::error::Code::InvalidWebhookUrl, TooManyWebhooks => meilisearch_types::error::Code::InvalidWebhooks, - TooManyHeaders(_) => meilisearch_types::error::Code::InvalidWebhooksHeaders, + TooManyHeaders(_) => meilisearch_types::error::Code::InvalidWebhookHeaders, ImmutableWebhook(_) => meilisearch_types::error::Code::ImmutableWebhook, WebhookNotFound(_) => meilisearch_types::error::Code::WebhookNotFound, - InvalidHeaderName(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, - InvalidHeaderValue(_, _) => meilisearch_types::error::Code::InvalidWebhooksHeaders, - InvalidUrl(_, _) => meilisearch_types::error::Code::InvalidWebhooksUrl, + InvalidHeaderName(_, _) => meilisearch_types::error::Code::InvalidWebhookHeaders, + InvalidHeaderValue(_, _) => meilisearch_types::error::Code::InvalidWebhookHeaders, + InvalidUrl(_, _) => meilisearch_types::error::Code::InvalidWebhookUrl, InvalidUuid(_) => meilisearch_types::error::Code::InvalidWebhookUuid, } } diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index 41362566a..a3be37a4e 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -349,9 +349,9 @@ async fn over_limits() { snapshot!(value, @r#" { "message": "Too many headers for the webhook `[uuid]`. Please limit the number of headers to 200. Hint: To remove an already defined header set its value to `null`", - "code": "invalid_webhooks_headers", + "code": "invalid_webhook_headers", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" + "link": "https://docs.meilisearch.com/errors#invalid_webhook_headers" } "#); } @@ -409,9 +409,9 @@ async fn create_and_patch() { snapshot!(value, @r#" { "message": "The URL for the webhook `[uuid]` is missing.", - "code": "invalid_webhooks_url", + "code": "invalid_webhook_url", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + "link": "https://docs.meilisearch.com/errors#invalid_webhook_url" } "#); @@ -475,9 +475,9 @@ async fn create_and_patch() { snapshot!(json_string!(value, { ".uuid" => "[uuid]" }), @r#" { "message": "The URL for the webhook `[uuid]` is missing.", - "code": "invalid_webhooks_url", + "code": "invalid_webhook_url", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + "link": "https://docs.meilisearch.com/errors#invalid_webhook_url" } "#); } @@ -492,9 +492,9 @@ async fn invalid_url_and_headers() { snapshot!(value, @r#" { "message": "Invalid URL `not-a-valid-url`: relative URL without a base", - "code": "invalid_webhooks_url", + "code": "invalid_webhook_url", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_url" + "link": "https://docs.meilisearch.com/errors#invalid_webhook_url" } "#); @@ -509,9 +509,9 @@ async fn invalid_url_and_headers() { snapshot!(value, @r#" { "message": "Invalid header name `invalid header name`: invalid HTTP header name", - "code": "invalid_webhooks_headers", + "code": "invalid_webhook_headers", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" + "link": "https://docs.meilisearch.com/errors#invalid_webhook_headers" } "#); @@ -526,9 +526,9 @@ async fn invalid_url_and_headers() { snapshot!(value, @r#" { "message": "Invalid header value `authorization`: failed to parse header value", - "code": "invalid_webhooks_headers", + "code": "invalid_webhook_headers", "type": "invalid_request", - "link": "https://docs.meilisearch.com/errors#invalid_webhooks_headers" + "link": "https://docs.meilisearch.com/errors#invalid_webhook_headers" } "#); } From 899be9c3ffa6935f56d24026a0050357bc23bfc4 Mon Sep 17 00:00:00 2001 From: Tamo Date: Tue, 5 Aug 2025 18:55:32 +0200 Subject: [PATCH 53/53] make sure we NEVER ever write the cli defined webhook to the database or dumps --- crates/dump/src/reader/v6/mod.rs | 2 +- crates/dump/src/writer.rs | 8 +- crates/index-scheduler/src/insta_snapshot.rs | 4 +- crates/index-scheduler/src/lib.rs | 128 +++++++++++++----- .../src/scheduler/process_dump_creation.rs | 2 +- crates/meilisearch-types/src/webhooks.rs | 13 +- crates/meilisearch/src/lib.rs | 2 +- crates/meilisearch/src/routes/webhooks.rs | 30 ++-- crates/meilisearch/tests/tasks/webhook.rs | 4 +- 9 files changed, 130 insertions(+), 63 deletions(-) diff --git a/crates/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs index d8ce430f9..9bc4b33c5 100644 --- a/crates/dump/src/reader/v6/mod.rs +++ b/crates/dump/src/reader/v6/mod.rs @@ -25,7 +25,7 @@ pub type Key = meilisearch_types::keys::Key; pub type ChatCompletionSettings = meilisearch_types::features::ChatCompletionSettings; pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures; pub type Network = meilisearch_types::features::Network; -pub type Webhooks = meilisearch_types::webhooks::Webhooks; +pub type Webhooks = meilisearch_types::webhooks::WebhooksDumpView; // ===== Other types to clarify the code of the compat module // everything related to the tasks diff --git a/crates/dump/src/writer.rs b/crates/dump/src/writer.rs index 448896e06..1d41b6aa5 100644 --- a/crates/dump/src/writer.rs +++ b/crates/dump/src/writer.rs @@ -8,7 +8,7 @@ use meilisearch_types::batches::Batch; use meilisearch_types::features::{ChatCompletionSettings, Network, RuntimeTogglableFeatures}; use meilisearch_types::keys::Key; use meilisearch_types::settings::{Checked, Settings}; -use meilisearch_types::webhooks::Webhooks; +use meilisearch_types::webhooks::WebhooksDumpView; use serde_json::{Map, Value}; use tempfile::TempDir; use time::OffsetDateTime; @@ -75,11 +75,7 @@ impl DumpWriter { Ok(std::fs::write(self.dir.path().join("network.json"), serde_json::to_string(&network)?)?) } - pub fn create_webhooks(&self, mut webhooks: Webhooks) -> Result<()> { - if webhooks == Webhooks::default() { - return Ok(()); - } - webhooks.webhooks.remove(&Uuid::nil()); // Don't store the cli webhook + pub fn create_webhooks(&self, webhooks: WebhooksDumpView) -> Result<()> { Ok(std::fs::write( self.dir.path().join("webhooks.json"), serde_json::to_string(&webhooks)?, diff --git a/crates/index-scheduler/src/insta_snapshot.rs b/crates/index-scheduler/src/insta_snapshot.rs index addd87be8..cb804d9b4 100644 --- a/crates/index-scheduler/src/insta_snapshot.rs +++ b/crates/index-scheduler/src/insta_snapshot.rs @@ -30,9 +30,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String { index_mapper, features: _, - cli_webhook_url: _, - cli_webhook_authorization: _, - cached_webhooks: _, + webhooks: _, test_breakpoint_sdr: _, planned_failures: _, run_loop_iteration: _, diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 419e6f21e..6ad7a8397 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -65,13 +65,14 @@ use meilisearch_types::milli::vector::{ use meilisearch_types::milli::{self, Index}; use meilisearch_types::task_view::TaskView; use meilisearch_types::tasks::{KindWithContent, Task}; -use meilisearch_types::webhooks::{Webhook, Webhooks}; +use meilisearch_types::webhooks::{Webhook, WebhooksDumpView, WebhooksView}; use milli::vector::db::IndexEmbeddingConfig; use processing::ProcessingTasks; pub use queue::Query; use queue::Queue; use roaring::RoaringBitmap; use scheduler::Scheduler; +use serde::{Deserialize, Serialize}; use time::OffsetDateTime; use uuid::Uuid; use versioning::Versioning; @@ -184,12 +185,8 @@ pub struct IndexScheduler { /// A database to store single-keyed data that is persisted across restarts. persisted: Database, - /// The webhook url that was set by the CLI. - cli_webhook_url: Option, - /// The Authorization header to send to the webhook URL that was set by the CLI. - cli_webhook_authorization: Option, - /// Webhook - cached_webhooks: Arc>, + /// Webhook, loaded and stored in the `persisted` database + webhooks: Arc, /// A map to retrieve the runtime representation of an embedder depending on its configuration. /// @@ -231,10 +228,7 @@ impl IndexScheduler { experimental_no_edition_2024_for_dumps: self.experimental_no_edition_2024_for_dumps, persisted: self.persisted, - cli_webhook_url: self.cli_webhook_url.clone(), - cli_webhook_authorization: self.cli_webhook_authorization.clone(), - cached_webhooks: self.cached_webhooks.clone(), - + webhooks: self.webhooks.clone(), embedders: self.embedders.clone(), #[cfg(test)] test_breakpoint_sdr: self.test_breakpoint_sdr.clone(), @@ -313,7 +307,8 @@ impl IndexScheduler { let persisted = env.create_database(&mut wtxn, Some(db_name::PERSISTED))?; let webhooks_db = persisted.remap_data_type::>(); let mut webhooks = webhooks_db.get(&wtxn, db_keys::WEBHOOKS)?.unwrap_or_default(); - webhooks.webhooks.remove(&Uuid::nil()); // remove the cli webhook if it was saved by mistake + webhooks + .with_cli(options.cli_webhook_url.clone(), options.cli_webhook_authorization.clone()); wtxn.commit()?; @@ -331,11 +326,7 @@ impl IndexScheduler { .indexer_config .experimental_no_edition_2024_for_dumps, persisted, - - cached_webhooks: Arc::new(RwLock::new(webhooks)), - cli_webhook_url: options.cli_webhook_url, - cli_webhook_authorization: options.cli_webhook_authorization, - + webhooks: Arc::new(webhooks), embedders: Default::default(), #[cfg(test)] @@ -829,8 +820,8 @@ impl IndexScheduler { } } - let webhooks = self.webhooks(); - if webhooks.webhooks.is_empty() { + let webhooks = self.webhooks.get_all(); + if webhooks.is_empty() { return; } let this = self.private_clone(); @@ -845,7 +836,7 @@ impl IndexScheduler { }; std::thread::spawn(move || { - for (uuid, Webhook { url, headers }) in webhooks.webhooks.iter() { + for (uuid, Webhook { url, headers }) in webhooks.iter() { let task_reader = TaskReader { rtxn: &rtxn, index_scheduler: &this, @@ -899,27 +890,27 @@ impl IndexScheduler { self.features.network() } - pub fn put_webhooks(&self, mut webhooks: Webhooks) -> Result<()> { + pub fn update_runtime_webhooks(&self, runtime: RuntimeWebhooks) -> Result<()> { + let webhooks = Webhooks::from_runtime(runtime); let mut wtxn = self.env.write_txn()?; let webhooks_db = self.persisted.remap_data_type::>(); - webhooks.webhooks.remove(&Uuid::nil()); // the cli webhook must not be saved webhooks_db.put(&mut wtxn, db_keys::WEBHOOKS, &webhooks)?; wtxn.commit()?; - *self.cached_webhooks.write().unwrap() = webhooks; + self.webhooks.update_runtime(webhooks.into_runtime()); Ok(()) } - pub fn webhooks(&self) -> Webhooks { - let webhooks = self.cached_webhooks.read().unwrap_or_else(|poisoned| poisoned.into_inner()); - let mut webhooks = Webhooks::clone(&*webhooks); - if let Some(url) = self.cli_webhook_url.as_ref().cloned() { - let mut headers = BTreeMap::new(); - if let Some(auth) = self.cli_webhook_authorization.as_ref().cloned() { - headers.insert(String::from("Authorization"), auth); - } - webhooks.webhooks.insert(Uuid::nil(), Webhook { url, headers }); - } - webhooks + pub fn webhooks_dump_view(&self) -> WebhooksDumpView { + // We must not dump the cli api key + WebhooksDumpView { webhooks: self.webhooks.get_runtime() } + } + + pub fn webhooks_view(&self) -> WebhooksView { + WebhooksView { webhooks: self.webhooks.get_all() } + } + + pub fn retrieve_runtime_webhooks(&self) -> RuntimeWebhooks { + self.webhooks.get_runtime() } pub fn embedders( @@ -1050,3 +1041,72 @@ pub struct IndexStats { /// Internal stats computed from the index. pub inner_stats: index_mapper::IndexStats, } + +/// These structure are not meant to be exposed to the end user, if needed, use the meilisearch-types::webhooks structure instead. +/// /!\ Everytime you deserialize this structure you should fill the cli_webhook later on with the `with_cli` method. /!\ +#[derive(Debug, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +struct Webhooks { + // The cli webhook should *never* be stored in a database. + // It represent a state that only exists for this execution of meilisearch + #[serde(skip)] + pub cli: Option, + + #[serde(default)] + pub runtime: RwLock, +} + +type RuntimeWebhooks = BTreeMap; + +impl Webhooks { + pub fn with_cli(&mut self, url: Option, auth: Option) { + if let Some(url) = url { + let webhook = CliWebhook { url, auth }; + self.cli = Some(webhook); + } + } + + pub fn from_runtime(webhooks: RuntimeWebhooks) -> Self { + Self { cli: None, runtime: RwLock::new(webhooks) } + } + + pub fn into_runtime(self) -> RuntimeWebhooks { + // safe because we own self and it cannot be cloned + self.runtime.into_inner().unwrap() + } + + pub fn update_runtime(&self, webhooks: RuntimeWebhooks) { + *self.runtime.write().unwrap() = webhooks; + } + + /// Returns all the webhooks in an unified view. The cli webhook is represented with an uuid set to 0 + pub fn get_all(&self) -> BTreeMap { + self.cli + .as_ref() + .map(|wh| (Uuid::nil(), Webhook::from(wh))) + .into_iter() + .chain(self.runtime.read().unwrap().iter().map(|(uuid, wh)| (*uuid, wh.clone()))) + .collect() + } + + /// Returns all the runtime webhooks. + pub fn get_runtime(&self) -> BTreeMap { + self.runtime.read().unwrap().iter().map(|(uuid, wh)| (*uuid, wh.clone())).collect() + } +} + +#[derive(Debug, Serialize, Deserialize, Default, Clone, PartialEq)] +struct CliWebhook { + pub url: String, + pub auth: Option, +} + +impl From<&CliWebhook> for Webhook { + fn from(webhook: &CliWebhook) -> Self { + let mut headers = BTreeMap::new(); + if let Some(ref auth) = webhook.auth { + headers.insert("Authorization".to_string(), auth.to_string()); + } + Self { url: webhook.url.to_string(), headers } + } +} diff --git a/crates/index-scheduler/src/scheduler/process_dump_creation.rs b/crates/index-scheduler/src/scheduler/process_dump_creation.rs index 8f47cbd0c..4f3ec0fdd 100644 --- a/crates/index-scheduler/src/scheduler/process_dump_creation.rs +++ b/crates/index-scheduler/src/scheduler/process_dump_creation.rs @@ -272,7 +272,7 @@ impl IndexScheduler { // 7. Dump the webhooks progress.update_progress(DumpCreationProgress::DumpTheWebhooks); - let webhooks = self.webhooks(); + let webhooks = self.webhooks_dump_view(); dump.create_webhooks(webhooks)?; let dump_uid = started_at.format(format_description!( diff --git a/crates/meilisearch-types/src/webhooks.rs b/crates/meilisearch-types/src/webhooks.rs index 0f0741d69..7a35850ab 100644 --- a/crates/meilisearch-types/src/webhooks.rs +++ b/crates/meilisearch-types/src/webhooks.rs @@ -11,9 +11,18 @@ pub struct Webhook { pub headers: BTreeMap, } -#[derive(Debug, Serialize, Deserialize, Default, Clone, PartialEq)] +#[derive(Debug, Serialize, Default, Clone, PartialEq)] #[serde(rename_all = "camelCase")] -pub struct Webhooks { +pub struct WebhooksView { + #[serde(default)] + pub webhooks: BTreeMap, +} + +// Same as the WebhooksView instead it should never contains the CLI webhooks. +// It's the right structure to use in the dump +#[derive(Debug, Deserialize, Serialize, Default, Clone, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct WebhooksDumpView { #[serde(default)] pub webhooks: BTreeMap, } diff --git a/crates/meilisearch/src/lib.rs b/crates/meilisearch/src/lib.rs index 533f0327f..ca9bb6f50 100644 --- a/crates/meilisearch/src/lib.rs +++ b/crates/meilisearch/src/lib.rs @@ -493,7 +493,7 @@ fn import_dump( // 2. Import the webhooks if let Some(webhooks) = dump_reader.webhooks() { - index_scheduler.put_webhooks(webhooks.clone())?; + index_scheduler.update_runtime_webhooks(webhooks.webhooks.clone())?; } // 3. Import the `Key`s. diff --git a/crates/meilisearch/src/routes/webhooks.rs b/crates/meilisearch/src/routes/webhooks.rs index e3547996a..7b3275a87 100644 --- a/crates/meilisearch/src/routes/webhooks.rs +++ b/crates/meilisearch/src/routes/webhooks.rs @@ -146,7 +146,7 @@ pub(super) struct WebhookResults { async fn get_webhooks( index_scheduler: GuardedData, Data>, ) -> Result { - let webhooks = index_scheduler.webhooks(); + let webhooks = index_scheduler.webhooks_view(); let results = webhooks .webhooks .into_iter() @@ -326,7 +326,7 @@ async fn get_webhook( uuid: Path, ) -> Result { let uuid = Uuid::from_str(&uuid.into_inner()).map_err(InvalidUuid)?; - let mut webhooks = index_scheduler.webhooks(); + let mut webhooks = index_scheduler.webhooks_view(); let webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; let webhook = WebhookWithMetadata::from(uuid, webhook); @@ -368,8 +368,8 @@ async fn post_webhook( return Err(TooManyHeaders(uuid).into()); } - let mut webhooks = index_scheduler.webhooks(); - if webhooks.webhooks.len() >= 20 { + let mut webhooks = index_scheduler.retrieve_runtime_webhooks(); + if webhooks.len() >= 20 { return Err(TooManyWebhooks.into()); } @@ -383,8 +383,8 @@ async fn post_webhook( }; check_changed(uuid, &webhook)?; - webhooks.webhooks.insert(uuid, webhook.clone()); - index_scheduler.put_webhooks(webhooks)?; + webhooks.insert(uuid, webhook.clone()); + index_scheduler.update_runtime_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::post_webhook(), &req); @@ -426,13 +426,17 @@ async fn patch_webhook( let webhook_settings = webhook_settings.into_inner(); debug!(parameters = ?(uuid, &webhook_settings), "Patch webhook"); - let mut webhooks = index_scheduler.webhooks(); - let old_webhook = webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; + if uuid.is_nil() { + return Err(ImmutableWebhook(uuid).into()); + } + + let mut webhooks = index_scheduler.retrieve_runtime_webhooks(); + let old_webhook = webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; let webhook = patch_webhook_inner(&uuid, old_webhook, webhook_settings)?; check_changed(uuid, &webhook)?; - webhooks.webhooks.insert(uuid, webhook.clone()); - index_scheduler.put_webhooks(webhooks)?; + webhooks.insert(uuid, webhook.clone()); + index_scheduler.update_runtime_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::patch_webhook(), &req); @@ -468,9 +472,9 @@ async fn delete_webhook( return Err(ImmutableWebhook(uuid).into()); } - let mut webhooks = index_scheduler.webhooks(); - webhooks.webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; - index_scheduler.put_webhooks(webhooks)?; + let mut webhooks = index_scheduler.retrieve_runtime_webhooks(); + webhooks.remove(&uuid).ok_or(WebhookNotFound(uuid))?; + index_scheduler.update_runtime_webhooks(webhooks)?; analytics.publish(PatchWebhooksAnalytics::delete_webhook(), &req); diff --git a/crates/meilisearch/tests/tasks/webhook.rs b/crates/meilisearch/tests/tasks/webhook.rs index a3be37a4e..bf2477b25 100644 --- a/crates/meilisearch/tests/tasks/webhook.rs +++ b/crates/meilisearch/tests/tasks/webhook.rs @@ -283,7 +283,6 @@ async fn reserved_names() { let (value, code) = server .patch_webhook(Uuid::nil().to_string(), json!({ "url": "http://localhost:8080" })) .await; - snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { "message": "Webhook `[uuid]` is immutable. The webhook defined from the command line cannot be modified using the API.", @@ -292,9 +291,9 @@ async fn reserved_names() { "link": "https://docs.meilisearch.com/errors#immutable_webhook" } "#); + snapshot!(code, @"400 Bad Request"); let (value, code) = server.delete_webhook(Uuid::nil().to_string()).await; - snapshot!(code, @"400 Bad Request"); snapshot!(value, @r#" { "message": "Webhook `[uuid]` is immutable. The webhook defined from the command line cannot be modified using the API.", @@ -303,6 +302,7 @@ async fn reserved_names() { "link": "https://docs.meilisearch.com/errors#immutable_webhook" } "#); + snapshot!(code, @"400 Bad Request"); } #[actix_web::test]