snowykami/meilisearch
1
0
Fork 0
mirror of https://github.com/meilisearch/meilisearch.git synced 2025-09-04 11:46:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Stop storing the cli webhook in the db

Browse Source
This commit is contained in:
Mubelotix
2025-08-05 11:44:53 +02:00
parent 8b27dec25c
commit 4f6a48c327
6 changed files with 41 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
crates/dump/src/reader/mod.rs
View File

@ -458,6 +458,11 @@ pub(crate) mod test {
// webhooks
// Important note: You might be surprised to see the cli webhook in the dump, as it's not supposed to be saved.
// This is because the dump comes from a version that did save it.
// It's no longer the case, but that's not what this test is about.
// It's ok to see the cli webhook disappear when this test gets updated.
let webhooks = dump.webhooks().unwrap();
insta::assert_json_snapshot!(webhooks, @r#"
{

2
crates/index-scheduler/src/insta_snapshot.rs
View File

@ -30,6 +30,8 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
index_mapper,
features: _,
cli_webhook_url: _,
cli_webhook_authorization: _,
cached_webhooks: _,
test_breakpoint_sdr: _,
planned_failures: _,

32
crates/index-scheduler/src/lib.rs
View File

@ -73,6 +73,7 @@ use queue::Queue;
use roaring::RoaringBitmap;
use scheduler::Scheduler;
use time::OffsetDateTime;
use uuid::Uuid;
use versioning::Versioning;
use crate::index_mapper::IndexMapper;
@ -107,6 +108,10 @@ pub struct IndexSchedulerOptions {
pub snapshots_path: PathBuf,
/// The path to the folder containing the dumps.
pub dumps_path: PathBuf,
/// The webhook url that was set by the CLI.
pub cli_webhook_url: Option<String>,
/// The Authorization header to send to the webhook URL that was set by the CLI.
pub cli_webhook_authorization: Option<String>,
/// The maximum size, in bytes, of the task index.
pub task_db_size: usize,
/// The size, in bytes, with which a meilisearch index is opened the first time of each meilisearch index.
@ -179,6 +184,10 @@ pub struct IndexScheduler {
/// A database to store single-keyed data that is persisted across restarts.
persisted: Database<Str, Str>,
/// The webhook url that was set by the CLI.
cli_webhook_url: Option<String>,
/// The Authorization header to send to the webhook URL that was set by the CLI.
cli_webhook_authorization: Option<String>,
/// Webhook
cached_webhooks: Arc<RwLock<Webhooks>>,
@ -221,7 +230,11 @@ impl IndexScheduler {
cleanup_enabled: self.cleanup_enabled,
experimental_no_edition_2024_for_dumps: self.experimental_no_edition_2024_for_dumps,
persisted: self.persisted,
cli_webhook_url: self.cli_webhook_url.clone(),
cli_webhook_authorization: self.cli_webhook_authorization.clone(),
cached_webhooks: self.cached_webhooks.clone(),
embedders: self.embedders.clone(),
#[cfg(test)]
test_breakpoint_sdr: self.test_breakpoint_sdr.clone(),
@ -299,7 +312,8 @@ impl IndexScheduler {
let persisted = env.create_database(&mut wtxn, Some(db_name::PERSISTED))?;
let webhooks_db = persisted.remap_data_type::<SerdeJson<Webhooks>>();
let webhooks = webhooks_db.get(&wtxn, db_keys::WEBHOOKS)?.unwrap_or_default();
let mut webhooks = webhooks_db.get(&wtxn, db_keys::WEBHOOKS)?.unwrap_or_default();
webhooks.webhooks.remove(&Uuid::nil()); // remove the cli webhook if it was saved by mistake
wtxn.commit()?;
@ -317,7 +331,10 @@ impl IndexScheduler {
.indexer_config
.experimental_no_edition_2024_for_dumps,
persisted,
cached_webhooks: Arc::new(RwLock::new(webhooks)),
cli_webhook_url: options.cli_webhook_url,
cli_webhook_authorization: options.cli_webhook_authorization,
embedders: Default::default(),
@ -869,9 +886,10 @@ impl IndexScheduler {
self.features.network()
}
pub fn put_webhooks(&self, webhooks: Webhooks) -> Result<()> {
pub fn put_webhooks(&self, mut webhooks: Webhooks) -> Result<()> {
let mut wtxn = self.env.write_txn()?;
let webhooks_db = self.persisted.remap_data_type::<SerdeJson<Webhooks>>();
webhooks.webhooks.remove(&Uuid::nil()); // the cli webhook must not be saved
webhooks_db.put(&mut wtxn, db_keys::WEBHOOKS, &webhooks)?;
wtxn.commit()?;
*self.cached_webhooks.write().unwrap() = webhooks;
@ -880,7 +898,15 @@ impl IndexScheduler {
pub fn webhooks(&self) -> Webhooks {
let webhooks = self.cached_webhooks.read().unwrap_or_else(|poisoned| poisoned.into_inner());
Webhooks::clone(&*webhooks)
let mut webhooks = Webhooks::clone(&*webhooks);
if let Some(url) = self.cli_webhook_url.as_ref().cloned() {
let mut headers = BTreeMap::new();
if let Some(auth) = self.cli_webhook_authorization.as_ref().cloned() {
headers.insert(String::from("Authorization"), auth);
}
webhooks.webhooks.insert(Uuid::nil(), Webhook { url, headers });
}
webhooks
}
pub fn embedders(

4
crates/index-scheduler/src/scheduler/mod.rs
View File

@ -26,7 +26,6 @@ use meilisearch_types::error::ResponseError;
use meilisearch_types::heed::{Env, WithoutTls};
use meilisearch_types::milli;
use meilisearch_types::tasks::Status;
use meilisearch_types::webhooks::Webhooks;
use process_batch::ProcessBatchInfo;
use rayon::current_num_threads;
use rayon::iter::{IntoParallelIterator, ParallelIterator};
@ -448,9 +447,8 @@ impl IndexScheduler {
})?;
// We shouldn't crash the tick function if we can't send data to the webhooks
let webhooks = self.cached_webhooks.read().unwrap_or_else(|p| p.into_inner());
let webhooks = self.webhooks();
if !webhooks.webhooks.is_empty() {
let webhooks = Webhooks::clone(&*webhooks);
let cloned_index_scheduler = self.private_clone();
std::thread::spawn(move || {
if let Err(e) = cloned_index_scheduler.notify_webhooks(webhooks, &ids) {

2
crates/index-scheduler/src/test_utils.rs
View File

@ -98,6 +98,8 @@ impl IndexScheduler {
indexes_path: tempdir.path().join("indexes"),
snapshots_path: tempdir.path().join("snapshots"),
dumps_path: tempdir.path().join("dumps"),
cli_webhook_url: None,
cli_webhook_authorization: None,
task_db_size: 1000 * 1000 * 10, // 10 MB, we don't use MiB on purpose.
index_base_map_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose.
enable_mdb_writemap: false,

29
crates/meilisearch/src/lib.rs
View File

@ -13,7 +13,6 @@ pub mod routes;
pub mod search;
pub mod search_queue;
use std::collections::BTreeMap;
use std::fs::File;
use std::io::{BufReader, BufWriter};
use std::path::Path;
@ -49,14 +48,12 @@ use meilisearch_types::tasks::KindWithContent;
use meilisearch_types::versioning::{
create_current_version_file, get_version, VersionFileError, VERSION_MINOR, VERSION_PATCH,
};
use meilisearch_types::webhooks::Webhook;
use meilisearch_types::{compression, heed, milli, VERSION_FILE_NAME};
pub use option::Opt;
use option::ScheduleSnapshot;
use search_queue::SearchQueue;
use tracing::{error, info_span};
use tracing_subscriber::filter::Targets;
use uuid::Uuid;
use crate::error::MeilisearchHttpError;
@ -226,6 +223,8 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Arc<
indexes_path: opt.db_path.join("indexes"),
snapshots_path: opt.snapshot_dir.clone(),
dumps_path: opt.dump_dir.clone(),
cli_webhook_url: opt.task_webhook_url.as_ref().map(|url| url.to_string()),
cli_webhook_authorization: opt.task_webhook_authorization_header.clone(),
task_db_size: opt.max_task_db_size.as_u64() as usize,
index_base_map_size: opt.max_index_size.as_u64() as usize,
enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage,
@ -328,30 +327,6 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Arc<
.unwrap();
}
// We set the webhook url
let cli_webhook = opt.task_webhook_url.as_ref().map(|u| Webhook {
url: u.to_string(),
headers: {
let mut headers = BTreeMap::new();
if let Some(value) = &opt.task_webhook_authorization_header {
headers.insert(String::from("Authorization"), value.to_string());
}
headers
},
});
let mut webhooks = index_scheduler.webhooks();
if webhooks.webhooks.get(&Uuid::nil()) != cli_webhook.as_ref() {
match cli_webhook {
Some(webhook) => {
webhooks.webhooks.insert(Uuid::nil(), webhook);
}
None => {
webhooks.webhooks.remove(&Uuid::nil());
}
}
index_scheduler.put_webhooks(webhooks)?;
}
Ok((index_scheduler, auth_controller))
}