From 0f3ef8de73f00b7d79f129ec802df1e3b43c7c62 Mon Sep 17 00:00:00 2001 From: Louis Dureuil Date: Tue, 23 Sep 2025 16:32:34 +0200 Subject: [PATCH] move Network route types to meilisearch_types and prefix existing ones by Db --- crates/dump/src/lib.rs | 8 +- crates/dump/src/reader/v6/mod.rs | 2 +- crates/dump/src/writer.rs | 4 +- crates/index-scheduler/src/features.rs | 16 ++-- crates/index-scheduler/src/lib.rs | 6 +- .../src/enterprise_edition/network.rs | 59 +++++++++++++- .../indexes/enterprise_edition/proxy.rs | 2 +- crates/meilisearch/src/routes/mod.rs | 3 +- crates/meilisearch/src/routes/network.rs | 80 +------------------ .../src/search/federated/perform.rs | 8 +- .../meilisearch/src/search/federated/proxy.rs | 4 +- 11 files changed, 84 insertions(+), 108 deletions(-) diff --git a/crates/dump/src/lib.rs b/crates/dump/src/lib.rs index fdbd701be..a926388b3 100644 --- a/crates/dump/src/lib.rs +++ b/crates/dump/src/lib.rs @@ -253,7 +253,7 @@ pub(crate) mod test { use big_s::S; use maplit::{btreemap, btreeset}; use meilisearch_types::batches::{Batch, BatchEnqueuedAt, BatchStats}; - use meilisearch_types::enterprise_edition::network::{Network, Remote}; + use meilisearch_types::enterprise_edition::network::{DbNetwork, DbRemote}; use meilisearch_types::facet_values_sort::FacetValuesSort; use meilisearch_types::features::RuntimeTogglableFeatures; use meilisearch_types::index_uid_pattern::IndexUidPattern; @@ -544,10 +544,10 @@ pub(crate) mod test { RuntimeTogglableFeatures::default() } - fn create_test_network() -> Network { - Network { + fn create_test_network() -> DbNetwork { + DbNetwork { local: Some("myself".to_string()), - remotes: maplit::btreemap! {"other".to_string() => Remote { url: "http://test".to_string(), search_api_key: Some("apiKey".to_string()), write_api_key: Some("docApiKey".to_string()) }}, + remotes: maplit::btreemap! {"other".to_string() => DbRemote { url: "http://test".to_string(), search_api_key: Some("apiKey".to_string()), write_api_key: Some("docApiKey".to_string()) }}, sharding: false, } } diff --git a/crates/dump/src/reader/v6/mod.rs b/crates/dump/src/reader/v6/mod.rs index b5549ec65..fe2b0cab8 100644 --- a/crates/dump/src/reader/v6/mod.rs +++ b/crates/dump/src/reader/v6/mod.rs @@ -24,7 +24,7 @@ pub type Batch = meilisearch_types::batches::Batch; pub type Key = meilisearch_types::keys::Key; pub type ChatCompletionSettings = meilisearch_types::features::ChatCompletionSettings; pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures; -pub type Network = meilisearch_types::enterprise_edition::network::Network; +pub type Network = meilisearch_types::enterprise_edition::network::DbNetwork; pub type Webhooks = meilisearch_types::webhooks::WebhooksDumpView; // ===== Other types to clarify the code of the compat module diff --git a/crates/dump/src/writer.rs b/crates/dump/src/writer.rs index 1f8000a50..04c5e3e88 100644 --- a/crates/dump/src/writer.rs +++ b/crates/dump/src/writer.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use flate2::write::GzEncoder; use flate2::Compression; use meilisearch_types::batches::Batch; -use meilisearch_types::enterprise_edition::network::Network; +use meilisearch_types::enterprise_edition::network::DbNetwork; use meilisearch_types::features::{ChatCompletionSettings, RuntimeTogglableFeatures}; use meilisearch_types::keys::Key; use meilisearch_types::settings::{Checked, Settings}; @@ -72,7 +72,7 @@ impl DumpWriter { )?) } - pub fn create_network(&self, network: Network) -> Result<()> { + pub fn create_network(&self, network: DbNetwork) -> Result<()> { Ok(std::fs::write(self.dir.path().join("network.json"), serde_json::to_string(&network)?)?) } diff --git a/crates/index-scheduler/src/features.rs b/crates/index-scheduler/src/features.rs index 5646a5d80..361353a1e 100644 --- a/crates/index-scheduler/src/features.rs +++ b/crates/index-scheduler/src/features.rs @@ -1,6 +1,6 @@ use std::sync::{Arc, RwLock}; -use meilisearch_types::enterprise_edition::network::Network; +use meilisearch_types::enterprise_edition::network::DbNetwork; use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures}; use meilisearch_types::heed::types::{SerdeJson, Str}; use meilisearch_types::heed::{Database, Env, RwTxn, WithoutTls}; @@ -24,7 +24,7 @@ mod db_keys { pub(crate) struct FeatureData { persisted: Database>, runtime: Arc>, - network: Arc>, + network: Arc>, } #[derive(Debug, Clone, Copy)] @@ -197,8 +197,8 @@ impl FeatureData { })); // Once this is stabilized, network should be stored along with webhooks in index-scheduler's persisted database - let network_db = runtime_features_db.remap_data_type::>(); - let network: Network = network_db.get(wtxn, db_keys::NETWORK)?.unwrap_or_default(); + let network_db = runtime_features_db.remap_data_type::>(); + let network: DbNetwork = network_db.get(wtxn, db_keys::NETWORK)?.unwrap_or_default(); Ok(Self { persisted: runtime_features_db, @@ -234,8 +234,8 @@ impl FeatureData { RoFeatures::new(self) } - pub fn put_network(&self, mut wtxn: RwTxn, new_network: Network) -> Result<()> { - self.persisted.remap_data_type::>().put( + pub fn put_network(&self, mut wtxn: RwTxn, new_network: DbNetwork) -> Result<()> { + self.persisted.remap_data_type::>().put( &mut wtxn, db_keys::NETWORK, &new_network, @@ -247,7 +247,7 @@ impl FeatureData { Ok(()) } - pub fn network(&self) -> Network { - Network::clone(&*self.network.read().unwrap()) + pub fn network(&self) -> DbNetwork { + DbNetwork::clone(&*self.network.read().unwrap()) } } diff --git a/crates/index-scheduler/src/lib.rs b/crates/index-scheduler/src/lib.rs index 5e31feab8..4e88013b7 100644 --- a/crates/index-scheduler/src/lib.rs +++ b/crates/index-scheduler/src/lib.rs @@ -54,7 +54,7 @@ pub use features::RoFeatures; use flate2::bufread::GzEncoder; use flate2::Compression; use meilisearch_types::batches::Batch; -use meilisearch_types::enterprise_edition::network::Network; +use meilisearch_types::enterprise_edition::network::DbNetwork; use meilisearch_types::features::{ ChatCompletionSettings, InstanceTogglableFeatures, RuntimeTogglableFeatures, }; @@ -892,13 +892,13 @@ impl IndexScheduler { Ok(()) } - pub fn put_network(&self, network: Network) -> Result<()> { + pub fn put_network(&self, network: DbNetwork) -> Result<()> { let wtxn = self.env.write_txn().map_err(Error::HeedTransaction)?; self.features.put_network(wtxn, network)?; Ok(()) } - pub fn network(&self) -> Network { + pub fn network(&self) -> DbNetwork { self.features.network() } diff --git a/crates/meilisearch-types/src/enterprise_edition/network.rs b/crates/meilisearch-types/src/enterprise_edition/network.rs index 9d5c51e25..0b4a2a534 100644 --- a/crates/meilisearch-types/src/enterprise_edition/network.rs +++ b/crates/meilisearch-types/src/enterprise_edition/network.rs @@ -5,21 +5,72 @@ use std::collections::BTreeMap; +use deserr::Deserr; use milli::update::new::indexer::enterprise_edition::sharding::Shards; +use milli::update::Setting; use serde::{Deserialize, Serialize}; +use utoipa::ToSchema; + +use crate::deserr::DeserrJsonError; +use crate::error::deserr_codes::{ + InvalidNetworkRemotes, InvalidNetworkSearchApiKey, InvalidNetworkSelf, InvalidNetworkSharding, + InvalidNetworkUrl, InvalidNetworkWriteApiKey, +}; + +#[derive(Clone, Debug, Deserr, ToSchema, Serialize, Deserialize, PartialEq, Eq)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] +pub struct Network { + #[schema(value_type = Option>, example = json!("http://localhost:7700"))] + #[deserr(default, error = DeserrJsonError)] + #[serde(default)] + pub remotes: Setting>>, + #[schema(value_type = Option, example = json!("ms-00"), rename = "self")] + #[serde(default, rename = "self")] + #[deserr(default, rename = "self", error = DeserrJsonError)] + pub local: Setting, + #[schema(value_type = Option, example = json!(true))] + #[serde(default)] + #[deserr(default, error = DeserrJsonError)] + pub sharding: Setting, +} + +#[derive(Clone, Debug, Deserr, ToSchema, Serialize, Deserialize, PartialEq, Eq)] +#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] +#[serde(rename_all = "camelCase")] +#[schema(rename_all = "camelCase")] +pub struct Remote { + #[schema(value_type = Option, example = json!({ + "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset, write_api_key: Setting::Reset }, + "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()), write_api_key: Setting::Set("bar".into()) }, + "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()), write_api_key: Setting::Set("foo".into()) }, + }))] + #[deserr(default, error = DeserrJsonError)] + #[serde(default)] + pub url: Setting, + #[schema(value_type = Option, example = json!("XWnBI8QHUc-4IlqbKPLUDuhftNq19mQtjc6JvmivzJU"))] + #[deserr(default, error = DeserrJsonError)] + #[serde(default)] + pub search_api_key: Setting, + #[schema(value_type = Option, example = json!("XWnBI8QHUc-4IlqbKPLUDuhftNq19mQtjc6JvmivzJU"))] + #[deserr(default, error = DeserrJsonError)] + #[serde(default)] + pub write_api_key: Setting, +} #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)] #[serde(rename_all = "camelCase")] -pub struct Network { +pub struct DbNetwork { #[serde(default, rename = "self")] pub local: Option, #[serde(default)] - pub remotes: BTreeMap, + pub remotes: BTreeMap, #[serde(default)] pub sharding: bool, } -impl Network { +impl DbNetwork { pub fn shards(&self) -> Option { if self.sharding { let this = self.local.as_deref().expect("Inconsistent `sharding` and `self`"); @@ -38,7 +89,7 @@ impl Network { #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] #[serde(rename_all = "camelCase")] -pub struct Remote { +pub struct DbRemote { pub url: String, #[serde(default)] pub search_api_key: Option, diff --git a/crates/meilisearch/src/routes/indexes/enterprise_edition/proxy.rs b/crates/meilisearch/src/routes/indexes/enterprise_edition/proxy.rs index 7cb3eb8cc..f918db17b 100644 --- a/crates/meilisearch/src/routes/indexes/enterprise_edition/proxy.rs +++ b/crates/meilisearch/src/routes/indexes/enterprise_edition/proxy.rs @@ -52,7 +52,7 @@ pub async fn proxy( index_scheduler: &IndexScheduler, index_uid: &str, req: &HttpRequest, - network: meilisearch_types::enterprise_edition::network::Network, + network: meilisearch_types::enterprise_edition::network::DbNetwork, body: Body, task: &meilisearch_types::tasks::Task, ) -> Result<(), MeilisearchHttpError> { diff --git a/crates/meilisearch/src/routes/mod.rs b/crates/meilisearch/src/routes/mod.rs index 5a6780cbb..9a65dd16d 100644 --- a/crates/meilisearch/src/routes/mod.rs +++ b/crates/meilisearch/src/routes/mod.rs @@ -39,7 +39,6 @@ use crate::routes::features::RuntimeTogglableFeatures; use crate::routes::indexes::documents::{DocumentDeletionByFilter, DocumentEditionByFunction}; use crate::routes::indexes::IndexView; use crate::routes::multi_search::SearchResults; -use crate::routes::network::{Network, Remote}; use crate::routes::swap_indexes::SwapIndexesPayload; use crate::routes::webhooks::{WebhookResults, WebhookSettings, WebhookWithMetadata}; use crate::search::{ @@ -102,7 +101,7 @@ mod webhooks; url = "/", description = "Local server", )), - components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, Network, Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export, WebhookSettings, WebhookResults, WebhookWithMetadata, meilisearch_types::milli::vector::VectorStoreBackend)) + components(schemas(PaginationView, PaginationView, IndexView, DocumentDeletionByFilter, AllBatches, BatchStats, ProgressStepView, ProgressView, BatchView, RuntimeTogglableFeatures, SwapIndexesPayload, DocumentEditionByFunction, MergeFacets, FederationOptions, SearchQueryWithIndex, Federation, FederatedSearch, FederatedSearchResult, SearchResults, SearchResultWithIndex, SimilarQuery, SimilarResult, PaginationView, BrowseQuery, UpdateIndexRequest, IndexUid, IndexCreateRequest, KeyView, Action, CreateApiKey, UpdateStderrLogs, LogMode, GetLogs, IndexStats, Stats, HealthStatus, HealthResponse, VersionResponse, Code, ErrorType, AllTasks, TaskView, Status, DetailsView, ResponseError, Settings, Settings, TypoSettings, MinWordSizeTyposSetting, FacetingSettings, PaginationSettings, SummarizedTaskView, Kind, meilisearch_types::enterprise_edition::network::Network, meilisearch_types::enterprise_edition::network::Remote, FilterableAttributesRule, FilterableAttributesPatterns, AttributePatterns, FilterableAttributesFeatures, FilterFeatures, Export, WebhookSettings, WebhookResults, WebhookWithMetadata, meilisearch_types::milli::vector::VectorStoreBackend)) )] pub struct MeilisearchApi; diff --git a/crates/meilisearch/src/routes/network.rs b/crates/meilisearch/src/routes/network.rs index b7fa2980c..5582e65b3 100644 --- a/crates/meilisearch/src/routes/network.rs +++ b/crates/meilisearch/src/routes/network.rs @@ -1,28 +1,21 @@ -use std::collections::BTreeMap; - use actix_web::web::{self, Data}; use actix_web::{HttpRequest, HttpResponse}; use deserr::actix_web::AwebJson; -use deserr::Deserr; use index_scheduler::IndexScheduler; -use itertools::{EitherOrBoth, Itertools}; use meilisearch_types::deserr::DeserrJsonError; -use meilisearch_types::enterprise_edition::network::{Network as DbNetwork, Remote as DbRemote}; -use meilisearch_types::error::deserr_codes::{ - InvalidNetworkRemotes, InvalidNetworkSearchApiKey, InvalidNetworkSelf, InvalidNetworkSharding, - InvalidNetworkUrl, InvalidNetworkWriteApiKey, -}; +use meilisearch_types::enterprise_edition::network::{Network, Remote}; use meilisearch_types::error::ResponseError; use meilisearch_types::keys::actions; use meilisearch_types::milli::update::Setting; use serde::Serialize; use tracing::debug; -use utoipa::{OpenApi, ToSchema}; +use utoipa::OpenApi; use crate::analytics::{Aggregate, Analytics}; use crate::extractors::authentication::policies::ActionPolicy; use crate::extractors::authentication::GuardedData; use crate::extractors::sequential_extractor::SeqHandler; +use crate::routes::SummarizedTaskView; #[derive(OpenApi)] #[openapi( @@ -83,73 +76,6 @@ async fn get_network( Ok(HttpResponse::Ok().json(network)) } -#[derive(Debug, Deserr, ToSchema, Serialize)] -#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -#[schema(rename_all = "camelCase")] -pub struct Remote { - #[schema(value_type = Option, example = json!({ - "ms-0": Remote { url: Setting::Set("http://localhost:7700".into()), search_api_key: Setting::Reset, write_api_key: Setting::Reset }, - "ms-1": Remote { url: Setting::Set("http://localhost:7701".into()), search_api_key: Setting::Set("foo".into()), write_api_key: Setting::Set("bar".into()) }, - "ms-2": Remote { url: Setting::Set("http://localhost:7702".into()), search_api_key: Setting::Set("bar".into()), write_api_key: Setting::Set("foo".into()) }, - }))] - #[deserr(default, error = DeserrJsonError)] - #[serde(default)] - pub url: Setting, - #[schema(value_type = Option, example = json!("XWnBI8QHUc-4IlqbKPLUDuhftNq19mQtjc6JvmivzJU"))] - #[deserr(default, error = DeserrJsonError)] - #[serde(default)] - pub search_api_key: Setting, - #[schema(value_type = Option, example = json!("XWnBI8QHUc-4IlqbKPLUDuhftNq19mQtjc6JvmivzJU"))] - #[deserr(default, error = DeserrJsonError)] - #[serde(default)] - pub write_api_key: Setting, -} - -#[derive(Debug, Deserr, ToSchema, Serialize)] -#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)] -#[serde(rename_all = "camelCase")] -#[schema(rename_all = "camelCase")] -pub struct Network { - #[schema(value_type = Option>, example = json!("http://localhost:7700"))] - #[deserr(default, error = DeserrJsonError)] - #[serde(default)] - pub remotes: Setting>>, - #[schema(value_type = Option, example = json!("ms-00"), rename = "self")] - #[serde(default, rename = "self")] - #[deserr(default, rename = "self", error = DeserrJsonError)] - pub local: Setting, - #[schema(value_type = Option, example = json!(true))] - #[serde(default)] - #[deserr(default, error = DeserrJsonError)] - pub sharding: Setting, -} - -impl Remote { - pub fn try_into_db_node(self, name: &str) -> Result { - Ok(DbRemote { - url: self - .url - .set() - .ok_or(ResponseError::from_msg( - format!("Missing field `.remotes.{name}.url`"), - meilisearch_types::error::Code::MissingNetworkUrl, - )) - .and_then(|url| { - if let Err(error) = url::Url::parse(&url) { - return Err(ResponseError::from_msg( - format!("Invalid `.remotes.{name}.url` (`{url}`): {error}"), - meilisearch_types::error::Code::InvalidNetworkUrl, - )); - } - Ok(url) - })?, - search_api_key: self.search_api_key.set(), - write_api_key: self.write_api_key.set(), - }) - } -} - #[derive(Serialize)] pub struct PatchNetworkAnalytics { network_size: usize, diff --git a/crates/meilisearch/src/search/federated/perform.rs b/crates/meilisearch/src/search/federated/perform.rs index 1af932c07..5c5318eec 100644 --- a/crates/meilisearch/src/search/federated/perform.rs +++ b/crates/meilisearch/src/search/federated/perform.rs @@ -9,7 +9,7 @@ use std::vec::{IntoIter, Vec}; use actix_http::StatusCode; use index_scheduler::{IndexScheduler, RoFeatures}; use itertools::Itertools; -use meilisearch_types::enterprise_edition::network::{Network, Remote}; +use meilisearch_types::enterprise_edition::network::{DbNetwork, DbRemote}; use meilisearch_types::error::ResponseError; use meilisearch_types::milli::order_by_map::OrderByMap; use meilisearch_types::milli::score_details::{ScoreDetails, WeightedScoreValue}; @@ -456,7 +456,7 @@ fn merge_metadata( } type LocalQueriesByIndex = BTreeMap>; -type RemoteQueriesByHost = BTreeMap)>; +type RemoteQueriesByHost = BTreeMap)>; struct PartitionedQueries { local_queries_by_index: LocalQueriesByIndex, @@ -477,7 +477,7 @@ impl PartitionedQueries { &mut self, federated_query: SearchQueryWithIndex, query_index: usize, - network: &Network, + network: &DbNetwork, features: RoFeatures, ) -> Result<(), ResponseError> { if let Some(pagination_field) = federated_query.has_pagination() { @@ -672,7 +672,7 @@ struct SearchByIndexParams<'a> { features: RoFeatures, is_proxy: bool, has_remote: bool, - network: &'a Network, + network: &'a DbNetwork, } struct SearchByIndex { diff --git a/crates/meilisearch/src/search/federated/proxy.rs b/crates/meilisearch/src/search/federated/proxy.rs index ac8cd2491..dfad81260 100644 --- a/crates/meilisearch/src/search/federated/proxy.rs +++ b/crates/meilisearch/src/search/federated/proxy.rs @@ -1,6 +1,6 @@ pub use error::ProxySearchError; use error::ReqwestErrorWithoutUrl; -use meilisearch_types::enterprise_edition::network::Remote; +use meilisearch_types::enterprise_edition::network::DbRemote; use rand::Rng as _; use reqwest::{Client, Response, StatusCode}; use serde::de::DeserializeOwned; @@ -94,7 +94,7 @@ pub struct ProxySearchParams { /// Performs a federated search on a remote host and returns the results pub async fn proxy_search( - node: &Remote, + node: &DbRemote, queries: Vec, federation: Federation, params: &ProxySearchParams,