Merge #4867

4867: Autogenerate the openAPI spec r=irevoire a=irevoire

# Pull Request

## Related issue
Fixes https://github.com/meilisearch/meilisearch/issues/5073

## What does this PR do?
- Introduce utoipa and the auto-generation of the openAPI file
- Introduce the scalar swagger when the `swagger` feature flag is enabled.

Generating the openAPI file takes between 15 and 20ms at startup time on my computer. That could be an issue if we plan to stabilize the feature.

Co-authored-by: Tamo <tamo@meilisearch.com>

crates/milli/Cargo.toml

@@ -100,6 +100,7 @@ uell = "0.1.0"
 enum-iterator = "2.1.0"
 bbqueue = { git = "https://github.com/meilisearch/bbqueue" }
 flume = { version = "0.11.1", default-features = false }
+utoipa = { version = "5.0.2", features = ["non_strict_integers", "preserve_order", "uuid", "time", "openapi_extensions"] }
 
 [dev-dependencies]
 mimalloc = { version = "0.1.43", default-features = false }

crates/milli/src/localized_attributes_rules.rs

@@ -2,6 +2,7 @@ use std::collections::HashMap;
 
 use charabia::Language;
 use serde::{Deserialize, Serialize};
+use utoipa::ToSchema;
 
 use crate::fields_ids_map::FieldsIdsMap;
 use crate::FieldId;
@@ -14,9 +15,10 @@ use crate::FieldId;
 /// The pattern `attribute_name*` matches any attribute name that starts with `attribute_name`.
 /// The pattern `*attribute_name` matches any attribute name that ends with `attribute_name`.
 /// The pattern `*attribute_name*` matches any attribute name that contains `attribute_name`.
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
 pub struct LocalizedAttributesRule {
     pub attribute_patterns: Vec<String>,
+    #[schema(value_type = Vec<String>)]
     pub locales: Vec<Language>,
 }
 

crates/milli/src/progress.rs

@@ -4,6 +4,7 @@ use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::{Arc, RwLock};
 
 use serde::Serialize;
+use utoipa::ToSchema;
 
 pub trait Step: 'static + Send + Sync {
     fn name(&self) -> Cow<'static, str>;
@@ -136,15 +137,17 @@ macro_rules! make_atomic_progress {
 make_atomic_progress!(Document alias AtomicDocumentStep => "document" );
 make_atomic_progress!(Payload alias AtomicPayloadStep => "payload" );
 
-#[derive(Debug, Serialize, Clone)]
+#[derive(Debug, Serialize, Clone, ToSchema)]
 #[serde(rename_all = "camelCase")]
+#[schema(rename_all = "camelCase")]
 pub struct ProgressView {
     pub steps: Vec<ProgressStepView>,
     pub percentage: f32,
 }
 
-#[derive(Debug, Serialize, Clone)]
+#[derive(Debug, Serialize, Clone, ToSchema)]
 #[serde(rename_all = "camelCase")]
+#[schema(rename_all = "camelCase")]
 pub struct ProgressStepView {
     pub current_step: Cow<'static, str>,
     pub finished: u32,

crates/milli/src/search/new/matches/mod.rs

@@ -13,6 +13,7 @@ use matching_words::{MatchType, PartialMatch};
 use r#match::{Match, MatchPosition};
 use serde::Serialize;
 use simple_token_kind::SimpleTokenKind;
+use utoipa::ToSchema;
 
 const DEFAULT_CROP_MARKER: &str = "…";
 const DEFAULT_HIGHLIGHT_PREFIX: &str = "<em>";
@@ -100,7 +101,7 @@ impl FormatOptions {
     }
 }
 
-#[derive(Serialize, Debug, Clone, PartialEq, Eq)]
+#[derive(Serialize, Debug, Clone, PartialEq, Eq, ToSchema)]
 pub struct MatchBounds {
     pub start: usize,
     pub length: usize,

crates/milli/src/vector/mod.rs

@@ -9,6 +9,7 @@ use heed::{RoTxn, RwTxn, Unspecified};
 use ordered_float::OrderedFloat;
 use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize};
+use utoipa::ToSchema;
 
 use self::error::{EmbedError, NewEmbedderError};
 use crate::prompt::{Prompt, PromptData};
@@ -710,18 +711,20 @@ impl Embedder {
 ///
 /// The intended use is to make the similarity score more comparable to the regular ranking score.
 /// This allows to correct effects where results are too "packed" around a certain value.
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Deserialize, Serialize, ToSchema)]
 #[serde(from = "DistributionShiftSerializable")]
 #[serde(into = "DistributionShiftSerializable")]
 pub struct DistributionShift {
     /// Value where the results are "packed".
     ///
     /// Similarity scores are translated so that they are packed around 0.5 instead
+    #[schema(value_type = f32)]
     pub current_mean: OrderedFloat<f32>,
 
     /// standard deviation of a similarity score.
     ///
     /// Set below 0.4 to make the results less packed around the mean, and above 0.4 to make them more packed.
+    #[schema(value_type = f32)]
     pub current_sigma: OrderedFloat<f32>,
 }
 

crates/milli/src/vector/settings.rs

@@ -4,6 +4,7 @@ use std::num::NonZeroUsize;
 use deserr::Deserr;
 use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize};
+use utoipa::ToSchema;
 
 use super::{ollama, openai, DistributionShift};
 use crate::prompt::{default_max_bytes, PromptData};
@@ -11,48 +12,61 @@ use crate::update::Setting;
 use crate::vector::EmbeddingConfig;
 use crate::UserError;
 
-#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)]
 #[serde(deny_unknown_fields, rename_all = "camelCase")]
 #[deserr(rename_all = camelCase, deny_unknown_fields)]
 pub struct EmbeddingSettings {
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<EmbedderSource>)]
     pub source: Setting<EmbedderSource>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<String>)]
     pub model: Setting<String>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<String>)]
     pub revision: Setting<String>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<String>)]
     pub api_key: Setting<String>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<String>)]
     pub dimensions: Setting<usize>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<bool>)]
     pub binary_quantized: Setting<bool>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<bool>)]
     pub document_template: Setting<String>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<usize>)]
     pub document_template_max_bytes: Setting<usize>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<String>)]
     pub url: Setting<String>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<serde_json::Value>)]
     pub request: Setting<serde_json::Value>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<serde_json::Value>)]
     pub response: Setting<serde_json::Value>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<BTreeMap<String, String>>)]
     pub headers: Setting<BTreeMap<String, String>>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default)]
+    #[schema(value_type = Option<DistributionShift>)]
     pub distribution: Setting<DistributionShift>,
 }
 
@@ -539,7 +553,7 @@ impl EmbeddingSettings {
     }
 }
 
-#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, Deserr)]
+#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)]
 #[serde(deny_unknown_fields, rename_all = "camelCase")]
 #[deserr(rename_all = camelCase, deny_unknown_fields)]
 pub enum EmbedderSource {