Merge pull request #5886 from meilisearch/fix-decoding-error

Allow missing `search_fragments` and `indexing_fragments`

crates/milli/src/vector/db.rs

@@ -12,6 +12,12 @@ use crate::vector::settings::RemoveFragments;
 use crate::vector::EmbeddingConfig;
 use crate::{CboRoaringBitmapCodec, DocumentId, UserError};
 
+/// DB representation of an embedder configuration.
+///
+/// # Warning
+///
+/// This type is serialized in and deserialized from the DB, any modification should either go
+/// through dumpless upgrade or be backward-compatible
 #[derive(Debug, Deserialize, Serialize)]
 pub struct IndexEmbeddingConfig {
     pub name: String,

crates/milli/src/vector/embedder/composite.rs

@@ -24,6 +24,12 @@ pub enum SubEmbedder {
     Rest(rest::Embedder),
 }
 
+/// Options of a subembedder, specific to each kind of embedder.
+///
+/// # Warning
+///
+/// This type is serialized in and deserialized from the DB, any modification should either go
+/// through dumpless upgrade or be backward-compatible
 #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
 pub enum SubEmbedderOptions {
     HuggingFace(hf::EmbedderOptions),
@@ -51,6 +57,12 @@ pub struct Embedder {
     pub(super) index: SubEmbedder,
 }
 
+/// Options of a composite embedder, specific to each kind of embedder.
+///
+/// # Warning
+///
+/// This type is serialized in and deserialized from the DB, any modification should either go
+/// through dumpless upgrade or be backward-compatible
 #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
 pub struct EmbedderOptions {
     pub search: SubEmbedderOptions,

crates/milli/src/vector/embedder/hf.rs

@@ -30,6 +30,12 @@ enum WeightSource {
     Pytorch,
 }
 
+/// Inert embedder options for a hf embedder.
+///
+/// # Warning
+///
+/// This type is serialized in and deserialized from the DB, any modification should either go
+/// through dumpless upgrade or be backward-compatible
 #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
 pub struct EmbedderOptions {
     pub model: String,

crates/milli/src/vector/embedder/manual.rs

@@ -7,6 +7,12 @@ pub struct Embedder {
     distribution: Option<DistributionShift>,
 }
 
+/// Inert embedder options for a manual embedder.
+///
+/// # Warning
+///
+/// This type is serialized in and deserialized from the DB, any modification should either go
+/// through dumpless upgrade or be backward-compatible
 #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
 pub struct EmbedderOptions {
     pub dimensions: usize,

crates/milli/src/vector/embedder/mod.rs

@@ -35,6 +35,11 @@ pub enum Embedder {
 }
 
 /// Configuration for an embedder.
+///
+/// # Warning
+///
+/// This type is serialized in and deserialized from the DB, any modification should either go
+/// through dumpless upgrade or be backward-compatible
 #[derive(Debug, Clone, Default, serde::Deserialize, serde::Serialize)]
 pub struct EmbeddingConfig {
     /// Options of the embedder, specific to each kind of embedder
@@ -53,6 +58,11 @@ impl EmbeddingConfig {
 }
 
 /// Options of an embedder, specific to each kind of embedder.
+///
+/// # Warning
+///
+/// This type is serialized in and deserialized from the DB, any modification should either go
+/// through dumpless upgrade or be backward-compatible
 #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
 pub enum EmbedderOptions {
     HuggingFace(hf::EmbedderOptions),

crates/milli/src/vector/embedder/ollama.rs

@@ -16,6 +16,12 @@ pub struct Embedder {
     rest_embedder: RestEmbedder,
 }
 
+/// Inert embedder options for an ollama embedder.
+///
+/// # Warning
+///
+/// This type is serialized in and deserialized from the DB, any modification should either go
+/// through dumpless upgrade or be backward-compatible
 #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
 pub struct EmbedderOptions {
     pub embedding_model: String,

crates/milli/src/vector/embedder/openai.rs

@@ -13,6 +13,12 @@ use crate::vector::error::{EmbedError, EmbedErrorKind, NewEmbedderError};
 use crate::vector::{Embedding, REQUEST_PARALLELISM};
 use crate::ThreadPoolNoAbort;
 
+/// Inert embedder options for an openai embedder.
+///
+/// # Warning
+///
+/// This type is serialized in and deserialized from the DB, any modification should either go
+/// through dumpless upgrade or be backward-compatible
 #[derive(Debug, Clone, Hash, PartialEq, Eq, serde::Deserialize, serde::Serialize)]
 pub struct EmbedderOptions {
     pub url: Option<String>,

crates/milli/src/vector/embedder/rest.rs

@@ -133,6 +133,12 @@ impl RequestData {
     }
 }
 
+/// Inert embedder options for a rest embedder.
+///
+/// # Warning
+///
+/// This type is serialized in and deserialized from the DB, any modification should either go
+/// through dumpless upgrade or be backward-compatible
 #[derive(Debug, Clone, PartialEq, Eq, Deserialize, Serialize)]
 pub struct EmbedderOptions {
     pub api_key: Option<String>,
@@ -140,7 +146,9 @@ pub struct EmbedderOptions {
     pub dimensions: Option<usize>,
     pub url: String,
     pub request: Value,
+    #[serde(default)] // backward compatibility
     pub search_fragments: BTreeMap<String, Value>,
+    #[serde(default)] // backward compatibility
     pub indexing_fragments: BTreeMap<String, Value>,
     pub response: Value,
     pub headers: BTreeMap<String, String>,