Merge branch 'release-v1.16.0' into fragment-filters

crates/index-scheduler/Cargo.toml

@@ -26,7 +26,7 @@ flate2 = "1.1.2"
 indexmap = "2.9.0"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-memmap2 = "0.9.5"
+memmap2 = "0.9.7"
 page_size = "0.6.0"
 rayon = "1.10.0"
 roaring = { version = "0.10.12", features = ["serde"] }

crates/index-scheduler/src/insta_snapshot.rs

@@ -20,6 +20,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
 
     let IndexScheduler {
         cleanup_enabled: _,
+        experimental_no_edition_2024_for_dumps: _,
         processing_tasks,
         env,
         version,

crates/index-scheduler/src/lib.rs

@@ -168,6 +168,9 @@ pub struct IndexScheduler {
     /// Whether we should automatically cleanup the task queue or not.
     pub(crate) cleanup_enabled: bool,
 
+    /// Whether we should use the old document indexer or the new one.
+    pub(crate) experimental_no_edition_2024_for_dumps: bool,
+
     /// The webhook url we should send tasks to after processing every batches.
     pub(crate) webhook_url: Option<String>,
     /// The Authorization header to send to the webhook URL.
@@ -210,6 +213,7 @@ impl IndexScheduler {
 
             index_mapper: self.index_mapper.clone(),
             cleanup_enabled: self.cleanup_enabled,
+            experimental_no_edition_2024_for_dumps: self.experimental_no_edition_2024_for_dumps,
             webhook_url: self.webhook_url.clone(),
             webhook_authorization_header: self.webhook_authorization_header.clone(),
             embedders: self.embedders.clone(),
@@ -296,6 +300,9 @@ impl IndexScheduler {
             index_mapper,
             env,
             cleanup_enabled: options.cleanup_enabled,
+            experimental_no_edition_2024_for_dumps: options
+                .indexer_config
+                .experimental_no_edition_2024_for_dumps,
             webhook_url: options.webhook_url,
             webhook_authorization_header: options.webhook_authorization_header,
             embedders: Default::default(),
@@ -594,6 +601,11 @@ impl IndexScheduler {
         Ok(nbr_index_processing_tasks > 0)
     }
 
+    /// Whether the index should use the old document indexer.
+    pub fn no_edition_2024_for_dumps(&self) -> bool {
+        self.experimental_no_edition_2024_for_dumps
+    }
+
     /// Return the tasks matching the query from the user's point of view along
     /// with the total number of tasks matching the query, ignoring from and limit.
     ///

crates/index-scheduler/src/scheduler/process_dump_creation.rs

@@ -5,6 +5,7 @@ use std::sync::atomic::Ordering;
 
 use dump::IndexMetadata;
 use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME;
+use meilisearch_types::milli::index::EmbeddingsWithMetadata;
 use meilisearch_types::milli::progress::{Progress, VariableNameStep};
 use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors};
 use meilisearch_types::milli::{self};
@@ -227,12 +228,21 @@ impl IndexScheduler {
                         return Err(Error::from_milli(user_err, Some(uid.to_string())));
                     };
 
-                    for (embedder_name, (embeddings, regenerate)) in embeddings {
+                    for (
+                        embedder_name,
+                        EmbeddingsWithMetadata { embeddings, regenerate, has_fragments },
+                    ) in embeddings
+                    {
                         let embeddings = ExplicitVectors {
                             embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors(
                                 embeddings,
                             )),
-                            regenerate,
+                            regenerate: regenerate &&
+                            // Meilisearch does not handle well dumps with fragments, because as the fragments
+                            // are marked as user-provided,
+                            // all embeddings would be regenerated on any settings change or document update.
+                            // To prevent this, we mark embeddings has non regenerate in this case.
+                            !has_fragments,
                         };
                         vectors.insert(embedder_name, serde_json::to_value(embeddings).unwrap());
                     }

crates/index-scheduler/src/scheduler/process_export.rs

@@ -9,6 +9,7 @@ use flate2::write::GzEncoder;
 use flate2::Compression;
 use meilisearch_types::index_uid_pattern::IndexUidPattern;
 use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME;
+use meilisearch_types::milli::index::EmbeddingsWithMetadata;
 use meilisearch_types::milli::progress::{Progress, VariableNameStep};
 use meilisearch_types::milli::update::{request_threads, Setting};
 use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors};
@@ -229,12 +230,21 @@ impl IndexScheduler {
                                 ));
                             };
 
-                            for (embedder_name, (embeddings, regenerate)) in embeddings {
+                            for (
+                                embedder_name,
+                                EmbeddingsWithMetadata { embeddings, regenerate, has_fragments },
+                            ) in embeddings
+                            {
                                 let embeddings = ExplicitVectors {
                                     embeddings: Some(
                                         VectorOrArrayOfVectors::from_array_of_vectors(embeddings),
                                     ),
-                                    regenerate,
+                                    regenerate: regenerate &&
+                                    // Meilisearch does not handle well dumps with fragments, because as the fragments
+                                    // are marked as user-provided,
+                                    // all embeddings would be regenerated on any settings change or document update.
+                                    // To prevent this, we mark embeddings has non regenerate in this case.
+                                    !has_fragments,
                                 };
                                 vectors.insert(
                                     embedder_name,

crates/index-scheduler/src/scheduler/test_embedders.rs

@@ -3,6 +3,7 @@ use std::collections::BTreeMap;
 use big_s::S;
 use insta::assert_json_snapshot;
 use meili_snap::{json_string, snapshot};
+use meilisearch_types::milli::index::EmbeddingsWithMetadata;
 use meilisearch_types::milli::update::Setting;
 use meilisearch_types::milli::vector::settings::EmbeddingSettings;
 use meilisearch_types::milli::vector::SearchQuery;
@@ -220,8 +221,8 @@ fn import_vectors() {
 
         let embeddings = index.embeddings(&rtxn, 0).unwrap();
 
-        assert_json_snapshot!(embeddings[&simple_hf_name].0[0] == lab_embed, @"true");
-        assert_json_snapshot!(embeddings[&fakerest_name].0[0] == beagle_embed, @"true");
+        assert_json_snapshot!(embeddings[&simple_hf_name].embeddings[0] == lab_embed, @"true");
+        assert_json_snapshot!(embeddings[&fakerest_name].embeddings[0] == beagle_embed, @"true");
 
         let doc = index.documents(&rtxn, std::iter::once(0)).unwrap()[0].1;
         let fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
@@ -311,9 +312,9 @@ fn import_vectors() {
             let embeddings = index.embeddings(&rtxn, 0).unwrap();
 
             // automatically changed to patou because set to regenerate
-            assert_json_snapshot!(embeddings[&simple_hf_name].0[0] == patou_embed, @"true");
+            assert_json_snapshot!(embeddings[&simple_hf_name].embeddings[0] == patou_embed, @"true");
             // remained beagle
-            assert_json_snapshot!(embeddings[&fakerest_name].0[0] == beagle_embed, @"true");
+            assert_json_snapshot!(embeddings[&fakerest_name].embeddings[0] == beagle_embed, @"true");
 
             let doc = index.documents(&rtxn, std::iter::once(0)).unwrap()[0].1;
             let fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
@@ -497,13 +498,13 @@ fn import_vectors_first_and_embedder_later() {
 
     let docid = index.external_documents_ids.get(&rtxn, "0").unwrap().unwrap();
     let embeddings = index.embeddings(&rtxn, docid).unwrap();
-    let (embedding, _) = &embeddings["my_doggo_embedder"];
-    assert!(!embedding.is_empty(), "{embedding:?}");
+    let EmbeddingsWithMetadata { embeddings, .. } = &embeddings["my_doggo_embedder"];
+    assert!(!embeddings.is_empty(), "{embeddings:?}");
 
     // the document with the id 3 should keep its original embedding
     let docid = index.external_documents_ids.get(&rtxn, "3").unwrap().unwrap();
     let embeddings = index.embeddings(&rtxn, docid).unwrap();
-    let (embeddings, _) = &embeddings["my_doggo_embedder"];
+    let EmbeddingsWithMetadata { embeddings, .. } = &embeddings["my_doggo_embedder"];
 
     snapshot!(embeddings.len(), @"1");
     assert!(embeddings[0].iter().all(|i| *i == 3.0), "{:?}", embeddings[0]);
@@ -558,7 +559,7 @@ fn import_vectors_first_and_embedder_later() {
         "###);
 
     let embeddings = index.embeddings(&rtxn, docid).unwrap();
-    let (embedding, _) = &embeddings["my_doggo_embedder"];
+    let EmbeddingsWithMetadata { embeddings: embedding, .. } = &embeddings["my_doggo_embedder"];
 
     assert!(!embedding.is_empty());
     assert!(!embedding[0].iter().all(|i| *i == 3.0), "{:?}", embedding[0]);
@@ -566,7 +567,7 @@ fn import_vectors_first_and_embedder_later() {
     // the document with the id 4 should generate an embedding
     let docid = index.external_documents_ids.get(&rtxn, "4").unwrap().unwrap();
     let embeddings = index.embeddings(&rtxn, docid).unwrap();
-    let (embedding, _) = &embeddings["my_doggo_embedder"];
+    let EmbeddingsWithMetadata { embeddings: embedding, .. } = &embeddings["my_doggo_embedder"];
 
     assert!(!embedding.is_empty());
 }
@@ -696,7 +697,7 @@ fn delete_document_containing_vector() {
     "###);
     let docid = index.external_documents_ids.get(&rtxn, "0").unwrap().unwrap();
     let embeddings = index.embeddings(&rtxn, docid).unwrap();
-    let (embedding, _) = &embeddings["manual"];
+    let EmbeddingsWithMetadata { embeddings: embedding, .. } = &embeddings["manual"];
     assert!(!embedding.is_empty(), "{embedding:?}");
 
     index_scheduler