Fix flaky Ollama tests

Cargo.lock

@@ -580,7 +580,7 @@ source = "git+https://github.com/meilisearch/bbqueue#e8af4a4bccc8eb36b2b0442c4a9
 
 [[package]]
 name = "benchmarks"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "anyhow",
  "bumpalo",
@@ -790,7 +790,7 @@ dependencies = [
 
 [[package]]
 name = "build-info"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "anyhow",
  "time",
@@ -1786,7 +1786,7 @@ dependencies = [
 
 [[package]]
 name = "dump"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "anyhow",
  "big_s",
@@ -2018,7 +2018,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
 [[package]]
 name = "file-store"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "tempfile",
  "thiserror 2.0.17",
@@ -2040,7 +2040,7 @@ dependencies = [
 
 [[package]]
 name = "filter-parser"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "insta",
  "levenshtein_automata",
@@ -2068,7 +2068,7 @@ dependencies = [
 
 [[package]]
 name = "flatten-serde-json"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "criterion",
  "serde_json",
@@ -2231,7 +2231,7 @@ dependencies = [
 
 [[package]]
 name = "fuzzers"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "arbitrary",
  "bumpalo",
@@ -3185,7 +3185,7 @@ dependencies = [
 
 [[package]]
 name = "index-scheduler"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "anyhow",
  "backoff",
@@ -3449,7 +3449,7 @@ dependencies = [
 
 [[package]]
 name = "json-depth-checker"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "criterion",
  "serde_json",
@@ -3939,7 +3939,7 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0"
 
 [[package]]
 name = "meili-snap"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "insta",
  "md5 0.8.0",
@@ -3950,7 +3950,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "actix-cors",
  "actix-http",
@@ -4048,7 +4048,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-auth"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "base64 0.22.1",
  "enum-iterator",
@@ -4067,7 +4067,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-types"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "actix-web",
  "anyhow",
@@ -4105,7 +4105,7 @@ dependencies = [
 
 [[package]]
 name = "meilitool"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "anyhow",
  "clap",
@@ -4139,7 +4139,7 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "arroy",
  "bbqueue",
@@ -4718,7 +4718,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
 
 [[package]]
 name = "permissive-json-pointer"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "big_s",
  "serde_json",
@@ -7758,7 +7758,7 @@ dependencies = [
 
 [[package]]
 name = "xtask"
-version = "1.30.0"
+version = "1.30.1"
 dependencies = [
  "anyhow",
  "build-info",

Cargo.toml

@@ -23,7 +23,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.30.0"
+version = "1.30.1"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",

crates/index-scheduler/src/scheduler/process_batch.rs

@@ -662,13 +662,8 @@ impl IndexScheduler {
         // 2. Get the task set for index = name that appeared before the index swap task
         let mut index_lhs_task_ids = self.queue.tasks.index_tasks(wtxn, lhs)?;
         index_lhs_task_ids.remove_range(task_id..);
-        let index_rhs_task_ids = if rename {
+        let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?;
-            let mut index_rhs_task_ids = self.queue.tasks.index_tasks(wtxn, rhs)?;
+        index_rhs_task_ids.remove_range(task_id..);
-            index_rhs_task_ids.remove_range(task_id..);
-            index_rhs_task_ids
-        } else {
-            RoaringBitmap::new()
-        };
 
         // 3. before_name -> new_name in the task's KindWithContent
         progress.update_progress(InnerSwappingTwoIndexes::UpdateTheTasks);

crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/first_swap_processed.snap

@@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
-1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
+1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }}
 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
-3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
+3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }}
 4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "b"), rename: false }, IndexSwap { indexes: ("c", "d"), rename: false }] }}
 5 {uid: 5, status: enqueued, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }}
 ----------------------------------------------------------------------
@@ -22,10 +22,10 @@ succeeded [0,1,2,3,4,]
 "indexSwap" [4,5,]
 ----------------------------------------------------------------------
 ### Index Tasks:
-a [4,5,]
+a [1,4,5,]
-b [0,1,4,]
+b [0,4,]
-c [4,5,]
+c [3,4,5,]
-d [2,3,4,]
+d [2,4,]
 ----------------------------------------------------------------------
 ### Index Mapper:
 a: { number_of_documents: 0, field_distribution: {} }

crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/second_swap_processed.snap

@@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
-1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
+1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }}
 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
-3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
+3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }}
 4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }}
 5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }}
 ----------------------------------------------------------------------
@@ -22,10 +22,10 @@ succeeded [0,1,2,3,4,5,]
 "indexSwap" [4,5,]
 ----------------------------------------------------------------------
 ### Index Tasks:
-a [5,]
+a [3,4,5,]
-b [0,1,4,]
+b [0,4,]
-c [4,5,]
+c [1,4,5,]
-d [2,3,4,]
+d [2,4,]
 ----------------------------------------------------------------------
 ### Index Mapper:
 a: { number_of_documents: 0, field_distribution: {} }

crates/index-scheduler/src/scheduler/snapshots/test.rs/swap_indexes/third_empty_swap_processed.snap

@@ -7,9 +7,9 @@ source: crates/index-scheduler/src/scheduler/test.rs
 ----------------------------------------------------------------------
 ### All Tasks:
 0 {uid: 0, batch_uid: 0, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
-1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "b", primary_key: Some("id") }}
+1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "c", primary_key: Some("id") }}
 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
-3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "d", primary_key: Some("id") }}
+3 {uid: 3, batch_uid: 3, status: succeeded, details: { primary_key: Some("id"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "a", primary_key: Some("id") }}
 4 {uid: 4, batch_uid: 4, status: succeeded, details: { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("c", "b"), rename: false }, IndexSwap { indexes: ("a", "d"), rename: false }] }}
 5 {uid: 5, batch_uid: 5, status: succeeded, details: { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }, kind: IndexSwap { swaps: [IndexSwap { indexes: ("a", "c"), rename: false }] }}
 6 {uid: 6, batch_uid: 6, status: succeeded, details: { swaps: [] }, kind: IndexSwap { swaps: [] }}
@@ -23,10 +23,10 @@ succeeded [0,1,2,3,4,5,6,]
 "indexSwap" [4,5,6,]
 ----------------------------------------------------------------------
 ### Index Tasks:
-a [5,]
+a [3,4,5,]
-b [0,1,4,]
+b [0,4,]
-c [4,5,]
+c [1,4,5,]
-d [2,3,4,]
+d [2,4,]
 ----------------------------------------------------------------------
 ### Index Mapper:
 a: { number_of_documents: 0, field_distribution: {} }

crates/index-scheduler/src/scheduler/test.rs

@@ -5,7 +5,7 @@ use crate::test_utils::Breakpoint::*;
 use crate::test_utils::{
     index_creation_task, read_json, replace_document_import_task, sample_documents,
 };
-use crate::IndexScheduler;
+use crate::{IndexScheduler, Query};
 use big_s::S;
 use meili_snap::{json_string, snapshot};
 use meilisearch_auth::AuthFilter;
@@ -404,6 +404,103 @@ fn swap_indexes() {
     snapshot!(snapshot_index_scheduler(&index_scheduler), name: "third_empty_swap_processed");
 }
 
+#[test]
+fn swap_indexes_with_correct_task_allocations() {
+    let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
+
+    let to_enqueue = [index_creation_task("a", "id"), index_creation_task("b", "id")];
+
+    for task in to_enqueue {
+        let _ = index_scheduler.register(task, None, false).unwrap();
+        index_scheduler.assert_internally_consistent();
+    }
+
+    handle.advance_n_successful_batches(2);
+
+    let (file0, count0) = sample_documents(&index_scheduler, 1, 1);
+    let (file1, count1) = sample_documents(&index_scheduler, 2, 2);
+    let (file2, count2) = sample_documents(&index_scheduler, 3, 3);
+    let (file3, count3) = sample_documents(&index_scheduler, 4, 4);
+
+    file0.persist().unwrap();
+    file1.persist().unwrap();
+    file2.persist().unwrap();
+    file3.persist().unwrap();
+
+    index_scheduler
+        .register(replace_document_import_task("a", Some("id"), 1, count0), None, false)
+        .unwrap();
+
+    index_scheduler
+        .register(replace_document_import_task("a", Some("id"), 2, count1), None, false)
+        .unwrap();
+
+    index_scheduler
+        .register(replace_document_import_task("b", Some("id"), 3, count2), None, false)
+        .unwrap();
+
+    index_scheduler
+        .register(replace_document_import_task("b", Some("id"), 4, count3), None, false)
+        .unwrap();
+
+    handle.advance_n_successful_batches(2);
+
+    let (a_tasks, _) = index_scheduler
+        .get_tasks_from_authorized_indexes(
+            &Query { index_uids: Some(vec!["a".to_string()]), ..Default::default() },
+            &AuthFilter::default(),
+        )
+        .unwrap();
+
+    assert_eq!(a_tasks.len(), 3);
+
+    let (b_tasks, _) = index_scheduler
+        .get_tasks_from_authorized_indexes(
+            &Query { index_uids: Some(vec!["b".to_string()]), ..Default::default() },
+            &AuthFilter::default(),
+        )
+        .unwrap();
+
+    assert_eq!(b_tasks.len(), 3);
+
+    index_scheduler
+        .register(
+            KindWithContent::IndexSwap {
+                swaps: vec![IndexSwap { indexes: ("a".to_owned(), "b".to_owned()), rename: false }],
+            },
+            None,
+            false,
+        )
+        .unwrap();
+
+    handle.advance_one_successful_batch();
+
+    let (a_after_tasks, _) = index_scheduler
+        .get_tasks_from_authorized_indexes(
+            &Query { index_uids: Some(vec!["a".to_string()]), ..Default::default() },
+            &AuthFilter::default(),
+        )
+        .unwrap();
+
+    let (b_after_tasks, _) = index_scheduler
+        .get_tasks_from_authorized_indexes(
+            &Query { index_uids: Some(vec!["b".to_string()]), ..Default::default() },
+            &AuthFilter::default(),
+        )
+        .unwrap();
+
+    assert_eq!(a_after_tasks.len(), 3);
+    assert_eq!(a_after_tasks.len(), b_after_tasks.len());
+
+    for (a, b) in a_tasks.iter().zip(b_after_tasks.iter()) {
+        assert_eq!(a.uid, b.uid);
+    }
+
+    for (b, a) in b_tasks.iter().zip(a_after_tasks.iter()) {
+        assert_eq!(b.uid, a.uid);
+    }
+}
+
 #[test]
 fn swap_indexes_errors() {
     let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);

crates/meilisearch/tests/swap_indexes/mod.rs

@@ -103,7 +103,7 @@ async fn swap_indexes() {
         {
           "uid": 1,
           "batchUid": 1,
-          "indexUid": "b",
+          "indexUid": "a",
           "status": "succeeded",
           "type": "documentAdditionOrUpdate",
           "canceledBy": null,
@@ -266,7 +266,7 @@ async fn swap_indexes() {
         {
           "uid": 4,
           "batchUid": 4,
-          "indexUid": "d",
+          "indexUid": "c",
           "status": "succeeded",
           "type": "documentAdditionOrUpdate",
           "canceledBy": null,
@@ -341,7 +341,7 @@ async fn swap_indexes() {
         {
           "uid": 0,
           "batchUid": 0,
-          "indexUid": "b",
+          "indexUid": "a",
           "status": "succeeded",
           "type": "documentAdditionOrUpdate",
           "canceledBy": null,

crates/meilisearch/tests/vector/ollama.rs

@@ -274,19 +274,19 @@ async fn test_both_apis() {
         "birthyear": 2011,
         "breed": "Beagle"
       },
-      {
-        "id": 3,
-        "name": "Max",
-        "gender": "M",
-        "birthyear": 1995,
-        "breed": "Labrador Retriever"
-      },
       {
         "id": 2,
         "name": "Vénus",
         "gender": "F",
         "birthyear": 2003,
         "breed": "Jack Russel Terrier"
+      },
+      {
+        "id": 3,
+        "name": "Max",
+        "gender": "M",
+        "birthyear": 1995,
+        "breed": "Labrador Retriever"
       }
     ]
     "###);

crates/milli/src/search/new/ranking_rule_graph/fid/mod.rs

@@ -79,16 +79,27 @@ impl RankingRuleGraphTrait for FidGraph {
 
         let mut edges = vec![];
         for fid in all_fields.iter().copied() {
-            let weight = weights_map
+            match weights_map.weight(fid) {
-                .weight(fid)
+                Some(weight) => {
-                .ok_or(InternalError::FieldidsWeightsMapMissingEntry { key: fid })?;
+                    if weight > current_max_weight {
-            if weight > current_max_weight {
+                        current_max_weight = weight;
-                current_max_weight = weight;
+                    }
+
+                    edges.push((
+                        weight as u32 * term.term_ids.len() as u32,
+                        conditions_interner
+                            .insert(FidCondition { term: term.clone(), fid: Some(fid) }),
+                    ));
+                }
+                None => {
+                    // Hotfix: this is a temporary solution to handle the case where the weight is not found in the weights map.
+                    // This is due to a database corruption in word_fid_docids database.
+                    tracing::warn!(
+                        "{:?}",
+                        InternalError::FieldidsWeightsMapMissingEntry { key: fid }
+                    );
+                }
             }
-            edges.push((
-                weight as u32 * term.term_ids.len() as u32,
-                conditions_interner.insert(FidCondition { term: term.clone(), fid: Some(fid) }),
-            ));
         }
 
         // always lookup the max_fid if we don't already and add an artificial condition for max scoring