fmt

Add chat-related metrics on the prometheus route

.github/workflows/db-change-missing.yml

@@ -4,22 +4,22 @@ on:
   pull_request:
     types: [opened, synchronize, reopened, labeled, unlabeled]
 
-env:
-  GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-
 jobs:
   check-labels:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Check db change labels
         id: check_labels
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           URL=/repos/meilisearch/meilisearch/pulls/${{ github.event.pull_request.number }}/labels
           echo ${{ github.event.pull_request.number }}
           echo $URL
-          LABELS=$(gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" /repos/meilisearch/meilisearch/issues/${{ github.event.pull_request.number }}/labels -q .[].name)
+          LABELS=$(gh api -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" /repos/${{ github.repository }}/issues/${{ github.event.pull_request.number }}/labels -q .[].name)
+          echo "Labels: $LABELS"
           if [[ ! "$LABELS" =~ "db change" && ! "$LABELS" =~ "no db change" ]]; then
             echo "::error::Pull request must contain either the 'db change' or 'no db change' label."
             exit 1

.github/workflows/publish-apt-brew-pkg.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Build deb package
         run: cargo deb -p meilisearch -o target/debian/meilisearch.deb
       - name: Upload debian pkg to release
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.11.1
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/debian/meilisearch.deb

.github/workflows/publish-binaries.yml

@@ -51,7 +51,7 @@ jobs:
       # No need to upload binaries for dry run (cron)
       - name: Upload binaries to release
         if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.11.1
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/release/meilisearch
@@ -81,7 +81,7 @@ jobs:
       # No need to upload binaries for dry run (cron)
       - name: Upload binaries to release
         if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.11.1
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/release/${{ matrix.artifact_name }}
@@ -113,7 +113,7 @@ jobs:
       - name: Upload the binary to release
         # No need to upload binaries for dry run (cron)
         if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.11.1
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/${{ matrix.target }}/release/meilisearch
@@ -178,7 +178,7 @@ jobs:
       - name: Upload the binary to release
         # No need to upload binaries for dry run (cron)
         if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.11.1
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/${{ matrix.target }}/release/meilisearch

.github/workflows/test-suite.yml

@@ -29,7 +29,7 @@ jobs:
       - name: Setup test with Rust stable
         uses: dtolnay/rust-toolchain@1.85
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.8
+        uses: Swatinem/rust-cache@v2.8.0
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -51,7 +51,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.8
+        uses: Swatinem/rust-cache@v2.8.0
       - uses: dtolnay/rust-toolchain@1.85
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
@@ -155,7 +155,7 @@ jobs:
           apt-get install build-essential -y
       - uses: dtolnay/rust-toolchain@1.85
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.8
+        uses: Swatinem/rust-cache@v2.8.0
       - name: Run tests in debug
         uses: actions-rs/cargo@v1
         with:
@@ -172,7 +172,7 @@ jobs:
           profile: minimal
           components: clippy
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.8
+        uses: Swatinem/rust-cache@v2.8.0
       - name: Run cargo clippy
         uses: actions-rs/cargo@v1
         with:
@@ -191,7 +191,7 @@ jobs:
           override: true
           components: rustfmt
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.8
+        uses: Swatinem/rust-cache@v2.8.0
       - name: Run cargo fmt
         # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
         # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate

.gitignore

@@ -18,5 +18,8 @@
 ##  ... unreviewed
 *.snap.new
 
+# Database snapshot
+crates/meilisearch/db.snapshot
+
 # Fuzzcheck data for the facet indexing fuzz test
 crates/milli/fuzz/update::facet::incremental::fuzz::fuzz/

CONTRIBUTING.md

@@ -57,9 +57,17 @@ This command will be triggered to each PR as a requirement for merging it.
 You can set the `LINDERA_CACHE` environment variable to speed up your successive builds by up to 2 minutes.
 It'll store some built artifacts in the directory of your choice.
 
-We recommend using the standard `$HOME/.cache/lindera` directory:
+We recommend using the `$HOME/.cache/meili/lindera` directory:
 ```sh
-export LINDERA_CACHE=$HOME/.cache/lindera
+export LINDERA_CACHE=$HOME/.cache/meili/lindera
+```
+
+You can set the `MILLI_BENCH_DATASETS_PATH` environment variable to further speed up your builds.
+It'll store some big files used for the benchmarks in the directory of your choice.
+
+We recommend using the `$HOME/.cache/meili/benches` directory:
+```sh
+export MILLI_BENCH_DATASETS_PATH=$HOME/.cache/meili/benches
 ```
 
 Furthermore, you can improve incremental compilation by setting the `MEILI_NO_VERGEN` environment variable.

Cargo.toml

@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.15.0"
+version = "1.16.0"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",

crates/benchmarks/Cargo.toml

@@ -11,27 +11,27 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.95"
-bumpalo = "3.16.0"
+anyhow = "1.0.98"
+bumpalo = "3.18.1"
 csv = "1.3.1"
 memmap2 = "0.9.5"
 milli = { path = "../milli" }
-mimalloc = { version = "0.1.43", default-features = false }
-serde_json = { version = "1.0.135", features = ["preserve_order"] }
-tempfile = "3.15.0"
+mimalloc = { version = "0.1.47", default-features = false }
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
+tempfile = "3.20.0"
 
 [dev-dependencies]
-criterion = { version = "0.5.1", features = ["html_reports"] }
+criterion = { version = "0.6.0", features = ["html_reports"] }
 rand = "0.8.5"
 rand_chacha = "0.3.1"
-roaring = "0.10.10"
+roaring = "0.10.12"
 
 [build-dependencies]
-anyhow = "1.0.95"
-bytes = "1.9.0"
-convert_case = "0.6.0"
-flate2 = "1.0.35"
-reqwest = { version = "0.12.12", features = ["blocking", "rustls-tls"], default-features = false }
+anyhow = "1.0.98"
+bytes = "1.10.1"
+convert_case = "0.8.0"
+flate2 = "1.1.2"
+reqwest = { version = "0.12.20", features = ["blocking", "rustls-tls"], default-features = false }
 
 [features]
 default = ["milli/all-tokenizations"]

crates/benchmarks/benches/indexing.rs

@@ -11,7 +11,7 @@ use milli::heed::{EnvOpenOptions, RwTxn};
 use milli::progress::Progress;
 use milli::update::new::indexer;
 use milli::update::{IndexerConfig, Settings};
-use milli::vector::EmbeddingConfigs;
+use milli::vector::RuntimeEmbedders;
 use milli::{FilterableAttributesRule, Index};
 use rand::seq::SliceRandom;
 use rand_chacha::rand_core::SeedableRng;
@@ -65,7 +65,7 @@ fn setup_settings<'t>(
     let sortable_fields = sortable_fields.iter().map(|s| s.to_string()).collect();
     builder.set_sortable_fields(sortable_fields);
 
-    builder.execute(|_| (), || false).unwrap();
+    builder.execute(&|| false, &Progress::default(), Default::default()).unwrap();
 }
 
 fn setup_index_with_settings(
@@ -166,9 +166,10 @@ fn indexing_songs_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -232,9 +233,10 @@ fn reindexing_songs_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -276,9 +278,10 @@ fn reindexing_songs_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -344,9 +347,10 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -420,9 +424,10 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -464,9 +469,10 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -504,9 +510,10 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -571,9 +578,10 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -637,9 +645,10 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -703,9 +712,10 @@ fn indexing_wiki(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -768,9 +778,10 @@ fn reindexing_wiki(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -812,9 +823,10 @@ fn reindexing_wiki(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -879,9 +891,10 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -955,9 +968,10 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1000,9 +1014,10 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1041,9 +1056,10 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1107,9 +1123,10 @@ fn indexing_movies_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1172,9 +1189,10 @@ fn reindexing_movies_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1216,9 +1234,10 @@ fn reindexing_movies_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1283,9 +1302,10 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1331,9 +1351,10 @@ fn delete_documents_from_ids(index: Index, document_ids_to_delete: Vec<RoaringBi
             new_fields_ids_map,
             Some(primary_key),
             &document_changes,
-            EmbeddingConfigs::default(),
+            RuntimeEmbedders::default(),
             &|| false,
             &Progress::default(),
+            &Default::default(),
         )
         .unwrap();
 
@@ -1395,9 +1416,10 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1439,9 +1461,10 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1479,9 +1502,10 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1568,9 +1592,10 @@ fn indexing_nested_movies_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1658,9 +1683,10 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1740,9 +1766,10 @@ fn indexing_nested_movies_without_faceted_fields(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1806,9 +1833,10 @@ fn indexing_geo(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1871,9 +1899,10 @@ fn reindexing_geo(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1915,9 +1944,10 @@ fn reindexing_geo(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 
@@ -1982,9 +2012,10 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) {
                     new_fields_ids_map,
                     primary_key,
                     &document_changes,
-                    EmbeddingConfigs::default(),
+                    RuntimeEmbedders::default(),
                     &|| false,
                     &Progress::default(),
+                    &Default::default(),
                 )
                 .unwrap();
 

crates/benchmarks/benches/search_geo.rs

@@ -2,7 +2,8 @@ mod datasets_paths;
 mod utils;
 
 use criterion::{criterion_group, criterion_main};
-use milli::{update::Settings, FilterableAttributesRule};
+use milli::update::Settings;
+use milli::FilterableAttributesRule;
 use utils::Conf;
 
 #[cfg(not(windows))]

crates/benchmarks/benches/search_songs.rs

@@ -2,7 +2,8 @@ mod datasets_paths;
 mod utils;
 
 use criterion::{criterion_group, criterion_main};
-use milli::{update::Settings, FilterableAttributesRule};
+use milli::update::Settings;
+use milli::FilterableAttributesRule;
 use utils::Conf;
 
 #[cfg(not(windows))]

crates/benchmarks/benches/utils.rs

@@ -13,7 +13,7 @@ use milli::heed::EnvOpenOptions;
 use milli::progress::Progress;
 use milli::update::new::indexer;
 use milli::update::{IndexerConfig, Settings};
-use milli::vector::EmbeddingConfigs;
+use milli::vector::RuntimeEmbedders;
 use milli::{Criterion, Filter, Index, Object, TermsMatchingStrategy};
 use serde_json::Value;
 
@@ -90,7 +90,7 @@ pub fn base_setup(conf: &Conf) -> Index {
 
     (conf.configure)(&mut builder);
 
-    builder.execute(|_| (), || false).unwrap();
+    builder.execute(&|| false, &Progress::default(), Default::default()).unwrap();
     wtxn.commit().unwrap();
 
     let config = IndexerConfig::default();
@@ -125,9 +125,10 @@ pub fn base_setup(conf: &Conf) -> Index {
         new_fields_ids_map,
         primary_key,
         &document_changes,
-        EmbeddingConfigs::default(),
+        RuntimeEmbedders::default(),
         &|| false,
         &Progress::default(),
+        &Default::default(),
     )
     .unwrap();
 

crates/benchmarks/build.rs

@@ -67,7 +67,7 @@ fn main() -> anyhow::Result<()> {
         writeln!(
             &mut manifest_paths_file,
             r#"pub const {}: &str = {:?};"#,
-            dataset.to_case(Case::ScreamingSnake),
+            dataset.to_case(Case::UpperSnake),
             out_file.display(),
         )?;
 

crates/build-info/Cargo.toml

@@ -11,8 +11,8 @@ license.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-time = { version = "0.3.37", features = ["parsing"] }
+time = { version = "0.3.41", features = ["parsing"] }
 
 [build-dependencies]
-anyhow = "1.0.95"
-vergen-git2 = "1.0.2"
+anyhow = "1.0.98"
+vergen-git2 = "1.0.7"

crates/dump/Cargo.toml

@@ -11,21 +11,21 @@ readme.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.95"
-flate2 = "1.0.35"
-http = "1.2.0"
+anyhow = "1.0.98"
+flate2 = "1.1.2"
+http = "1.3.1"
 meilisearch-types = { path = "../meilisearch-types" }
-once_cell = "1.20.2"
+once_cell = "1.21.3"
 regex = "1.11.1"
-roaring = { version = "0.10.10", features = ["serde"] }
-serde = { version = "1.0.217", features = ["derive"] }
-serde_json = { version = "1.0.135", features = ["preserve_order"] }
-tar = "0.4.43"
-tempfile = "3.15.0"
-thiserror = "2.0.9"
-time = { version = "0.3.37", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+roaring = { version = "0.10.12", features = ["serde"] }
+serde = { version = "1.0.219", features = ["derive"] }
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
+tar = "0.4.44"
+tempfile = "3.20.0"
+thiserror = "2.0.12"
+time = { version = "0.3.41", features = ["serde-well-known", "formatting", "parsing", "macros"] }
 tracing = "0.1.41"
-uuid = { version = "1.11.0", features = ["serde", "v4"] }
+uuid = { version = "1.17.0", features = ["serde", "v4"] }
 
 [dev-dependencies]
 big_s = "1.0.2"

crates/dump/src/lib.rs

@@ -1,12 +1,17 @@
 #![allow(clippy::type_complexity)]
 #![allow(clippy::wrong_self_convention)]
 
+use std::collections::BTreeMap;
+
 use meilisearch_types::batches::BatchId;
+use meilisearch_types::byte_unit::Byte;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::keys::Key;
 use meilisearch_types::milli::update::IndexDocumentsMethod;
 use meilisearch_types::settings::Unchecked;
-use meilisearch_types::tasks::{Details, IndexSwap, KindWithContent, Status, Task, TaskId};
+use meilisearch_types::tasks::{
+    Details, ExportIndexSettings, IndexSwap, KindWithContent, Status, Task, TaskId,
+};
 use meilisearch_types::InstanceUid;
 use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize};
@@ -141,6 +146,12 @@ pub enum KindDump {
         instance_uid: Option<InstanceUid>,
     },
     SnapshotCreation,
+    Export {
+        url: String,
+        api_key: Option<String>,
+        payload_size: Option<Byte>,
+        indexes: BTreeMap<String, ExportIndexSettings>,
+    },
     UpgradeDatabase {
         from: (u32, u32, u32),
     },
@@ -213,6 +224,15 @@ impl From<KindWithContent> for KindDump {
                 KindDump::DumpCreation { keys, instance_uid }
             }
             KindWithContent::SnapshotCreation => KindDump::SnapshotCreation,
+            KindWithContent::Export { url, api_key, payload_size, indexes } => KindDump::Export {
+                url,
+                api_key,
+                payload_size,
+                indexes: indexes
+                    .into_iter()
+                    .map(|(pattern, settings)| (pattern.to_string(), settings))
+                    .collect(),
+            },
             KindWithContent::UpgradeDatabase { from: version } => {
                 KindDump::UpgradeDatabase { from: version }
             }
@@ -305,6 +325,7 @@ pub(crate) mod test {
             localized_attributes: Setting::NotSet,
             facet_search: Setting::NotSet,
             prefix_search: Setting::NotSet,
+            chat: Setting::NotSet,
             _kind: std::marker::PhantomData,
         };
         settings.check()
@@ -328,6 +349,7 @@ pub(crate) mod test {
                 write_channel_congestion: None,
                 internal_database_sizes: Default::default(),
             },
+            embedder_stats: Default::default(),
             enqueued_at: Some(BatchEnqueuedAt {
                 earliest: datetime!(2022-11-11 0:00 UTC),
                 oldest: datetime!(2022-11-11 0:00 UTC),

crates/dump/src/reader/compat/v5_to_v6.rs

@@ -405,6 +405,7 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
             search_cutoff_ms: v6::Setting::NotSet,
             facet_search: v6::Setting::NotSet,
             prefix_search: v6::Setting::NotSet,
+            chat: v6::Setting::NotSet,
             _kind: std::marker::PhantomData,
         }
     }

crates/dump/src/reader/mod.rs

@@ -116,6 +116,15 @@ impl DumpReader {
         }
     }
 
+    pub fn chat_completions_settings(
+        &mut self,
+    ) -> Result<Box<dyn Iterator<Item = Result<(String, v6::ChatCompletionSettings)>> + '_>> {
+        match self {
+            DumpReader::Current(current) => current.chat_completions_settings(),
+            DumpReader::Compat(_compat) => Ok(Box::new(std::iter::empty())),
+        }
+    }
+
     pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
         match self {
             DumpReader::Current(current) => Ok(current.features()),

crates/dump/src/reader/v6/mod.rs

@@ -1,3 +1,4 @@
+use std::ffi::OsStr;
 use std::fs::{self, File};
 use std::io::{BufRead, BufReader, ErrorKind};
 use std::path::Path;
@@ -21,6 +22,7 @@ pub type Unchecked = meilisearch_types::settings::Unchecked;
 pub type Task = crate::TaskDump;
 pub type Batch = meilisearch_types::batches::Batch;
 pub type Key = meilisearch_types::keys::Key;
+pub type ChatCompletionSettings = meilisearch_types::features::ChatCompletionSettings;
 pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures;
 pub type Network = meilisearch_types::features::Network;
 
@@ -192,6 +194,34 @@ impl V6Reader {
         )
     }
 
+    pub fn chat_completions_settings(
+        &mut self,
+    ) -> Result<Box<dyn Iterator<Item = Result<(String, ChatCompletionSettings)>> + '_>> {
+        let entries = match fs::read_dir(self.dump.path().join("chat-completions-settings")) {
+            Ok(entries) => entries,
+            Err(e) if e.kind() == ErrorKind::NotFound => return Ok(Box::new(std::iter::empty())),
+            Err(e) => return Err(e.into()),
+        };
+        Ok(Box::new(
+            entries
+                .map(|entry| -> Result<Option<_>> {
+                    let entry = entry?;
+                    let file_name = entry.file_name();
+                    let path = Path::new(&file_name);
+                    if entry.file_type()?.is_file() && path.extension() == Some(OsStr::new("json"))
+                    {
+                        let name = path.file_stem().unwrap().to_str().unwrap().to_string();
+                        let file = File::open(entry.path())?;
+                        let settings = serde_json::from_reader(file)?;
+                        Ok(Some((name, settings)))
+                    } else {
+                        Ok(None)
+                    }
+                })
+                .filter_map(|entry| entry.transpose()),
+        ))
+    }
+
     pub fn features(&self) -> Option<RuntimeTogglableFeatures> {
         self.features
     }

crates/dump/src/writer.rs

@@ -5,7 +5,7 @@ use std::path::PathBuf;
 use flate2::write::GzEncoder;
 use flate2::Compression;
 use meilisearch_types::batches::Batch;
-use meilisearch_types::features::{Network, RuntimeTogglableFeatures};
+use meilisearch_types::features::{ChatCompletionSettings, Network, RuntimeTogglableFeatures};
 use meilisearch_types::keys::Key;
 use meilisearch_types::settings::{Checked, Settings};
 use serde_json::{Map, Value};
@@ -51,6 +51,10 @@ impl DumpWriter {
         KeyWriter::new(self.dir.path().to_path_buf())
     }
 
+    pub fn create_chat_completions_settings(&self) -> Result<ChatCompletionsSettingsWriter> {
+        ChatCompletionsSettingsWriter::new(self.dir.path().join("chat-completions-settings"))
+    }
+
     pub fn create_tasks_queue(&self) -> Result<TaskWriter> {
         TaskWriter::new(self.dir.path().join("tasks"))
     }
@@ -104,6 +108,24 @@ impl KeyWriter {
     }
 }
 
+pub struct ChatCompletionsSettingsWriter {
+    path: PathBuf,
+}
+
+impl ChatCompletionsSettingsWriter {
+    pub(crate) fn new(path: PathBuf) -> Result<Self> {
+        std::fs::create_dir(&path)?;
+        Ok(ChatCompletionsSettingsWriter { path })
+    }
+
+    pub fn push_settings(&mut self, name: &str, settings: &ChatCompletionSettings) -> Result<()> {
+        let mut settings_file = File::create(self.path.join(name).with_extension("json"))?;
+        serde_json::to_writer(&mut settings_file, &settings)?;
+        settings_file.flush()?;
+        Ok(())
+    }
+}
+
 pub struct TaskWriter {
     queue: BufWriter<File>,
     update_files: PathBuf,

crates/file-store/Cargo.toml

@@ -11,7 +11,7 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-tempfile = "3.15.0"
-thiserror = "2.0.9"
+tempfile = "3.20.0"
+thiserror = "2.0.12"
 tracing = "0.1.41"
-uuid = { version = "1.11.0", features = ["serde", "v4"] }
+uuid = { version = "1.17.0", features = ["serde", "v4"] }

crates/filter-parser/Cargo.toml

@@ -14,7 +14,7 @@ license.workspace = true
 [dependencies]
 nom = "7.1.3"
 nom_locate = "4.2.0"
-unescaper = "0.1.5"
+unescaper = "0.1.6"
 
 [dev-dependencies]
 # fixed version due to format breakages in v1.40

crates/flatten-serde-json/Cargo.toml

@@ -16,7 +16,7 @@ license.workspace = true
 serde_json = "1.0"
 
 [dev-dependencies]
-criterion = { version = "0.5.1", features = ["html_reports"] }
+criterion = { version = "0.6.0", features = ["html_reports"] }
 
 [[bench]]
 name = "benchmarks"

crates/fuzzers/Cargo.toml

@@ -12,11 +12,11 @@ license.workspace = true
 
 [dependencies]
 arbitrary = { version = "1.4.1", features = ["derive"] }
-bumpalo = "3.16.0"
-clap = { version = "4.5.24", features = ["derive"] }
-either = "1.13.0"
+bumpalo = "3.18.1"
+clap = { version = "4.5.40", features = ["derive"] }
+either = "1.15.0"
 fastrand = "2.3.0"
 milli = { path = "../milli" }
-serde = { version = "1.0.217", features = ["derive"] }
-serde_json = { version = "1.0.135", features = ["preserve_order"] }
-tempfile = "3.15.0"
+serde = { version = "1.0.219", features = ["derive"] }
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
+tempfile = "3.20.0"

crates/fuzzers/src/bin/fuzz-indexing.rs

@@ -13,7 +13,7 @@ use milli::heed::EnvOpenOptions;
 use milli::progress::Progress;
 use milli::update::new::indexer;
 use milli::update::IndexerConfig;
-use milli::vector::EmbeddingConfigs;
+use milli::vector::RuntimeEmbedders;
 use milli::Index;
 use serde_json::Value;
 use tempfile::TempDir;
@@ -89,7 +89,7 @@ fn main() {
                             let mut new_fields_ids_map = db_fields_ids_map.clone();
 
                             let indexer_alloc = Bump::new();
-                            let embedders = EmbeddingConfigs::default();
+                            let embedders = RuntimeEmbedders::default();
                             let mut indexer = indexer::DocumentOperation::new();
 
                             let mut operations = Vec::new();
@@ -144,6 +144,7 @@ fn main() {
                                 embedders,
                                 &|| false,
                                 &Progress::default(),
+                                &Default::default(),
                             )
                             .unwrap();
 

crates/index-scheduler/Cargo.toml

@@ -11,31 +11,31 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.95"
+anyhow = "1.0.98"
 bincode = "1.3.3"
 byte-unit = "5.1.6"
-bumpalo = "3.16.0"
+bumpalo = "3.18.1"
 bumparaw-collections = "0.1.4"
-convert_case = "0.6.0"
+convert_case = "0.8.0"
 csv = "1.3.1"
 derive_builder = "0.20.2"
 dump = { path = "../dump" }
 enum-iterator = "2.1.0"
 file-store = { path = "../file-store" }
-flate2 = "1.0.35"
-indexmap = "2.7.0"
+flate2 = "1.1.2"
+indexmap = "2.9.0"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
 memmap2 = "0.9.5"
 page_size = "0.6.0"
 rayon = "1.10.0"
-roaring = { version = "0.10.10", features = ["serde"] }
-serde = { version = "1.0.217", features = ["derive"] }
-serde_json = { version = "1.0.138", features = ["preserve_order"] }
+roaring = { version = "0.10.12", features = ["serde"] }
+serde = { version = "1.0.219", features = ["derive"] }
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
 synchronoise = "1.0.1"
-tempfile = "3.15.0"
-thiserror = "2.0.9"
-time = { version = "0.3.37", features = [
+tempfile = "3.20.0"
+thiserror = "2.0.12"
+time = { version = "0.3.41", features = [
     "serde-well-known",
     "formatting",
     "parsing",
@@ -43,7 +43,8 @@ time = { version = "0.3.37", features = [
 ] }
 tracing = "0.1.41"
 ureq = "2.12.1"
-uuid = { version = "1.11.0", features = ["serde", "v4"] }
+uuid = { version = "1.17.0", features = ["serde", "v4"] }
+backoff = "0.4.0"
 
 [dev-dependencies]
 big_s = "1.0.2"

crates/index-scheduler/src/dump.rs

@@ -4,6 +4,7 @@ use std::io;
 use dump::{KindDump, TaskDump, UpdateFile};
 use meilisearch_types::batches::{Batch, BatchId};
 use meilisearch_types::heed::RwTxn;
+use meilisearch_types::index_uid_pattern::IndexUidPattern;
 use meilisearch_types::milli;
 use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
 use roaring::RoaringBitmap;
@@ -211,6 +212,23 @@ impl<'a> Dump<'a> {
                     KindWithContent::DumpCreation { keys, instance_uid }
                 }
                 KindDump::SnapshotCreation => KindWithContent::SnapshotCreation,
+                KindDump::Export { url, api_key, payload_size, indexes } => {
+                    KindWithContent::Export {
+                        url,
+                        api_key,
+                        payload_size,
+                        indexes: indexes
+                            .into_iter()
+                            .map(|(pattern, settings)| {
+                                Ok((
+                                    IndexUidPattern::try_from(pattern)
+                                        .map_err(|_| Error::CorruptedDump)?,
+                                    settings,
+                                ))
+                            })
+                            .collect::<Result<_, Error>>()?,
+                    }
+                }
                 KindDump::UpgradeDatabase { from } => KindWithContent::UpgradeDatabase { from },
             },
         };

crates/index-scheduler/src/error.rs

@@ -151,6 +151,10 @@ pub enum Error {
     CorruptedTaskQueue,
     #[error(transparent)]
     DatabaseUpgrade(Box<Self>),
+    #[error(transparent)]
+    Export(Box<Self>),
+    #[error("Failed to export documents to remote server {code} ({type}): {message} <{link}>")]
+    FromRemoteWhenExporting { message: String, code: String, r#type: String, link: String },
     #[error("Failed to rollback for index `{index}`: {rollback_outcome} ")]
     RollbackFailed { index: String, rollback_outcome: RollbackOutcome },
     #[error(transparent)]
@@ -212,6 +216,7 @@ impl Error {
             | Error::BatchNotFound(_)
             | Error::TaskDeletionWithEmptyQuery
             | Error::TaskCancelationWithEmptyQuery
+            | Error::FromRemoteWhenExporting { .. }
             | Error::AbortedTask
             | Error::Dump(_)
             | Error::Heed(_)
@@ -221,6 +226,7 @@ impl Error {
             | Error::IoError(_)
             | Error::Persist(_)
             | Error::FeatureNotEnabled(_)
+            | Error::Export(_)
             | Error::Anyhow(_) => true,
             Error::CreateBatch(_)
             | Error::CorruptedTaskQueue
@@ -282,6 +288,7 @@ impl ErrorCode for Error {
             Error::Dump(e) => e.error_code(),
             Error::Milli { error, .. } => error.error_code(),
             Error::ProcessBatchPanicked(_) => Code::Internal,
+            Error::FromRemoteWhenExporting { .. } => Code::Internal,
             Error::Heed(e) => e.error_code(),
             Error::HeedTransaction(e) => e.error_code(),
             Error::FileStore(e) => e.error_code(),
@@ -294,6 +301,7 @@ impl ErrorCode for Error {
             Error::CorruptedTaskQueue => Code::Internal,
             Error::CorruptedDump => Code::Internal,
             Error::DatabaseUpgrade(_) => Code::Internal,
+            Error::Export(_) => Code::Internal,
             Error::RollbackFailed { .. } => Code::Internal,
             Error::UnrecoverableError(_) => Code::Internal,
             Error::IndexSchedulerVersionMismatch { .. } => Code::Internal,

crates/index-scheduler/src/features.rs

@@ -131,6 +131,32 @@ impl RoFeatures {
             .into())
         }
     }
+
+    pub fn check_chat_completions(&self, disabled_action: &'static str) -> Result<()> {
+        if self.runtime.chat_completions {
+            Ok(())
+        } else {
+            Err(FeatureNotEnabledError {
+                disabled_action,
+                feature: "chat completions",
+                issue_link: "https://github.com/orgs/meilisearch/discussions/835",
+            }
+            .into())
+        }
+    }
+
+    pub fn check_multimodal(&self, disabled_action: &'static str) -> Result<()> {
+        if self.runtime.multimodal {
+            Ok(())
+        } else {
+            Err(FeatureNotEnabledError {
+                disabled_action,
+                feature: "multimodal",
+                issue_link: "https://github.com/orgs/meilisearch/discussions/846",
+            }
+            .into())
+        }
+    }
 }
 
 impl FeatureData {

crates/index-scheduler/src/insta_snapshot.rs

@@ -34,6 +34,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         planned_failures: _,
         run_loop_iteration: _,
         embedders: _,
+        chat_settings: _,
     } = scheduler;
 
     let rtxn = env.read_txn().unwrap();
@@ -288,6 +289,9 @@ fn snapshot_details(d: &Details) -> String {
         Details::IndexSwap { swaps } => {
             format!("{{ swaps: {swaps:?} }}")
         }
+        Details::Export { url, api_key, payload_size, indexes } => {
+            format!("{{ url: {url:?}, api_key: {api_key:?}, payload_size: {payload_size:?}, indexes: {indexes:?} }}")
+        }
         Details::UpgradeDatabase { from, to } => {
             format!("{{ from: {from:?}, to: {to:?} }}")
         }
@@ -342,6 +346,7 @@ pub fn snapshot_batch(batch: &Batch) -> String {
         uid,
         details,
         stats,
+        embedder_stats,
         started_at,
         finished_at,
         progress: _,
@@ -365,6 +370,12 @@ pub fn snapshot_batch(batch: &Batch) -> String {
     snap.push_str(&format!("uid: {uid}, "));
     snap.push_str(&format!("details: {}, ", serde_json::to_string(details).unwrap()));
     snap.push_str(&format!("stats: {}, ", serde_json::to_string(&stats).unwrap()));
+    if !embedder_stats.skip_serializing() {
+        snap.push_str(&format!(
+            "embedder stats: {}, ",
+            serde_json::to_string(&embedder_stats).unwrap()
+        ));
+    }
     snap.push_str(&format!("stop reason: {}, ", serde_json::to_string(&stop_reason).unwrap()));
     snap.push('}');
     snap

crates/index-scheduler/src/lib.rs

@@ -51,16 +51,21 @@ pub use features::RoFeatures;
 use flate2::bufread::GzEncoder;
 use flate2::Compression;
 use meilisearch_types::batches::Batch;
-use meilisearch_types::features::{InstanceTogglableFeatures, Network, RuntimeTogglableFeatures};
+use meilisearch_types::features::{
+    ChatCompletionSettings, InstanceTogglableFeatures, Network, RuntimeTogglableFeatures,
+};
 use meilisearch_types::heed::byteorder::BE;
-use meilisearch_types::heed::types::I128;
-use meilisearch_types::heed::{self, Env, RoTxn, WithoutTls};
-use meilisearch_types::milli::index::IndexEmbeddingConfig;
+use meilisearch_types::heed::types::{DecodeIgnore, SerdeJson, Str, I128};
+use meilisearch_types::heed::{self, Database, Env, RoTxn, WithoutTls};
 use meilisearch_types::milli::update::IndexerConfig;
-use meilisearch_types::milli::vector::{Embedder, EmbedderOptions, EmbeddingConfigs};
+use meilisearch_types::milli::vector::json_template::JsonTemplate;
+use meilisearch_types::milli::vector::{
+    Embedder, EmbedderOptions, RuntimeEmbedder, RuntimeEmbedders, RuntimeFragment,
+};
 use meilisearch_types::milli::{self, Index};
 use meilisearch_types::task_view::TaskView;
 use meilisearch_types::tasks::{KindWithContent, Task};
+use milli::vector::db::IndexEmbeddingConfig;
 use processing::ProcessingTasks;
 pub use queue::Query;
 use queue::Queue;
@@ -75,6 +80,7 @@ use crate::utils::clamp_to_page_size;
 pub(crate) type BEI128 = I128<BE>;
 
 const TASK_SCHEDULER_SIZE_THRESHOLD_PERCENT_INT: u64 = 40;
+const CHAT_SETTINGS_DB_NAME: &str = "chat-settings";
 
 #[derive(Debug)]
 pub struct IndexSchedulerOptions {
@@ -153,6 +159,9 @@ pub struct IndexScheduler {
     /// In charge of fetching and setting the status of experimental features.
     features: features::FeatureData,
 
+    /// Stores the custom chat prompts and other settings of the indexes.
+    pub(crate) chat_settings: Database<Str, SerdeJson<ChatCompletionSettings>>,
+
     /// Everything related to the processing of the tasks
     pub scheduler: scheduler::Scheduler,
 
@@ -211,11 +220,16 @@ impl IndexScheduler {
             #[cfg(test)]
             run_loop_iteration: self.run_loop_iteration.clone(),
             features: self.features.clone(),
+            chat_settings: self.chat_settings,
         }
     }
 
     pub(crate) const fn nb_db() -> u32 {
-        Versioning::nb_db() + Queue::nb_db() + IndexMapper::nb_db() + features::FeatureData::nb_db()
+        Versioning::nb_db()
+            + Queue::nb_db()
+            + IndexMapper::nb_db()
+            + features::FeatureData::nb_db()
+            + 1 // chat-prompts
     }
 
     /// Create an index scheduler and start its run loop.
@@ -269,6 +283,7 @@ impl IndexScheduler {
         let features = features::FeatureData::new(&env, &mut wtxn, options.instance_features)?;
         let queue = Queue::new(&env, &mut wtxn, &options)?;
         let index_mapper = IndexMapper::new(&env, &mut wtxn, &options, budget)?;
+        let chat_settings = env.create_database(&mut wtxn, Some(CHAT_SETTINGS_DB_NAME))?;
         wtxn.commit()?;
 
         // allow unreachable_code to get rids of the warning in the case of a test build.
@@ -292,12 +307,17 @@ impl IndexScheduler {
             #[cfg(test)]
             run_loop_iteration: Arc::new(RwLock::new(0)),
             features,
+            chat_settings,
         };
 
         this.run();
         Ok(this)
     }
 
+    fn read_txn(&self) -> Result<RoTxn<WithoutTls>> {
+        self.env.read_txn().map_err(|e| e.into())
+    }
+
     /// Return `Ok(())` if the index scheduler is able to access one of its database.
     pub fn health(&self) -> Result<()> {
         let rtxn = self.env.read_txn()?;
@@ -374,10 +394,6 @@ impl IndexScheduler {
         }
     }
 
-    pub fn read_txn(&self) -> Result<RoTxn<WithoutTls>> {
-        self.env.read_txn().map_err(|e| e.into())
-    }
-
     /// Start the run loop for the given index scheduler.
     ///
     /// This function will execute in a different thread and must be called
@@ -495,7 +511,7 @@ impl IndexScheduler {
 
     /// Returns the total number of indexes available for the specified filter.
     /// And a `Vec` of the index_uid + its stats
-    pub fn get_paginated_indexes_stats(
+    pub fn paginated_indexes_stats(
         &self,
         filters: &meilisearch_auth::AuthFilter,
         from: usize,
@@ -536,6 +552,24 @@ impl IndexScheduler {
         ret.map(|ret| (total, ret))
     }
 
+    /// Returns the total number of chat workspaces available ~~for the specified filter~~.
+    /// And a `Vec` of the workspace_uids
+    pub fn paginated_chat_workspace_uids(
+        &self,
+        from: usize,
+        limit: usize,
+    ) -> Result<(usize, Vec<String>)> {
+        let rtxn = self.read_txn()?;
+        let total = self.chat_settings.len(&rtxn)?;
+        let mut iter = self.chat_settings.iter(&rtxn)?.skip(from);
+        iter.by_ref()
+            .take(limit)
+            .map(|ret| ret.map_err(Error::from))
+            .map(|ret| ret.map(|(uid, _)| uid.to_string()))
+            .collect::<Result<Vec<_>, Error>>()
+            .map(|ret| (total as usize, ret))
+    }
+
     /// The returned structure contains:
     /// 1. The name of the property being observed can be `statuses`, `types`, or `indexes`.
     /// 2. The name of the specific data related to the property can be `enqueued` for the `statuses`, `settingsUpdate` for the `types`, or the name of the index for the `indexes`, for example.
@@ -820,29 +854,42 @@ impl IndexScheduler {
         &self,
         index_uid: String,
         embedding_configs: Vec<IndexEmbeddingConfig>,
-    ) -> Result<EmbeddingConfigs> {
+    ) -> Result<RuntimeEmbedders> {
         let res: Result<_> = embedding_configs
             .into_iter()
             .map(
                 |IndexEmbeddingConfig {
                      name,
                      config: milli::vector::EmbeddingConfig { embedder_options, prompt, quantized },
-                     ..
-                 }| {
-                    let prompt = Arc::new(
-                        prompt
-                            .try_into()
-                            .map_err(meilisearch_types::milli::Error::from)
-                            .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?,
-                    );
+                     fragments,
+                 }|
+                 -> Result<(String, Arc<RuntimeEmbedder>)> {
+                    let document_template = prompt
+                        .try_into()
+                        .map_err(meilisearch_types::milli::Error::from)
+                        .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?;
+
+                    let fragments = fragments
+                        .into_inner()
+                        .into_iter()
+                        .map(|fragment| {
+                            let value = embedder_options.fragment(&fragment.name).unwrap();
+                            let template = JsonTemplate::new(value.clone()).unwrap();
+                            RuntimeFragment { name: fragment.name, id: fragment.id, template }
+                        })
+                        .collect();
                     // optimistically return existing embedder
                     {
                         let embedders = self.embedders.read().unwrap();
                         if let Some(embedder) = embedders.get(&embedder_options) {
-                            return Ok((
-                                name,
-                                (embedder.clone(), prompt, quantized.unwrap_or_default()),
+                            let runtime = Arc::new(RuntimeEmbedder::new(
+                                embedder.clone(),
+                                document_template,
+                                fragments,
+                                quantized.unwrap_or_default(),
                             ));
+
+                            return Ok((name, runtime));
                         }
                     }
 
@@ -858,11 +905,44 @@ impl IndexScheduler {
                         let mut embedders = self.embedders.write().unwrap();
                         embedders.insert(embedder_options, embedder.clone());
                     }
-                    Ok((name, (embedder, prompt, quantized.unwrap_or_default())))
+
+                    let runtime = Arc::new(RuntimeEmbedder::new(
+                        embedder.clone(),
+                        document_template,
+                        fragments,
+                        quantized.unwrap_or_default(),
+                    ));
+
+                    Ok((name, runtime))
                 },
             )
             .collect();
-        res.map(EmbeddingConfigs::new)
+        res.map(RuntimeEmbedders::new)
+    }
+
+    pub fn chat_settings(&self, uid: &str) -> Result<Option<ChatCompletionSettings>> {
+        let rtxn = self.env.read_txn()?;
+        self.chat_settings.get(&rtxn, uid).map_err(Into::into)
+    }
+
+    /// Return true if chat workspace exists.
+    pub fn chat_workspace_exists(&self, name: &str) -> Result<bool> {
+        let rtxn = self.env.read_txn()?;
+        Ok(self.chat_settings.remap_data_type::<DecodeIgnore>().get(&rtxn, name)?.is_some())
+    }
+
+    pub fn put_chat_settings(&self, uid: &str, settings: &ChatCompletionSettings) -> Result<()> {
+        let mut wtxn = self.env.write_txn()?;
+        self.chat_settings.put(&mut wtxn, uid, settings)?;
+        wtxn.commit()?;
+        Ok(())
+    }
+
+    pub fn delete_chat_settings(&self, uid: &str) -> Result<bool> {
+        let mut wtxn = self.env.write_txn()?;
+        let deleted = self.chat_settings.delete(&mut wtxn, uid)?;
+        wtxn.commit()?;
+        Ok(deleted)
     }
 }
 

crates/index-scheduler/src/processing.rs

@@ -103,6 +103,7 @@ make_enum_progress! {
     pub enum DumpCreationProgress {
         StartTheDumpCreation,
         DumpTheApiKeys,
+        DumpTheChatCompletionSettings,
         DumpTheTasks,
         DumpTheBatches,
         DumpTheIndexes,
@@ -175,8 +176,17 @@ make_enum_progress! {
     }
 }
 
+make_enum_progress! {
+    pub enum Export {
+        EnsuringCorrectnessOfTheTarget,
+        ExportingTheSettings,
+        ExportingTheDocuments,
+    }
+}
+
 make_atomic_progress!(Task alias AtomicTaskStep => "task" );
 make_atomic_progress!(Document alias AtomicDocumentStep => "document" );
+make_atomic_progress!(Index alias AtomicIndexStep => "index" );
 make_atomic_progress!(Batch alias AtomicBatchStep => "batch" );
 make_atomic_progress!(UpdateFile alias AtomicUpdateFileStep => "update file" );
 

crates/index-scheduler/src/queue/batches.rs

@@ -179,6 +179,7 @@ impl BatchQueue {
                 progress: None,
                 details: batch.details,
                 stats: batch.stats,
+                embedder_stats: batch.embedder_stats.as_ref().into(),
                 started_at: batch.started_at,
                 finished_at: batch.finished_at,
                 enqueued_at: batch.enqueued_at,

crates/index-scheduler/src/scheduler/autobatcher.rs

@@ -71,6 +71,7 @@ impl From<KindWithContent> for AutobatchKind {
             KindWithContent::TaskCancelation { .. }
             | KindWithContent::TaskDeletion { .. }
             | KindWithContent::DumpCreation { .. }
+            | KindWithContent::Export { .. }
             | KindWithContent::UpgradeDatabase { .. }
             | KindWithContent::SnapshotCreation => {
                 panic!("The autobatcher should never be called with tasks that don't apply to an index.")

crates/index-scheduler/src/scheduler/create_batch.rs

@@ -1,4 +1,5 @@
 use std::fmt;
+use std::io::ErrorKind;
 
 use meilisearch_types::heed::RoTxn;
 use meilisearch_types::milli::update::IndexDocumentsMethod;
@@ -47,6 +48,9 @@ pub(crate) enum Batch {
     IndexSwap {
         task: Task,
     },
+    Export {
+        task: Task,
+    },
     UpgradeDatabase {
         tasks: Vec<Task>,
     },
@@ -103,6 +107,7 @@ impl Batch {
             Batch::TaskCancelation { task, .. }
             | Batch::Dump(task)
             | Batch::IndexCreation { task, .. }
+            | Batch::Export { task }
             | Batch::IndexUpdate { task, .. } => {
                 RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
             }
@@ -142,6 +147,7 @@ impl Batch {
             | TaskDeletions(_)
             | SnapshotCreation(_)
             | Dump(_)
+            | Export { .. }
             | UpgradeDatabase { .. }
             | IndexSwap { .. } => None,
             IndexOperation { op, .. } => Some(op.index_uid()),
@@ -167,6 +173,7 @@ impl fmt::Display for Batch {
             Batch::IndexUpdate { .. } => f.write_str("IndexUpdate")?,
             Batch::IndexDeletion { .. } => f.write_str("IndexDeletion")?,
             Batch::IndexSwap { .. } => f.write_str("IndexSwap")?,
+            Batch::Export { .. } => f.write_str("Export")?,
             Batch::UpgradeDatabase { .. } => f.write_str("UpgradeDatabase")?,
         };
         match index_uid {
@@ -426,9 +433,10 @@ impl IndexScheduler {
     /// 0. We get the *last* task to cancel.
     /// 1. We get the tasks to upgrade.
     /// 2. We get the *next* task to delete.
-    /// 3. We get the *next* snapshot to process.
-    /// 4. We get the *next* dump to process.
-    /// 5. We get the *next* tasks to process for a specific index.
+    /// 3. We get the *next* export to process.
+    /// 4. We get the *next* snapshot to process.
+    /// 5. We get the *next* dump to process.
+    /// 6. We get the *next* tasks to process for a specific index.
     #[tracing::instrument(level = "trace", skip(self, rtxn), target = "indexing::scheduler")]
     pub(crate) fn create_next_batch(
         &self,
@@ -500,7 +508,17 @@ impl IndexScheduler {
             return Ok(Some((Batch::TaskDeletions(tasks), current_batch)));
         }
 
-        // 3. we batch the snapshot.
+        // 3. we batch the export.
+        let to_export = self.queue.tasks.get_kind(rtxn, Kind::Export)? & enqueued;
+        if !to_export.is_empty() {
+            let task_id = to_export.iter().next().expect("There must be at least one export task");
+            let mut task = self.queue.tasks.get_task(rtxn, task_id)?.unwrap();
+            current_batch.processing([&mut task]);
+            current_batch.reason(BatchStopReason::TaskKindCannotBeBatched { kind: Kind::Export });
+            return Ok(Some((Batch::Export { task }, current_batch)));
+        }
+
+        // 4. we batch the snapshot.
         let to_snapshot = self.queue.tasks.get_kind(rtxn, Kind::SnapshotCreation)? & enqueued;
         if !to_snapshot.is_empty() {
             let mut tasks = self.queue.tasks.get_existing_tasks(rtxn, to_snapshot)?;
@@ -510,7 +528,7 @@ impl IndexScheduler {
             return Ok(Some((Batch::SnapshotCreation(tasks), current_batch)));
         }
 
-        // 4. we batch the dumps.
+        // 5. we batch the dumps.
         let to_dump = self.queue.tasks.get_kind(rtxn, Kind::DumpCreation)? & enqueued;
         if let Some(to_dump) = to_dump.min() {
             let mut task =
@@ -523,7 +541,7 @@ impl IndexScheduler {
             return Ok(Some((Batch::Dump(task), current_batch)));
         }
 
-        // 5. We make a batch from the unprioritised tasks. Start by taking the next enqueued task.
+        // 6. We make a batch from the unprioritised tasks. Start by taking the next enqueued task.
         let task_id = if let Some(task_id) = enqueued.min() { task_id } else { return Ok(None) };
         let mut task =
             self.queue.tasks.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
@@ -577,7 +595,11 @@ impl IndexScheduler {
                 .and_then(|task| task.ok_or(Error::CorruptedTaskQueue))?;
 
             if let Some(uuid) = task.content_uuid() {
-                let content_size = self.queue.file_store.compute_size(uuid)?;
+                let content_size = match self.queue.file_store.compute_size(uuid) {
+                    Ok(content_size) => content_size,
+                    Err(file_store::Error::IoError(err)) if err.kind() == ErrorKind::NotFound => 0,
+                    Err(otherwise) => return Err(otherwise.into()),
+                };
                 total_size = total_size.saturating_add(content_size);
             }
 

crates/index-scheduler/src/scheduler/mod.rs

@@ -4,6 +4,7 @@ mod autobatcher_test;
 mod create_batch;
 mod process_batch;
 mod process_dump_creation;
+mod process_export;
 mod process_index_operation;
 mod process_snapshot_creation;
 mod process_upgrade;
@@ -237,7 +238,7 @@ impl IndexScheduler {
                 #[cfg(test)]
                 self.breakpoint(crate::test_utils::Breakpoint::ProcessBatchSucceeded);
 
-                let (task_progress, task_progress_obj) = AtomicTaskStep::new(tasks.len() as u32);
+                let (task_progress, task_progress_obj) = AtomicTaskStep::new(tasks.len() as u64);
                 progress.update_progress(task_progress_obj);
                 process_batch_info = info;
                 let mut success = 0;
@@ -316,7 +317,7 @@ impl IndexScheduler {
             Err(err) => {
                 #[cfg(test)]
                 self.breakpoint(crate::test_utils::Breakpoint::ProcessBatchFailed);
-                let (task_progress, task_progress_obj) = AtomicTaskStep::new(ids.len() as u32);
+                let (task_progress, task_progress_obj) = AtomicTaskStep::new(ids.len() as u64);
                 progress.update_progress(task_progress_obj);
 
                 if matches!(err, Error::DatabaseUpgrade(_)) {
@@ -375,9 +376,11 @@ impl IndexScheduler {
                 post_commit_dabases_sizes
                     .get(dbname)
                     .map(|post_size| {
-                        use byte_unit::{Byte, UnitType::Binary};
                         use std::cmp::Ordering::{Equal, Greater, Less};
 
+                        use byte_unit::Byte;
+                        use byte_unit::UnitType::Binary;
+
                         let post = Byte::from_u64(*post_size as u64).get_appropriate_unit(Binary);
                         let diff_size = post_size.abs_diff(*pre_size) as u64;
                         let diff = Byte::from_u64(diff_size).get_appropriate_unit(Binary);

crates/index-scheduler/src/scheduler/process_batch.rs

@@ -162,8 +162,13 @@ impl IndexScheduler {
                     .set_currently_updating_index(Some((index_uid.clone(), index.clone())));
 
                 let pre_commit_dabases_sizes = index.database_sizes(&index_wtxn)?;
-                let (tasks, congestion) =
-                    self.apply_index_operation(&mut index_wtxn, &index, op, &progress)?;
+                let (tasks, congestion) = self.apply_index_operation(
+                    &mut index_wtxn,
+                    &index,
+                    op,
+                    &progress,
+                    current_batch.embedder_stats.clone(),
+                )?;
 
                 {
                     progress.update_progress(FinalizingIndexStep::Committing);
@@ -238,10 +243,12 @@ impl IndexScheduler {
                     );
                     builder.set_primary_key(primary_key);
                     let must_stop_processing = self.scheduler.must_stop_processing.clone();
+
                     builder
                         .execute(
-                            |indexing_step| tracing::debug!(update = ?indexing_step),
-                            || must_stop_processing.get(),
+                            &|| must_stop_processing.get(),
+                            &progress,
+                            current_batch.embedder_stats.clone(),
                         )
                         .map_err(|e| Error::from_milli(e, Some(index_uid.to_string())))?;
                     index_wtxn.commit()?;
@@ -346,8 +353,8 @@ impl IndexScheduler {
                 for (step, swap) in swaps.iter().enumerate() {
                     progress.update_progress(VariableNameStep::<SwappingTheIndexes>::new(
                         format!("swapping index {} and {}", swap.indexes.0, swap.indexes.1),
-                        step as u32,
-                        swaps.len() as u32,
+                        step as u64,
+                        swaps.len() as u64,
                     ));
                     self.apply_index_swap(
                         &mut wtxn,
@@ -361,6 +368,46 @@ impl IndexScheduler {
                 task.status = Status::Succeeded;
                 Ok((vec![task], ProcessBatchInfo::default()))
             }
+            Batch::Export { mut task } => {
+                let KindWithContent::Export { url, api_key, payload_size, indexes } = &task.kind
+                else {
+                    unreachable!()
+                };
+
+                let ret = catch_unwind(AssertUnwindSafe(|| {
+                    self.process_export(
+                        url,
+                        api_key.as_deref(),
+                        payload_size.as_ref(),
+                        indexes,
+                        progress,
+                    )
+                }));
+
+                let stats = match ret {
+                    Ok(Ok(stats)) => stats,
+                    Ok(Err(Error::AbortedTask)) => return Err(Error::AbortedTask),
+                    Ok(Err(e)) => return Err(Error::Export(Box::new(e))),
+                    Err(e) => {
+                        let msg = match e.downcast_ref::<&'static str>() {
+                            Some(s) => *s,
+                            None => match e.downcast_ref::<String>() {
+                                Some(s) => &s[..],
+                                None => "Box<dyn Any>",
+                            },
+                        };
+                        return Err(Error::Export(Box::new(Error::ProcessBatchPanicked(
+                            msg.to_string(),
+                        ))));
+                    }
+                };
+
+                task.status = Status::Succeeded;
+                if let Some(Details::Export { indexes, .. }) = task.details.as_mut() {
+                    *indexes = stats;
+                }
+                Ok((vec![task], ProcessBatchInfo::default()))
+            }
             Batch::UpgradeDatabase { mut tasks } => {
                 let KindWithContent::UpgradeDatabase { from } = tasks.last().unwrap().kind else {
                     unreachable!();
@@ -425,7 +472,7 @@ impl IndexScheduler {
         // 3. before_name -> new_name in the task's KindWithContent
         progress.update_progress(InnerSwappingTwoIndexes::UpdateTheTasks);
         let tasks_to_update = &index_lhs_task_ids | &index_rhs_task_ids;
-        let (atomic, task_progress) = AtomicTaskStep::new(tasks_to_update.len() as u32);
+        let (atomic, task_progress) = AtomicTaskStep::new(tasks_to_update.len() as u64);
         progress.update_progress(task_progress);
 
         for task_id in tasks_to_update {
@@ -482,7 +529,7 @@ impl IndexScheduler {
         // The tasks that have been removed *per batches*.
         let mut affected_batches: HashMap<BatchId, RoaringBitmap> = HashMap::new();
 
-        let (atomic_progress, task_progress) = AtomicTaskStep::new(to_delete_tasks.len() as u32);
+        let (atomic_progress, task_progress) = AtomicTaskStep::new(to_delete_tasks.len() as u64);
         progress.update_progress(task_progress);
         for task_id in to_delete_tasks.iter() {
             let task =
@@ -528,7 +575,7 @@ impl IndexScheduler {
 
         progress.update_progress(TaskDeletionProgress::DeletingTasksMetadata);
         let (atomic_progress, task_progress) = AtomicTaskStep::new(
-            (affected_indexes.len() + affected_statuses.len() + affected_kinds.len()) as u32,
+            (affected_indexes.len() + affected_statuses.len() + affected_kinds.len()) as u64,
         );
         progress.update_progress(task_progress);
         for index in affected_indexes.iter() {
@@ -547,7 +594,7 @@ impl IndexScheduler {
         }
 
         progress.update_progress(TaskDeletionProgress::DeletingTasks);
-        let (atomic_progress, task_progress) = AtomicTaskStep::new(to_delete_tasks.len() as u32);
+        let (atomic_progress, task_progress) = AtomicTaskStep::new(to_delete_tasks.len() as u64);
         progress.update_progress(task_progress);
         for task in to_delete_tasks.iter() {
             self.queue.tasks.all_tasks.delete(wtxn, &task)?;
@@ -564,7 +611,7 @@ impl IndexScheduler {
             }
         }
         progress.update_progress(TaskDeletionProgress::DeletingBatches);
-        let (atomic_progress, batch_progress) = AtomicBatchStep::new(affected_batches.len() as u32);
+        let (atomic_progress, batch_progress) = AtomicBatchStep::new(affected_batches.len() as u64);
         progress.update_progress(batch_progress);
         for (batch_id, to_delete_tasks) in affected_batches {
             if let Some(mut tasks) = self.queue.batch_to_tasks_mapping.get(wtxn, &batch_id)? {
@@ -708,9 +755,11 @@ impl IndexScheduler {
                     from.1,
                     from.2
                 );
-                match std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
+                let ret = catch_unwind(std::panic::AssertUnwindSafe(|| {
                     self.process_rollback(from, progress)
-                })) {
+                }));
+
+                match ret {
                     Ok(Ok(())) => {}
                     Ok(Err(err)) => return Err(Error::DatabaseUpgrade(Box::new(err))),
                     Err(e) => {
@@ -737,7 +786,7 @@ impl IndexScheduler {
         }
 
         // 3. We now have a list of tasks to cancel, cancel them
-        let (task_progress, progress_obj) = AtomicTaskStep::new(tasks_to_cancel.len() as u32);
+        let (task_progress, progress_obj) = AtomicTaskStep::new(tasks_to_cancel.len() as u64);
         progress.update_progress(progress_obj);
 
         let mut tasks = self.queue.tasks.get_existing_tasks(
@@ -748,7 +797,7 @@ impl IndexScheduler {
         )?;
 
         progress.update_progress(TaskCancelationProgress::UpdatingTasks);
-        let (task_progress, progress_obj) = AtomicTaskStep::new(tasks_to_cancel.len() as u32);
+        let (task_progress, progress_obj) = AtomicTaskStep::new(tasks_to_cancel.len() as u64);
         progress.update_progress(progress_obj);
         for task in tasks.iter_mut() {
             task.status = Status::Canceled;

crates/index-scheduler/src/scheduler/process_dump_creation.rs

@@ -43,12 +43,21 @@ impl IndexScheduler {
 
         let rtxn = self.env.read_txn()?;
 
-        // 2. dump the tasks
+        // 2. dump the chat completion settings
+        // TODO should I skip the export if the chat completion has been disabled?
+        progress.update_progress(DumpCreationProgress::DumpTheChatCompletionSettings);
+        let mut dump_chat_completion_settings = dump.create_chat_completions_settings()?;
+        for result in self.chat_settings.iter(&rtxn)? {
+            let (name, chat_settings) = result?;
+            dump_chat_completion_settings.push_settings(name, &chat_settings)?;
+        }
+
+        // 3. dump the tasks
         progress.update_progress(DumpCreationProgress::DumpTheTasks);
         let mut dump_tasks = dump.create_tasks_queue()?;
 
         let (atomic, update_task_progress) =
-            AtomicTaskStep::new(self.queue.tasks.all_tasks.len(&rtxn)? as u32);
+            AtomicTaskStep::new(self.queue.tasks.all_tasks.len(&rtxn)? as u64);
         progress.update_progress(update_task_progress);
 
         for ret in self.queue.tasks.all_tasks.iter(&rtxn)? {
@@ -81,7 +90,7 @@ impl IndexScheduler {
 
             let mut dump_content_file = dump_tasks.push_task(&t.into())?;
 
-            // 2.1. Dump the `content_file` associated with the task if there is one and the task is not finished yet.
+            // 3.1. Dump the `content_file` associated with the task if there is one and the task is not finished yet.
             if let Some(content_file) = content_file {
                 if self.scheduler.must_stop_processing.get() {
                     return Err(Error::AbortedTask);
@@ -105,12 +114,12 @@ impl IndexScheduler {
         }
         dump_tasks.flush()?;
 
-        // 3. dump the batches
+        // 4. dump the batches
         progress.update_progress(DumpCreationProgress::DumpTheBatches);
         let mut dump_batches = dump.create_batches_queue()?;
 
         let (atomic_batch_progress, update_batch_progress) =
-            AtomicBatchStep::new(self.queue.batches.all_batches.len(&rtxn)? as u32);
+            AtomicBatchStep::new(self.queue.batches.all_batches.len(&rtxn)? as u64);
         progress.update_progress(update_batch_progress);
 
         for ret in self.queue.batches.all_batches.iter(&rtxn)? {
@@ -138,9 +147,9 @@ impl IndexScheduler {
         }
         dump_batches.flush()?;
 
-        // 4. Dump the indexes
+        // 5. Dump the indexes
         progress.update_progress(DumpCreationProgress::DumpTheIndexes);
-        let nb_indexes = self.index_mapper.index_mapping.len(&rtxn)? as u32;
+        let nb_indexes = self.index_mapper.index_mapping.len(&rtxn)? as u64;
         let mut count = 0;
         let () = self.index_mapper.try_for_each_index(&rtxn, |uid, index| -> Result<()> {
             progress.update_progress(VariableNameStep::<DumpCreationProgress>::new(
@@ -165,20 +174,17 @@ impl IndexScheduler {
 
             let fields_ids_map = index.fields_ids_map(&rtxn)?;
             let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
-            let embedding_configs = index
-                .embedding_configs(&rtxn)
-                .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
 
             let nb_documents = index
                 .number_of_documents(&rtxn)
                 .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?
-                as u32;
+                as u64;
             let (atomic, update_document_progress) = AtomicDocumentStep::new(nb_documents);
             progress.update_progress(update_document_progress);
             let documents = index
                 .all_documents(&rtxn)
                 .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
-            // 4.1. Dump the documents
+            // 5.1. Dump the documents
             for ret in documents {
                 if self.scheduler.must_stop_processing.get() {
                     return Err(Error::AbortedTask);
@@ -221,16 +227,12 @@ impl IndexScheduler {
                         return Err(Error::from_milli(user_err, Some(uid.to_string())));
                     };
 
-                    for (embedder_name, embeddings) in embeddings {
-                        let user_provided = embedding_configs
-                            .iter()
-                            .find(|conf| conf.name == embedder_name)
-                            .is_some_and(|conf| conf.user_provided.contains(id));
+                    for (embedder_name, (embeddings, regenerate)) in embeddings {
                         let embeddings = ExplicitVectors {
                             embeddings: Some(VectorOrArrayOfVectors::from_array_of_vectors(
                                 embeddings,
                             )),
-                            regenerate: !user_provided,
+                            regenerate,
                         };
                         vectors.insert(embedder_name, serde_json::to_value(embeddings).unwrap());
                     }
@@ -240,7 +242,7 @@ impl IndexScheduler {
                 atomic.fetch_add(1, Ordering::Relaxed);
             }
 
-            // 4.2. Dump the settings
+            // 5.2. Dump the settings
             let settings = meilisearch_types::settings::settings(
                 index,
                 &rtxn,
@@ -251,7 +253,7 @@ impl IndexScheduler {
             Ok(())
         })?;
 
-        // 5. Dump experimental feature settings
+        // 6. Dump experimental feature settings
         progress.update_progress(DumpCreationProgress::DumpTheExperimentalFeatures);
         let features = self.features().runtime_features();
         dump.create_experimental_features(features)?;

crates/index-scheduler/src/scheduler/process_export.rs

@@ -0,0 +1,365 @@
+use std::collections::BTreeMap;
+use std::io::{self, Write as _};
+use std::sync::atomic;
+use std::time::Duration;
+
+use backoff::ExponentialBackoff;
+use byte_unit::Byte;
+use flate2::write::GzEncoder;
+use flate2::Compression;
+use meilisearch_types::index_uid_pattern::IndexUidPattern;
+use meilisearch_types::milli::constants::RESERVED_VECTORS_FIELD_NAME;
+use meilisearch_types::milli::progress::{Progress, VariableNameStep};
+use meilisearch_types::milli::update::{request_threads, Setting};
+use meilisearch_types::milli::vector::parsed_vectors::{ExplicitVectors, VectorOrArrayOfVectors};
+use meilisearch_types::milli::{self, obkv_to_json, Filter, InternalError};
+use meilisearch_types::settings::{self, SecretPolicy};
+use meilisearch_types::tasks::{DetailsExportIndexSettings, ExportIndexSettings};
+use serde::Deserialize;
+use ureq::{json, Response};
+
+use super::MustStopProcessing;
+use crate::processing::AtomicDocumentStep;
+use crate::{Error, IndexScheduler, Result};
+
+impl IndexScheduler {
+    pub(super) fn process_export(
+        &self,
+        base_url: &str,
+        api_key: Option<&str>,
+        payload_size: Option<&Byte>,
+        indexes: &BTreeMap<IndexUidPattern, ExportIndexSettings>,
+        progress: Progress,
+    ) -> Result<BTreeMap<IndexUidPattern, DetailsExportIndexSettings>> {
+        #[cfg(test)]
+        self.maybe_fail(crate::test_utils::FailureLocation::ProcessExport)?;
+
+        let indexes: Vec<_> = self
+            .index_names()?
+            .into_iter()
+            .flat_map(|uid| {
+                indexes
+                    .iter()
+                    .find(|(pattern, _)| pattern.matches_str(&uid))
+                    .map(|(pattern, settings)| (pattern, uid, settings))
+            })
+            .collect();
+
+        let mut output = BTreeMap::new();
+        let agent = ureq::AgentBuilder::new().timeout(Duration::from_secs(5)).build();
+        let must_stop_processing = self.scheduler.must_stop_processing.clone();
+        for (i, (_pattern, uid, export_settings)) in indexes.iter().enumerate() {
+            if must_stop_processing.get() {
+                return Err(Error::AbortedTask);
+            }
+
+            progress.update_progress(VariableNameStep::<ExportIndex>::new(
+                format!("Exporting index `{uid}`"),
+                i as u64,
+                indexes.len() as u64,
+            ));
+
+            let ExportIndexSettings { filter, override_settings } = export_settings;
+            let index = self.index(uid)?;
+            let index_rtxn = index.read_txn()?;
+
+            // First, check if the index already exists
+            let url = format!("{base_url}/indexes/{uid}");
+            let response = retry(&must_stop_processing, || {
+                let mut request = agent.get(&url);
+                if let Some(api_key) = api_key {
+                    request = request.set("Authorization", &format!("Bearer {api_key}"));
+                }
+
+                request.send_bytes(Default::default()).map_err(into_backoff_error)
+            });
+            let index_exists = match response {
+                Ok(response) => response.status() == 200,
+                Err(Error::FromRemoteWhenExporting { code, .. }) if code == "index_not_found" => {
+                    false
+                }
+                Err(e) => return Err(e),
+            };
+
+            let primary_key = index
+                .primary_key(&index_rtxn)
+                .map_err(|e| Error::from_milli(e.into(), Some(uid.to_string())))?;
+
+            // Create the index
+            if !index_exists {
+                let url = format!("{base_url}/indexes");
+                retry(&must_stop_processing, || {
+                    let mut request = agent.post(&url);
+                    if let Some(api_key) = api_key {
+                        request = request.set("Authorization", &format!("Bearer {api_key}"));
+                    }
+                    let index_param = json!({ "uid": uid, "primaryKey": primary_key });
+                    request.send_json(&index_param).map_err(into_backoff_error)
+                })?;
+            }
+
+            // Patch the index primary key
+            if index_exists && *override_settings {
+                let url = format!("{base_url}/indexes/{uid}");
+                retry(&must_stop_processing, || {
+                    let mut request = agent.patch(&url);
+                    if let Some(api_key) = api_key {
+                        request = request.set("Authorization", &format!("Bearer {api_key}"));
+                    }
+                    let index_param = json!({ "primaryKey": primary_key });
+                    request.send_json(&index_param).map_err(into_backoff_error)
+                })?;
+            }
+
+            // Send the index settings
+            if !index_exists || *override_settings {
+                let mut settings =
+                    settings::settings(&index, &index_rtxn, SecretPolicy::RevealSecrets)
+                        .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
+                // Remove the experimental chat setting if not enabled
+                if self.features().check_chat_completions("exporting chat settings").is_err() {
+                    settings.chat = Setting::NotSet;
+                }
+                // Retry logic for sending settings
+                let url = format!("{base_url}/indexes/{uid}/settings");
+                let bearer = api_key.map(|api_key| format!("Bearer {api_key}"));
+                retry(&must_stop_processing, || {
+                    let mut request = agent.patch(&url);
+                    if let Some(bearer) = bearer.as_ref() {
+                        request = request.set("Authorization", bearer);
+                    }
+                    request.send_json(settings.clone()).map_err(into_backoff_error)
+                })?;
+            }
+
+            let filter = filter
+                .as_ref()
+                .map(Filter::from_json)
+                .transpose()
+                .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?
+                .flatten();
+
+            let filter_universe = filter
+                .map(|f| f.evaluate(&index_rtxn, &index))
+                .transpose()
+                .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
+            let whole_universe = index
+                .documents_ids(&index_rtxn)
+                .map_err(|e| Error::from_milli(e.into(), Some(uid.to_string())))?;
+            let universe = filter_universe.unwrap_or(whole_universe);
+
+            let fields_ids_map = index.fields_ids_map(&index_rtxn)?;
+            let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
+
+            // We don't need to keep this one alive as we will
+            // spawn many threads to process the documents
+            drop(index_rtxn);
+
+            let total_documents = universe.len();
+            let (step, progress_step) = AtomicDocumentStep::new(total_documents);
+            progress.update_progress(progress_step);
+
+            output.insert(
+                IndexUidPattern::new_unchecked(uid.clone()),
+                DetailsExportIndexSettings {
+                    settings: (*export_settings).clone(),
+                    matched_documents: Some(total_documents),
+                },
+            );
+
+            let limit = payload_size.map(|ps| ps.as_u64() as usize).unwrap_or(50 * 1024 * 1024); // defaults to 50 MiB
+            let documents_url = format!("{base_url}/indexes/{uid}/documents");
+
+            request_threads()
+                .broadcast(|ctx| {
+                    let index_rtxn = index
+                        .read_txn()
+                        .map_err(|e| Error::from_milli(e.into(), Some(uid.to_string())))?;
+
+                    let mut buffer = Vec::new();
+                    let mut tmp_buffer = Vec::new();
+                    let mut compressed_buffer = Vec::new();
+                    for (i, docid) in universe.iter().enumerate() {
+                        if i % ctx.num_threads() != ctx.index() {
+                            continue;
+                        }
+
+                        let document = index
+                            .document(&index_rtxn, docid)
+                            .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
+
+                        let mut document = obkv_to_json(&all_fields, &fields_ids_map, document)
+                            .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
+
+                        // TODO definitely factorize this code
+                        'inject_vectors: {
+                            let embeddings = index
+                                .embeddings(&index_rtxn, docid)
+                                .map_err(|e| Error::from_milli(e, Some(uid.to_string())))?;
+
+                            if embeddings.is_empty() {
+                                break 'inject_vectors;
+                            }
+
+                            let vectors = document
+                                .entry(RESERVED_VECTORS_FIELD_NAME)
+                                .or_insert(serde_json::Value::Object(Default::default()));
+
+                            let serde_json::Value::Object(vectors) = vectors else {
+                                return Err(Error::from_milli(
+                                    milli::Error::UserError(
+                                        milli::UserError::InvalidVectorsMapType {
+                                            document_id: {
+                                                if let Ok(Some(Ok(index))) = index
+                                                    .external_id_of(
+                                                        &index_rtxn,
+                                                        std::iter::once(docid),
+                                                    )
+                                                    .map(|it| it.into_iter().next())
+                                                {
+                                                    index
+                                                } else {
+                                                    format!("internal docid={docid}")
+                                                }
+                                            },
+                                            value: vectors.clone(),
+                                        },
+                                    ),
+                                    Some(uid.to_string()),
+                                ));
+                            };
+
+                            for (embedder_name, (embeddings, regenerate)) in embeddings {
+                                let embeddings = ExplicitVectors {
+                                    embeddings: Some(
+                                        VectorOrArrayOfVectors::from_array_of_vectors(embeddings),
+                                    ),
+                                    regenerate,
+                                };
+                                vectors.insert(
+                                    embedder_name,
+                                    serde_json::to_value(embeddings).unwrap(),
+                                );
+                            }
+                        }
+
+                        tmp_buffer.clear();
+                        serde_json::to_writer(&mut tmp_buffer, &document)
+                            .map_err(milli::InternalError::from)
+                            .map_err(|e| Error::from_milli(e.into(), Some(uid.to_string())))?;
+
+                        // Make sure we put at least one document in the buffer even
+                        // though we might go above the buffer limit before sending
+                        if !buffer.is_empty() && buffer.len() + tmp_buffer.len() > limit {
+                            // We compress the documents before sending them
+                            let mut encoder =
+                                GzEncoder::new(&mut compressed_buffer, Compression::default());
+                            encoder
+                                .write_all(&buffer)
+                                .map_err(|e| Error::from_milli(e.into(), Some(uid.clone())))?;
+                            encoder
+                                .finish()
+                                .map_err(|e| Error::from_milli(e.into(), Some(uid.clone())))?;
+
+                            retry(&must_stop_processing, || {
+                                let mut request = agent.post(&documents_url);
+                                request = request.set("Content-Type", "application/x-ndjson");
+                                request = request.set("Content-Encoding", "gzip");
+                                if let Some(api_key) = api_key {
+                                    request = request
+                                        .set("Authorization", &(format!("Bearer {api_key}")));
+                                }
+                                request.send_bytes(&compressed_buffer).map_err(into_backoff_error)
+                            })?;
+                            buffer.clear();
+                            compressed_buffer.clear();
+                        }
+                        buffer.extend_from_slice(&tmp_buffer);
+
+                        if i % 100 == 0 {
+                            step.fetch_add(100, atomic::Ordering::Relaxed);
+                        }
+                    }
+
+                    retry(&must_stop_processing, || {
+                        let mut request = agent.post(&documents_url);
+                        request = request.set("Content-Type", "application/x-ndjson");
+                        if let Some(api_key) = api_key {
+                            request = request.set("Authorization", &(format!("Bearer {api_key}")));
+                        }
+                        request.send_bytes(&buffer).map_err(into_backoff_error)
+                    })?;
+
+                    Ok(())
+                })
+                .map_err(|e| {
+                    Error::from_milli(
+                        milli::Error::InternalError(InternalError::PanicInThreadPool(e)),
+                        Some(uid.to_string()),
+                    )
+                })?;
+
+            step.store(total_documents, atomic::Ordering::Relaxed);
+        }
+
+        Ok(output)
+    }
+}
+
+fn retry<F>(must_stop_processing: &MustStopProcessing, send_request: F) -> Result<ureq::Response>
+where
+    F: Fn() -> Result<ureq::Response, backoff::Error<ureq::Error>>,
+{
+    match backoff::retry(ExponentialBackoff::default(), || {
+        if must_stop_processing.get() {
+            return Err(backoff::Error::Permanent(ureq::Error::Status(
+                u16::MAX,
+                // 444: Connection Closed Without Response
+                Response::new(444, "Abort", "Aborted task").unwrap(),
+            )));
+        }
+        send_request()
+    }) {
+        Ok(response) => Ok(response),
+        Err(backoff::Error::Permanent(e)) => Err(ureq_error_into_error(e)),
+        Err(backoff::Error::Transient { err, retry_after: _ }) => Err(ureq_error_into_error(err)),
+    }
+}
+
+fn into_backoff_error(err: ureq::Error) -> backoff::Error<ureq::Error> {
+    match err {
+        // Those code status must trigger an automatic retry
+        // <https://www.restapitutorial.com/advanced/responses/retries>
+        ureq::Error::Status(408 | 429 | 500 | 502 | 503 | 504, _) => {
+            backoff::Error::Transient { err, retry_after: None }
+        }
+        ureq::Error::Status(_, _) => backoff::Error::Permanent(err),
+        ureq::Error::Transport(_) => backoff::Error::Transient { err, retry_after: None },
+    }
+}
+
+/// Converts a `ureq::Error` into an `Error`.
+fn ureq_error_into_error(error: ureq::Error) -> Error {
+    #[derive(Deserialize)]
+    struct MeiliError {
+        message: String,
+        code: String,
+        r#type: String,
+        link: String,
+    }
+
+    match error {
+        // This is a workaround to handle task abortion - the error propagation path
+        // makes it difficult to cleanly surface the abortion at this level.
+        ureq::Error::Status(u16::MAX, _) => Error::AbortedTask,
+        ureq::Error::Status(_, response) => match response.into_json() {
+            Ok(MeiliError { message, code, r#type, link }) => {
+                Error::FromRemoteWhenExporting { message, code, r#type, link }
+            }
+            Err(e) => e.into(),
+        },
+        ureq::Error::Transport(transport) => io::Error::new(io::ErrorKind::Other, transport).into(),
+    }
+}
+
+enum ExportIndex {}

crates/index-scheduler/src/scheduler/process_index_operation.rs

@@ -1,11 +1,13 @@
+use std::sync::Arc;
+
 use bumpalo::collections::CollectIn;
 use bumpalo::Bump;
 use meilisearch_types::heed::RwTxn;
 use meilisearch_types::milli::documents::PrimaryKey;
-use meilisearch_types::milli::progress::Progress;
+use meilisearch_types::milli::progress::{EmbedderStats, Progress};
 use meilisearch_types::milli::update::new::indexer::{self, UpdateByFunction};
 use meilisearch_types::milli::update::DocumentAdditionResult;
-use meilisearch_types::milli::{self, ChannelCongestion, Filter, ThreadPoolNoAbortBuilder};
+use meilisearch_types::milli::{self, ChannelCongestion, Filter};
 use meilisearch_types::settings::apply_settings_to_builder;
 use meilisearch_types::tasks::{Details, KindWithContent, Status, Task};
 use meilisearch_types::Index;
@@ -24,7 +26,7 @@ impl IndexScheduler {
     /// The list of processed tasks.
     #[tracing::instrument(
         level = "trace",
-        skip(self, index_wtxn, index, progress),
+        skip(self, index_wtxn, index, progress, embedder_stats),
         target = "indexing::scheduler"
     )]
     pub(crate) fn apply_index_operation<'i>(
@@ -33,6 +35,7 @@ impl IndexScheduler {
         index: &'i Index,
         operation: IndexOperation,
         progress: &Progress,
+        embedder_stats: Arc<EmbedderStats>,
     ) -> Result<(Vec<Task>, Option<ChannelCongestion>)> {
         let indexer_alloc = Bump::new();
         let started_processing_at = std::time::Instant::now();
@@ -86,8 +89,9 @@ impl IndexScheduler {
                 let mut content_files_iter = content_files.iter();
                 let mut indexer = indexer::DocumentOperation::new();
                 let embedders = index
+                    .embedding_configs()
                     .embedding_configs(index_wtxn)
-                    .map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?;
+                    .map_err(|e| Error::from_milli(e.into(), Some(index_uid.clone())))?;
                 let embedders = self.embedders(index_uid.clone(), embedders)?;
                 for operation in operations {
                     match operation {
@@ -113,18 +117,8 @@ impl IndexScheduler {
                     }
                 }
 
-                let local_pool;
                 let indexer_config = self.index_mapper.indexer_config();
-                let pool = match &indexer_config.thread_pool {
-                    Some(pool) => pool,
-                    None => {
-                        local_pool = ThreadPoolNoAbortBuilder::new()
-                            .thread_name(|i| format!("indexing-thread-{i}"))
-                            .build()
-                            .unwrap();
-                        &local_pool
-                    }
-                };
+                let pool = &indexer_config.thread_pool;
 
                 progress.update_progress(DocumentOperationProgress::ComputingDocumentChanges);
                 let (document_changes, operation_stats, primary_key) = indexer
@@ -187,6 +181,7 @@ impl IndexScheduler {
                             embedders,
                             &|| must_stop_processing.get(),
                             progress,
+                            &embedder_stats,
                         )
                         .map_err(|e| Error::from_milli(e, Some(index_uid.clone())))?,
                     );
@@ -266,18 +261,8 @@ impl IndexScheduler {
 
                 let mut congestion = None;
                 if task.error.is_none() {
-                    let local_pool;
                     let indexer_config = self.index_mapper.indexer_config();
-                    let pool = match &indexer_config.thread_pool {
-                        Some(pool) => pool,
-                        None => {
-                            local_pool = ThreadPoolNoAbortBuilder::new()
-                                .thread_name(|i| format!("indexing-thread-{i}"))
-                                .build()
-                                .unwrap();
-                            &local_pool
-                        }
-                    };
+                    let pool = &indexer_config.thread_pool;
 
                     let candidates_count = candidates.len();
                     progress.update_progress(DocumentEditionProgress::ComputingDocumentChanges);
@@ -290,8 +275,9 @@ impl IndexScheduler {
                         })
                         .unwrap()?;
                     let embedders = index
+                        .embedding_configs()
                         .embedding_configs(index_wtxn)
-                        .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?;
+                        .map_err(|err| Error::from_milli(err.into(), Some(index_uid.clone())))?;
                     let embedders = self.embedders(index_uid.clone(), embedders)?;
 
                     progress.update_progress(DocumentEditionProgress::Indexing);
@@ -308,6 +294,7 @@ impl IndexScheduler {
                             embedders,
                             &|| must_stop_processing.get(),
                             progress,
+                            &embedder_stats,
                         )
                         .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?,
                     );
@@ -429,18 +416,8 @@ impl IndexScheduler {
 
                 let mut congestion = None;
                 if !tasks.iter().all(|res| res.error.is_some()) {
-                    let local_pool;
                     let indexer_config = self.index_mapper.indexer_config();
-                    let pool = match &indexer_config.thread_pool {
-                        Some(pool) => pool,
-                        None => {
-                            local_pool = ThreadPoolNoAbortBuilder::new()
-                                .thread_name(|i| format!("indexing-thread-{i}"))
-                                .build()
-                                .unwrap();
-                            &local_pool
-                        }
-                    };
+                    let pool = &indexer_config.thread_pool;
 
                     progress.update_progress(DocumentDeletionProgress::DeleteDocuments);
                     let mut indexer = indexer::DocumentDeletion::new();
@@ -448,8 +425,9 @@ impl IndexScheduler {
                     indexer.delete_documents_by_docids(to_delete);
                     let document_changes = indexer.into_changes(&indexer_alloc, primary_key);
                     let embedders = index
+                        .embedding_configs()
                         .embedding_configs(index_wtxn)
-                        .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?;
+                        .map_err(|err| Error::from_milli(err.into(), Some(index_uid.clone())))?;
                     let embedders = self.embedders(index_uid.clone(), embedders)?;
 
                     progress.update_progress(DocumentDeletionProgress::Indexing);
@@ -466,6 +444,7 @@ impl IndexScheduler {
                             embedders,
                             &|| must_stop_processing.get(),
                             progress,
+                            &embedder_stats,
                         )
                         .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?,
                     );
@@ -498,14 +477,11 @@ impl IndexScheduler {
                 }
 
                 progress.update_progress(SettingsProgress::ApplyTheSettings);
-                builder
-                    .execute(
-                        |indexing_step| tracing::debug!(update = ?indexing_step),
-                        || must_stop_processing.get(),
-                    )
+                let congestion = builder
+                    .execute(&|| must_stop_processing.get(), progress, embedder_stats)
                     .map_err(|err| Error::from_milli(err, Some(index_uid.clone())))?;
 
-                Ok((tasks, None))
+                Ok((tasks, congestion))
             }
             IndexOperation::DocumentClearAndSetting {
                 index_uid,
@@ -521,6 +497,7 @@ impl IndexScheduler {
                         tasks: cleared_tasks,
                     },
                     progress,
+                    embedder_stats.clone(),
                 )?;
 
                 let (settings_tasks, _congestion) = self.apply_index_operation(
@@ -528,6 +505,7 @@ impl IndexScheduler {
                     index,
                     IndexOperation::Settings { index_uid, settings, tasks: settings_tasks },
                     progress,
+                    embedder_stats,
                 )?;
 
                 let mut tasks = settings_tasks;

crates/index-scheduler/src/scheduler/process_snapshot_creation.rs

@@ -58,7 +58,7 @@ impl IndexScheduler {
         // 2.4 Only copy the update files of the enqueued tasks
         progress.update_progress(SnapshotCreationProgress::SnapshotTheUpdateFiles);
         let enqueued = self.queue.tasks.get_status(&rtxn, Status::Enqueued)?;
-        let (atomic, update_file_progress) = AtomicUpdateFileStep::new(enqueued.len() as u32);
+        let (atomic, update_file_progress) = AtomicUpdateFileStep::new(enqueued.len() as u64);
         progress.update_progress(update_file_progress);
         for task_id in enqueued {
             let task =
@@ -74,12 +74,12 @@ impl IndexScheduler {
         // 3. Snapshot every indexes
         progress.update_progress(SnapshotCreationProgress::SnapshotTheIndexes);
         let index_mapping = self.index_mapper.index_mapping;
-        let nb_indexes = index_mapping.len(&rtxn)? as u32;
+        let nb_indexes = index_mapping.len(&rtxn)? as u64;
 
         for (i, result) in index_mapping.iter(&rtxn)?.enumerate() {
             let (name, uuid) = result?;
             progress.update_progress(VariableNameStep::<SnapshotCreationProgress>::new(
-                name, i as u32, nb_indexes,
+                name, i as u64, nb_indexes,
             ));
             let index = self.index_mapper.index(&rtxn, name)?;
             let dst = temp_snapshot_dir.path().join("indexes").join(uuid.to_string());

crates/index-scheduler/src/scheduler/process_upgrade.rs

@@ -22,8 +22,8 @@ impl IndexScheduler {
             }
             progress.update_progress(VariableNameStep::<UpgradeIndex>::new(
                 format!("Upgrading index `{uid}`"),
-                i as u32,
-                indexes.len() as u32,
+                i as u64,
+                indexes.len() as u64,
             ));
             let index = self.index(uid)?;
             let mut index_wtxn = index.write_txn()?;
@@ -65,8 +65,8 @@ impl IndexScheduler {
         for (i, uid) in indexes.iter().enumerate() {
             progress.update_progress(VariableNameStep::<UpgradeIndex>::new(
                 format!("Rollbacking index `{uid}`"),
-                i as u32,
-                indexes.len() as u32,
+                i as u64,
+                indexes.len() as u64,
             ));
             let index_schd_rtxn = self.env.read_txn()?;
 

crates/index-scheduler/src/scheduler/snapshots/index_scheduler__scheduler__test__settings_update-7.snap

@@ -0,0 +1,17 @@
+---
+source: crates/index-scheduler/src/scheduler/test.rs
+expression: config.embedder_options
+---
+{
+  "Rest": {
+    "api_key": "My super secret",
+    "distribution": null,
+    "dimensions": 4,
+    "url": "http://localhost:7777",
+    "request": "{{text}}",
+    "search_fragments": {},
+    "indexing_fragments": {},
+    "response": "{{embedding}}",
+    "headers": {}
+  }
+}

crates/index-scheduler/src/scheduler/snapshots/index_scheduler__scheduler__test_embedders__import_vectors-14.snap

@@ -0,0 +1,12 @@
+---
+source: crates/index-scheduler/src/scheduler/test_embedders.rs
+expression: simple_hf_config.embedder_options
+---
+{
+  "HuggingFace": {
+    "model": "sentence-transformers/all-MiniLM-L6-v2",
+    "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+    "distribution": null,
+    "pooling": "useModel"
+  }
+}

crates/index-scheduler/src/scheduler/snapshots/index_scheduler__scheduler__test_embedders__import_vectors-27.snap

@@ -0,0 +1,15 @@
+---
+source: crates/index-scheduler/src/scheduler/test_embedders.rs
+expression: doc
+---
+{
+  "doggo": "Intel",
+  "breed": "beagle",
+  "_vectors": {
+    "noise": [
+      0.1,
+      0.2,
+      0.3
+    ]
+  }
+}

crates/index-scheduler/src/scheduler/snapshots/index_scheduler__scheduler__test_embedders__import_vectors-40.snap

@@ -0,0 +1,15 @@
+---
+source: crates/index-scheduler/src/scheduler/test_embedders.rs
+expression: doc
+---
+{
+  "doggo": "kefir",
+  "breed": "patou",
+  "_vectors": {
+    "noise": [
+      0.1,
+      0.2,
+      0.3
+    ]
+  }
+}

crates/index-scheduler/src/scheduler/snapshots/index_scheduler__scheduler__test_embedders__import_vectors-8.snap

@@ -1,12 +1,17 @@
 ---
 source: crates/index-scheduler/src/scheduler/test_embedders.rs
-expression: simple_hf_config.embedder_options
+expression: fakerest_config.embedder_options
 ---
 {
-  "HuggingFace": {
-    "model": "sentence-transformers/all-MiniLM-L6-v2",
-    "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+  "Rest": {
+    "api_key": "My super secret",
     "distribution": null,
-    "pooling": "useModel"
+    "dimensions": 384,
+    "url": "http://localhost:7777",
+    "request": "{{text}}",
+    "search_fragments": {},
+    "indexing_fragments": {},
+    "response": "{{embedding}}",
+    "headers": {}
   }
 }

crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/after_registering_settings_task.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

crates/index-scheduler/src/scheduler/snapshots/test.rs/test_settings_update/settings_update_processed.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/Intel to kefir succeeds.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_embedders.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: None, method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------

crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/Intel to kefir.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_embedders.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: None, method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------

crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/adding Intel succeeds.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_embedders.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:

crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after adding Intel.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_embedders.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:

crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/after_registering_settings_task_vectors.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_embedders.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

crates/index-scheduler/src/scheduler/snapshots/test_embedders.rs/import_vectors/settings_update_processed_vectors.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_embedders.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, pooling: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), indexing_fragments: NotSet, search_fragments: NotSet, request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), pooling: NotSet, api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, indexing_fragments: NotSet, search_fragments: NotSet, request: NotSet, response: NotSet, headers: NotSet, search_embedder: NotSet, indexing_embedder: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_adding_the_documents.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
 2 {uid: 2, batch_uid: 2, status: succeeded, details: { received_document_ids: 1, deleted_documents: Some(1) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1"] }}
 3 {uid: 3, batch_uid: 2, status: failed, error: ResponseError { code: 200, message: "Index `doggos`: Invalid type for filter subexpression: expected: String, Array, found: true.", error_code: "invalid_document_filter", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#invalid_document_filter" }, details: { original_filter: true, deleted_documents: Some(0) }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: Bool(true) }}

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/registered_the_document_deletions.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, batch_uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
 2 {uid: 2, status: enqueued, details: { received_document_ids: 1, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1"] }}
 3 {uid: 3, status: enqueued, details: { original_filter: true, deleted_documents: None }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: Bool(true) }}

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/fail_in_process_batch_for_document_deletion/registered_the_setting_and_document_addition.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set([Field("catto")]), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, facet_search: NotSet, prefix_search: NotSet, chat: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 16, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} }
 [timestamp] [4,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.16.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", }
 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", }
 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", }

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 16, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 16, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 ----------------------------------------------------------------------
 ### Status:

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 16, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 ----------------------------------------------------------------------
 ### Status:
@@ -37,7 +37,7 @@ catto [1,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.16.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 16, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 ----------------------------------------------------------------------
@@ -40,7 +40,7 @@ doggo [2,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.16.0"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 15, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 16, 0) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse") }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone") }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -43,7 +43,7 @@ doggo [2,3,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.15.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.16.0"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/scheduler/test.rs

@@ -3,11 +3,11 @@ use std::collections::BTreeMap;
 use big_s::S;
 use meili_snap::{json_string, snapshot};
 use meilisearch_auth::AuthFilter;
-use meilisearch_types::milli::index::IndexEmbeddingConfig;
 use meilisearch_types::milli::update::IndexDocumentsMethod::*;
 use meilisearch_types::milli::{self};
 use meilisearch_types::settings::SettingEmbeddingSettings;
 use meilisearch_types::tasks::{IndexSwap, KindWithContent};
+use milli::vector::db::IndexEmbeddingConfig;
 use roaring::RoaringBitmap;
 
 use crate::insta_snapshot::snapshot_index_scheduler;
@@ -690,11 +690,20 @@ fn test_settings_update() {
     let index = index_scheduler.index("doggos").unwrap();
     let rtxn = index.read_txn().unwrap();
 
-    let configs = index.embedding_configs(&rtxn).unwrap();
-    let IndexEmbeddingConfig { name, config, user_provided } = configs.first().unwrap();
+    let embedders = index.embedding_configs();
+    let configs = embedders.embedding_configs(&rtxn).unwrap();
+    let IndexEmbeddingConfig { name, config, fragments } = configs.first().unwrap();
+    let info = embedders.embedder_info(&rtxn, name).unwrap().unwrap();
+    insta::assert_snapshot!(info.embedder_id, @"0");
+    insta::assert_debug_snapshot!(info.embedding_status.user_provided_docids(), @"RoaringBitmap<[]>");
+    insta::assert_debug_snapshot!(info.embedding_status.skip_regenerate_docids(), @"RoaringBitmap<[]>");
     insta::assert_snapshot!(name, @"default");
-    insta::assert_debug_snapshot!(user_provided, @"RoaringBitmap<[]>");
     insta::assert_json_snapshot!(config.embedder_options);
+    insta::assert_debug_snapshot!(fragments, @r###"
+    FragmentConfigs(
+        [],
+    )
+    "###);
 }
 
 #[test]
@@ -732,6 +741,7 @@ fn basic_get_stats() {
         "documentDeletion": 0,
         "documentEdition": 0,
         "dumpCreation": 0,
+        "export": 0,
         "indexCreation": 3,
         "indexDeletion": 0,
         "indexSwap": 0,
@@ -765,6 +775,7 @@ fn basic_get_stats() {
         "documentDeletion": 0,
         "documentEdition": 0,
         "dumpCreation": 0,
+        "export": 0,
         "indexCreation": 3,
         "indexDeletion": 0,
         "indexSwap": 0,
@@ -805,6 +816,7 @@ fn basic_get_stats() {
         "documentDeletion": 0,
         "documentEdition": 0,
         "dumpCreation": 0,
+        "export": 0,
         "indexCreation": 3,
         "indexDeletion": 0,
         "indexSwap": 0,
@@ -846,6 +858,7 @@ fn basic_get_stats() {
         "documentDeletion": 0,
         "documentEdition": 0,
         "dumpCreation": 0,
+        "export": 0,
         "indexCreation": 3,
         "indexDeletion": 0,
         "indexSwap": 0,
@@ -894,7 +907,7 @@ fn create_and_list_index() {
 
     let err = index_scheduler.index("kefir").map(|_| ()).unwrap_err();
     snapshot!(err, @"Index `kefir` not found.");
-    let empty = index_scheduler.get_paginated_indexes_stats(&AuthFilter::default(), 0, 20).unwrap();
+    let empty = index_scheduler.paginated_indexes_stats(&AuthFilter::default(), 0, 20).unwrap();
     snapshot!(format!("{empty:?}"), @"(0, [])");
 
     // After advancing just once the index should've been created, the wtxn has been released and commited
@@ -902,7 +915,7 @@ fn create_and_list_index() {
     handle.advance_till([InsideProcessBatch]);
 
     index_scheduler.index("kefir").unwrap();
-    let list = index_scheduler.get_paginated_indexes_stats(&AuthFilter::default(), 0, 20).unwrap();
+    let list = index_scheduler.paginated_indexes_stats(&AuthFilter::default(), 0, 20).unwrap();
     snapshot!(json_string!(list, { "[1][0][1].created_at" => "[date]", "[1][0][1].updated_at" => "[date]", "[1][0][1].used_database_size" => "[bytes]", "[1][0][1].database_size" => "[bytes]" }), @r###"
     [
       1,

crates/index-scheduler/src/scheduler/test_embedders.rs

@@ -3,13 +3,14 @@ use std::collections::BTreeMap;
 use big_s::S;
 use insta::assert_json_snapshot;
 use meili_snap::{json_string, snapshot};
-use meilisearch_types::milli::index::IndexEmbeddingConfig;
 use meilisearch_types::milli::update::Setting;
 use meilisearch_types::milli::vector::settings::EmbeddingSettings;
+use meilisearch_types::milli::vector::SearchQuery;
 use meilisearch_types::milli::{self, obkv_to_json};
 use meilisearch_types::settings::{SettingEmbeddingSettings, Settings, Unchecked};
 use meilisearch_types::tasks::KindWithContent;
 use milli::update::IndexDocumentsMethod::*;
+use milli::vector::db::IndexEmbeddingConfig;
 
 use crate::insta_snapshot::snapshot_index_scheduler;
 use crate::test_utils::read_json;
@@ -85,28 +86,51 @@ fn import_vectors() {
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
 
-        let configs = index.embedding_configs(&rtxn).unwrap();
+        let embedders = index.embedding_configs();
+        let configs = embedders.embedding_configs(&rtxn).unwrap();
         // for consistency with the below
         #[allow(clippy::get_first)]
-        let IndexEmbeddingConfig { name, config: fakerest_config, user_provided } =
+        let IndexEmbeddingConfig { name, config: fakerest_config, fragments } =
             configs.get(0).unwrap();
+        let info = embedders.embedder_info(&rtxn, name).unwrap().unwrap();
+        insta::assert_snapshot!(info.embedder_id, @"0");
+        insta::assert_debug_snapshot!(info.embedding_status.user_provided_docids(), @"RoaringBitmap<[]>");
+        insta::assert_debug_snapshot!(info.embedding_status.skip_regenerate_docids(), @"RoaringBitmap<[]>");
         insta::assert_snapshot!(name, @"A_fakerest");
-        insta::assert_debug_snapshot!(user_provided, @"RoaringBitmap<[]>");
+        insta::assert_debug_snapshot!(fragments, @r###"
+        FragmentConfigs(
+            [],
+        )
+        "###);
         insta::assert_json_snapshot!(fakerest_config.embedder_options);
         let fakerest_name = name.clone();
 
-        let IndexEmbeddingConfig { name, config: simple_hf_config, user_provided } =
+        let IndexEmbeddingConfig { name, config: simple_hf_config, fragments } =
             configs.get(1).unwrap();
+        let info = embedders.embedder_info(&rtxn, name).unwrap().unwrap();
+        insta::assert_snapshot!(info.embedder_id, @"1");
+        insta::assert_debug_snapshot!(info.embedding_status.user_provided_docids(), @"RoaringBitmap<[]>");
+        insta::assert_debug_snapshot!(info.embedding_status.skip_regenerate_docids(), @"RoaringBitmap<[]>");
         insta::assert_snapshot!(name, @"B_small_hf");
-        insta::assert_debug_snapshot!(user_provided, @"RoaringBitmap<[]>");
+        insta::assert_debug_snapshot!(fragments, @r###"
+        FragmentConfigs(
+            [],
+        )
+        "###);
         insta::assert_json_snapshot!(simple_hf_config.embedder_options);
         let simple_hf_name = name.clone();
 
         let configs = index_scheduler.embedders("doggos".to_string(), configs).unwrap();
-        let (hf_embedder, _, _) = configs.get(&simple_hf_name).unwrap();
-        let beagle_embed = hf_embedder.embed_search("Intel the beagle best doggo", None).unwrap();
-        let lab_embed = hf_embedder.embed_search("Max the lab best doggo", None).unwrap();
-        let patou_embed = hf_embedder.embed_search("kefir the patou best doggo", None).unwrap();
+        let hf_runtime = configs.get(&simple_hf_name).unwrap();
+        let hf_embedder = &hf_runtime.embedder;
+        let beagle_embed = hf_embedder
+            .embed_search(SearchQuery::Text("Intel the beagle best doggo"), None)
+            .unwrap();
+        let lab_embed =
+            hf_embedder.embed_search(SearchQuery::Text("Max the lab best doggo"), None).unwrap();
+        let patou_embed = hf_embedder
+            .embed_search(SearchQuery::Text("kefir the patou best doggo"), None)
+            .unwrap();
         (fakerest_name, simple_hf_name, beagle_embed, lab_embed, patou_embed)
     };
 
@@ -166,22 +190,38 @@ fn import_vectors() {
         let rtxn = index.read_txn().unwrap();
 
         // Ensure the document have been inserted into the relevant bitamp
-        let configs = index.embedding_configs(&rtxn).unwrap();
+        let embedders = index.embedding_configs();
+        let configs = embedders.embedding_configs(&rtxn).unwrap();
         // for consistency with the below
         #[allow(clippy::get_first)]
-        let IndexEmbeddingConfig { name, config: _, user_provided: user_defined } =
-            configs.get(0).unwrap();
+        let IndexEmbeddingConfig { name, config: _, fragments } = configs.get(0).unwrap();
+        let info = embedders.embedder_info(&rtxn, name).unwrap().unwrap();
+        insta::assert_snapshot!(info.embedder_id, @"0");
+        insta::assert_debug_snapshot!(info.embedding_status.user_provided_docids(), @"RoaringBitmap<[0]>");
+        insta::assert_debug_snapshot!(info.embedding_status.skip_regenerate_docids(), @"RoaringBitmap<[0]>");
         insta::assert_snapshot!(name, @"A_fakerest");
-        insta::assert_debug_snapshot!(user_defined, @"RoaringBitmap<[0]>");
+        insta::assert_debug_snapshot!(fragments, @r###"
+        FragmentConfigs(
+            [],
+        )
+        "###);
 
-        let IndexEmbeddingConfig { name, config: _, user_provided } = configs.get(1).unwrap();
+        let IndexEmbeddingConfig { name, config: _, fragments } = configs.get(1).unwrap();
+        let info = embedders.embedder_info(&rtxn, name).unwrap().unwrap();
+        insta::assert_snapshot!(info.embedder_id, @"1");
+        insta::assert_debug_snapshot!(info.embedding_status.user_provided_docids(), @"RoaringBitmap<[0]>");
+        insta::assert_debug_snapshot!(info.embedding_status.skip_regenerate_docids(), @"RoaringBitmap<[]>");
         insta::assert_snapshot!(name, @"B_small_hf");
-        insta::assert_debug_snapshot!(user_provided, @"RoaringBitmap<[]>");
+        insta::assert_debug_snapshot!(fragments, @r###"
+        FragmentConfigs(
+            [],
+        )
+        "###);
 
         let embeddings = index.embeddings(&rtxn, 0).unwrap();
 
-        assert_json_snapshot!(embeddings[&simple_hf_name][0] == lab_embed, @"true");
-        assert_json_snapshot!(embeddings[&fakerest_name][0] == beagle_embed, @"true");
+        assert_json_snapshot!(embeddings[&simple_hf_name].0[0] == lab_embed, @"true");
+        assert_json_snapshot!(embeddings[&fakerest_name].0[0] == beagle_embed, @"true");
 
         let doc = index.documents(&rtxn, std::iter::once(0)).unwrap()[0].1;
         let fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
@@ -239,25 +279,41 @@ fn import_vectors() {
             let index = index_scheduler.index("doggos").unwrap();
             let rtxn = index.read_txn().unwrap();
 
+            let embedders = index.embedding_configs();
             // Ensure the document have been inserted into the relevant bitamp
-            let configs = index.embedding_configs(&rtxn).unwrap();
+            let configs = embedders.embedding_configs(&rtxn).unwrap();
             // for consistency with the below
             #[allow(clippy::get_first)]
-            let IndexEmbeddingConfig { name, config: _, user_provided: user_defined } =
-                configs.get(0).unwrap();
+            let IndexEmbeddingConfig { name, config: _, fragments } = configs.get(0).unwrap();
+            let info = embedders.embedder_info(&rtxn, name).unwrap().unwrap();
+            insta::assert_snapshot!(info.embedder_id, @"0");
+            insta::assert_debug_snapshot!(info.embedding_status.user_provided_docids(), @"RoaringBitmap<[0]>");
+            insta::assert_debug_snapshot!(info.embedding_status.skip_regenerate_docids(), @"RoaringBitmap<[0]>");
             insta::assert_snapshot!(name, @"A_fakerest");
-            insta::assert_debug_snapshot!(user_defined, @"RoaringBitmap<[0]>");
+            insta::assert_debug_snapshot!(fragments, @r###"
+            FragmentConfigs(
+                [],
+            )
+            "###);
 
-            let IndexEmbeddingConfig { name, config: _, user_provided } = configs.get(1).unwrap();
+            let IndexEmbeddingConfig { name, config: _, fragments } = configs.get(1).unwrap();
+            let info = embedders.embedder_info(&rtxn, name).unwrap().unwrap();
+            insta::assert_snapshot!(info.embedder_id, @"1");
+            insta::assert_debug_snapshot!(info.embedding_status.user_provided_docids(), @"RoaringBitmap<[]>");
+            insta::assert_debug_snapshot!(info.embedding_status.skip_regenerate_docids(), @"RoaringBitmap<[]>");
             insta::assert_snapshot!(name, @"B_small_hf");
-            insta::assert_debug_snapshot!(user_provided, @"RoaringBitmap<[]>");
+            insta::assert_debug_snapshot!(fragments, @r###"
+            FragmentConfigs(
+                [],
+            )
+            "###);
 
             let embeddings = index.embeddings(&rtxn, 0).unwrap();
 
             // automatically changed to patou because set to regenerate
-            assert_json_snapshot!(embeddings[&simple_hf_name][0] == patou_embed, @"true");
+            assert_json_snapshot!(embeddings[&simple_hf_name].0[0] == patou_embed, @"true");
             // remained beagle
-            assert_json_snapshot!(embeddings[&fakerest_name][0] == beagle_embed, @"true");
+            assert_json_snapshot!(embeddings[&fakerest_name].0[0] == beagle_embed, @"true");
 
             let doc = index.documents(&rtxn, std::iter::once(0)).unwrap()[0].1;
             let fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
@@ -399,8 +455,8 @@ fn import_vectors_first_and_embedder_later() {
         .collect::<Vec<_>>();
     // the all the vectors linked to the new specified embedder have been removed
     // Only the unknown embedders stays in the document DB
-    snapshot!(serde_json::to_string(&documents).unwrap(), @r###"[{"id":0,"doggo":"kefir"},{"id":1,"doggo":"intel","_vectors":{"unknown embedder":[1.0,2.0,3.0]}},{"id":2,"doggo":"max","_vectors":{"unknown embedder":[4.0,5.0]}},{"id":3,"doggo":"marcel"},{"id":4,"doggo":"sora"}]"###);
-    let conf = index.embedding_configs(&rtxn).unwrap();
+    snapshot!(serde_json::to_string(&documents).unwrap(), @r###"[{"id":0,"doggo":"kefir"},{"id":1,"doggo":"intel","_vectors":{"unknown embedder":[1,2,3]}},{"id":2,"doggo":"max","_vectors":{"unknown embedder":[4,5]}},{"id":3,"doggo":"marcel"},{"id":4,"doggo":"sora"}]"###);
+    let conf = index.embedding_configs().embedding_configs(&rtxn).unwrap();
     // even though we specified the vector for the ID 3, it shouldn't be marked
     // as user provided since we explicitely marked it as NOT user provided.
     snapshot!(format!("{conf:#?}"), @r###"
@@ -426,19 +482,28 @@ fn import_vectors_first_and_embedder_later() {
                 },
                 quantized: None,
             },
-            user_provided: RoaringBitmap<[1, 2]>,
+            fragments: FragmentConfigs(
+                [],
+            ),
         },
     ]
     "###);
+    let info =
+        index.embedding_configs().embedder_info(&rtxn, "my_doggo_embedder").unwrap().unwrap();
+    insta::assert_snapshot!(info.embedder_id, @"0");
+
+    insta::assert_debug_snapshot!(info.embedding_status.user_provided_docids(), @"RoaringBitmap<[1, 2, 3]>");
+    insta::assert_debug_snapshot!(info.embedding_status.skip_regenerate_docids(), @"RoaringBitmap<[1, 2]>");
+
     let docid = index.external_documents_ids.get(&rtxn, "0").unwrap().unwrap();
     let embeddings = index.embeddings(&rtxn, docid).unwrap();
-    let embedding = &embeddings["my_doggo_embedder"];
+    let (embedding, _) = &embeddings["my_doggo_embedder"];
     assert!(!embedding.is_empty(), "{embedding:?}");
 
     // the document with the id 3 should keep its original embedding
     let docid = index.external_documents_ids.get(&rtxn, "3").unwrap().unwrap();
     let embeddings = index.embeddings(&rtxn, docid).unwrap();
-    let embeddings = &embeddings["my_doggo_embedder"];
+    let (embeddings, _) = &embeddings["my_doggo_embedder"];
 
     snapshot!(embeddings.len(), @"1");
     assert!(embeddings[0].iter().all(|i| *i == 3.0), "{:?}", embeddings[0]);
@@ -493,7 +558,7 @@ fn import_vectors_first_and_embedder_later() {
         "###);
 
     let embeddings = index.embeddings(&rtxn, docid).unwrap();
-    let embedding = &embeddings["my_doggo_embedder"];
+    let (embedding, _) = &embeddings["my_doggo_embedder"];
 
     assert!(!embedding.is_empty());
     assert!(!embedding[0].iter().all(|i| *i == 3.0), "{:?}", embedding[0]);
@@ -501,7 +566,7 @@ fn import_vectors_first_and_embedder_later() {
     // the document with the id 4 should generate an embedding
     let docid = index.external_documents_ids.get(&rtxn, "4").unwrap().unwrap();
     let embeddings = index.embeddings(&rtxn, docid).unwrap();
-    let embedding = &embeddings["my_doggo_embedder"];
+    let (embedding, _) = &embeddings["my_doggo_embedder"];
 
     assert!(!embedding.is_empty());
 }
@@ -603,33 +668,35 @@ fn delete_document_containing_vector() {
         .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
         .collect::<Vec<_>>();
     snapshot!(serde_json::to_string(&documents).unwrap(), @r###"[{"id":0,"doggo":"kefir"}]"###);
-    let conf = index.embedding_configs(&rtxn).unwrap();
+    let conf = index.embedding_configs().embedding_configs(&rtxn).unwrap();
     snapshot!(format!("{conf:#?}"), @r###"
-        [
-            IndexEmbeddingConfig {
-                name: "manual",
-                config: EmbeddingConfig {
-                    embedder_options: UserProvided(
-                        EmbedderOptions {
-                            dimensions: 3,
-                            distribution: None,
-                        },
-                    ),
-                    prompt: PromptData {
-                        template: "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
-                        max_bytes: Some(
-                            400,
-                        ),
+    [
+        IndexEmbeddingConfig {
+            name: "manual",
+            config: EmbeddingConfig {
+                embedder_options: UserProvided(
+                    EmbedderOptions {
+                        dimensions: 3,
+                        distribution: None,
                     },
-                    quantized: None,
+                ),
+                prompt: PromptData {
+                    template: "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
+                    max_bytes: Some(
+                        400,
+                    ),
                 },
-                user_provided: RoaringBitmap<[0]>,
+                quantized: None,
             },
-        ]
-        "###);
+            fragments: FragmentConfigs(
+                [],
+            ),
+        },
+    ]
+    "###);
     let docid = index.external_documents_ids.get(&rtxn, "0").unwrap().unwrap();
     let embeddings = index.embeddings(&rtxn, docid).unwrap();
-    let embedding = &embeddings["manual"];
+    let (embedding, _) = &embeddings["manual"];
     assert!(!embedding.is_empty(), "{embedding:?}");
 
     index_scheduler
@@ -647,30 +714,32 @@ fn delete_document_containing_vector() {
         .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
         .collect::<Vec<_>>();
     snapshot!(serde_json::to_string(&documents).unwrap(), @"[]");
-    let conf = index.embedding_configs(&rtxn).unwrap();
+    let conf = index.embedding_configs().embedding_configs(&rtxn).unwrap();
     snapshot!(format!("{conf:#?}"), @r###"
-        [
-            IndexEmbeddingConfig {
-                name: "manual",
-                config: EmbeddingConfig {
-                    embedder_options: UserProvided(
-                        EmbedderOptions {
-                            dimensions: 3,
-                            distribution: None,
-                        },
-                    ),
-                    prompt: PromptData {
-                        template: "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
-                        max_bytes: Some(
-                            400,
-                        ),
+    [
+        IndexEmbeddingConfig {
+            name: "manual",
+            config: EmbeddingConfig {
+                embedder_options: UserProvided(
+                    EmbedderOptions {
+                        dimensions: 3,
+                        distribution: None,
                     },
-                    quantized: None,
+                ),
+                prompt: PromptData {
+                    template: "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
+                    max_bytes: Some(
+                        400,
+                    ),
                 },
-                user_provided: RoaringBitmap<[]>,
+                quantized: None,
             },
-        ]
-        "###);
+            fragments: FragmentConfigs(
+                [],
+            ),
+        },
+    ]
+    "###);
 }
 
 #[test]
@@ -800,7 +869,7 @@ fn delete_embedder_with_user_provided_vectors() {
             .unwrap()
             .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
             .collect::<Vec<_>>();
-        snapshot!(serde_json::to_string(&documents).unwrap(), @r###"[{"id":0,"doggo":"kefir","_vectors":{"manual":{"embeddings":[[0.0,0.0,0.0]],"regenerate":false}}},{"id":1,"doggo":"intel","_vectors":{"manual":{"embeddings":[[1.0,1.0,1.0]],"regenerate":false}}}]"###);
+        snapshot!(serde_json::to_string(&documents).unwrap(), @r###"[{"id":0,"doggo":"kefir","_vectors":{"manual":{"regenerate":false,"embeddings":[[0.0,0.0,0.0]]}}},{"id":1,"doggo":"intel","_vectors":{"manual":{"regenerate":false,"embeddings":[[1.0,1.0,1.0]]}}}]"###);
     }
 
     {
@@ -835,6 +904,6 @@ fn delete_embedder_with_user_provided_vectors() {
             .collect::<Vec<_>>();
 
         // FIXME: redaction
-        snapshot!(json_string!(serde_json::to_string(&documents).unwrap(), { "[]._vectors.doggo_embedder.embeddings" => "[vector]" }),  @r###""[{\"id\":0,\"doggo\":\"kefir\",\"_vectors\":{\"manual\":{\"embeddings\":[[0.0,0.0,0.0]],\"regenerate\":false},\"my_doggo_embedder\":{\"embeddings\":[[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]],\"regenerate\":false}}},{\"id\":1,\"doggo\":\"intel\",\"_vectors\":{\"manual\":{\"embeddings\":[[1.0,1.0,1.0]],\"regenerate\":false}}}]""###);
+        snapshot!(json_string!(serde_json::to_string(&documents).unwrap(), { "[]._vectors.doggo_embedder.embeddings" => "[vector]" }),  @r###""[{\"id\":0,\"doggo\":\"kefir\",\"_vectors\":{\"manual\":{\"regenerate\":false,\"embeddings\":[[0.0,0.0,0.0]]},\"my_doggo_embedder\":{\"regenerate\":false,\"embeddings\":[[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]]}}},{\"id\":1,\"doggo\":\"intel\",\"_vectors\":{\"manual\":{\"regenerate\":false,\"embeddings\":[[1.0,1.0,1.0]]}}}]""###);
     }
 }

crates/index-scheduler/src/scheduler/test_failure.rs

@@ -2,10 +2,9 @@ use std::time::Instant;
 
 use big_s::S;
 use meili_snap::snapshot;
-use meilisearch_types::milli::obkv_to_json;
 use meilisearch_types::milli::update::IndexDocumentsMethod::*;
 use meilisearch_types::milli::update::Setting;
-use meilisearch_types::milli::FilterableAttributesRule;
+use meilisearch_types::milli::{obkv_to_json, FilterableAttributesRule};
 use meilisearch_types::tasks::{Kind, KindWithContent};
 
 use crate::insta_snapshot::snapshot_index_scheduler;

crates/index-scheduler/src/test_utils.rs

@@ -37,6 +37,7 @@ pub(crate) enum FailureLocation {
     InsideCreateBatch,
     InsideProcessBatch,
     PanicInsideProcessBatch,
+    ProcessExport,
     ProcessUpgrade,
     AcquiringWtxn,
     UpdatingTaskAfterProcessBatchSuccess { task_uid: u32 },

crates/index-scheduler/src/utils.rs

@@ -1,7 +1,9 @@
 //! Utility functions on the DBs. Mainly getter and setters.
 
+use crate::milli::progress::EmbedderStats;
 use std::collections::{BTreeSet, HashSet};
 use std::ops::Bound;
+use std::sync::Arc;
 
 use meilisearch_types::batches::{Batch, BatchEnqueuedAt, BatchId, BatchStats};
 use meilisearch_types::heed::{Database, RoTxn, RwTxn};
@@ -27,6 +29,7 @@ pub struct ProcessingBatch {
     pub uid: BatchId,
     pub details: DetailsView,
     pub stats: BatchStats,
+    pub embedder_stats: Arc<EmbedderStats>,
 
     pub statuses: HashSet<Status>,
     pub kinds: HashSet<Kind>,
@@ -48,6 +51,7 @@ impl ProcessingBatch {
             uid,
             details: DetailsView::default(),
             stats: BatchStats::default(),
+            embedder_stats: Default::default(),
 
             statuses,
             kinds: HashSet::default(),
@@ -146,6 +150,7 @@ impl ProcessingBatch {
             progress: None,
             details: self.details.clone(),
             stats: self.stats.clone(),
+            embedder_stats: self.embedder_stats.as_ref().into(),
             started_at: self.started_at,
             finished_at: self.finished_at,
             enqueued_at: self.enqueued_at,
@@ -273,6 +278,7 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) {
         K::TaskCancelation { .. }
         | K::TaskDeletion { .. }
         | K::DumpCreation { .. }
+        | K::Export { .. }
         | K::UpgradeDatabase { .. }
         | K::SnapshotCreation => (),
     };
@@ -600,6 +606,9 @@ impl crate::IndexScheduler {
                     Details::Dump { dump_uid: _ } => {
                         assert_eq!(kind.as_kind(), Kind::DumpCreation);
                     }
+                    Details::Export { url: _, api_key: _, payload_size: _, indexes: _ } => {
+                        assert_eq!(kind.as_kind(), Kind::Export);
+                    }
                     Details::UpgradeDatabase { from: _, to: _ } => {
                         assert_eq!(kind.as_kind(), Kind::UpgradeDatabase);
                     }

crates/json-depth-checker/Cargo.toml

@@ -15,7 +15,7 @@ license.workspace = true
 serde_json = "1.0"
 
 [dev-dependencies]
-criterion = "0.5.1"
+criterion = "0.6.0"
 
 [[bench]]
 name = "depth"

crates/meili-snap/Cargo.toml

@@ -14,4 +14,6 @@ license.workspace = true
 # fixed version due to format breakages in v1.40
 insta = { version = "=1.39.0", features = ["json", "redactions"] }
 md5 = "0.7.0"
-once_cell = "1.20"
+once_cell = "1.21"
+regex-lite = "0.1.6"
+uuid = { version = "1.17.0", features = ["v4"] }

crates/meili-snap/src/lib.rs

@@ -4,9 +4,16 @@ use std::path::{Path, PathBuf};
 use std::sync::Mutex;
 
 pub use insta;
+use insta::internals::{Content, ContentPath};
 use once_cell::sync::Lazy;
+use regex_lite::Regex;
 
 static SNAPSHOT_NAMES: Lazy<Mutex<HashMap<PathBuf, usize>>> = Lazy::new(Mutex::default);
+/// A regex to match UUIDs in messages, specifically looking for the UUID v4 format
+static UUID_IN_MESSAGE_RE: Lazy<Regex> = Lazy::new(|| {
+    Regex::new(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}")
+        .unwrap()
+});
 
 /// Return the md5 hash of the given string
 pub fn hash_snapshot(snap: &str) -> String {
@@ -26,6 +33,39 @@ pub fn default_snapshot_settings_for_test<'a>(
     let filename = path.file_name().unwrap().to_str().unwrap();
     settings.set_omit_expression(true);
 
+    fn uuid_in_message_redaction(content: Content, _content_path: ContentPath) -> Content {
+        match &content {
+            Content::String(s) => {
+                let uuid_replaced = UUID_IN_MESSAGE_RE.replace_all(s, "[uuid]");
+                Content::String(uuid_replaced.to_string())
+            }
+            _ => content,
+        }
+    }
+
+    fn uuid_in_json_key_redaction(content: Content, _content_path: ContentPath) -> Content {
+        match content {
+            Content::Map(map) => {
+                let new_map = map
+                    .iter()
+                    .map(|(key, value)| match key {
+                        Content::String(s) => {
+                            let uuid_replaced = UUID_IN_MESSAGE_RE.replace_all(s, "[uuid]");
+                            (Content::String(uuid_replaced.to_string()), value.clone())
+                        }
+                        _ => (key.clone(), value.clone()),
+                    })
+                    .collect();
+                Content::Map(new_map)
+            }
+            _ => content,
+        }
+    }
+
+    settings.add_dynamic_redaction(".**.message", uuid_in_message_redaction);
+    settings.add_dynamic_redaction(".**.indexUid", uuid_in_message_redaction);
+    settings.add_dynamic_redaction(".**.facetsByIndex", uuid_in_json_key_redaction);
+
     let test_name = test_name.strip_suffix("::{{closure}}").unwrap_or(test_name);
     let test_name = test_name.rsplit("::").next().unwrap().to_owned();
 
@@ -232,6 +272,9 @@ macro_rules! json_string {
 #[cfg(test)]
 mod tests {
     use crate as meili_snap;
+    use crate::UUID_IN_MESSAGE_RE;
+    use uuid::Uuid;
+
     #[test]
     fn snap() {
         snapshot_hash!(10, @"d3d9446802a44259755d38e6d163e820");
@@ -279,4 +322,14 @@ mod tests {
             // snapshot_hash!("", name: "", @"d41d8cd98f00b204e9800998ecf8427e");
         }
     }
+
+    #[test]
+    fn uuid_in_message_regex() {
+        let uuid1 = Uuid::new_v4();
+        let uuid2 = Uuid::new_v4();
+        let uuid3 = Uuid::new_v4();
+        let to_replace = format!("1 {uuid1} 2 {uuid2} 3 {uuid3} 4");
+        let replaced = UUID_IN_MESSAGE_RE.replace_all(to_replace.as_str(), "[uuid]");
+        assert_eq!(replaced, "1 [uuid] 2 [uuid] 3 [uuid] 4");
+    }
 }

crates/meilisearch-auth/Cargo.toml

@@ -17,10 +17,10 @@ hmac = "0.12.1"
 maplit = "1.0.2"
 meilisearch-types = { path = "../meilisearch-types" }
 rand = "0.8.5"
-roaring = { version = "0.10.10", features = ["serde"] }
-serde = { version = "1.0.217", features = ["derive"] }
-serde_json = { version = "1.0.135", features = ["preserve_order"] }
-sha2 = "0.10.8"
-thiserror = "2.0.9"
-time = { version = "0.3.37", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-uuid = { version = "1.11.0", features = ["serde", "v4"] }
+roaring = { version = "0.10.12", features = ["serde"] }
+serde = { version = "1.0.219", features = ["derive"] }
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
+sha2 = "0.10.9"
+thiserror = "2.0.12"
+time = { version = "0.3.41", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+uuid = { version = "1.17.0", features = ["serde", "v4"] }

crates/meilisearch-auth/src/lib.rs

@@ -165,6 +165,7 @@ impl AuthController {
     }
 }
 
+#[derive(Debug)]
 pub struct AuthFilter {
     search_rules: Option<SearchRules>,
     key_authorized_indexes: SearchRules,
@@ -349,6 +350,7 @@ pub struct IndexSearchRules {
 }
 
 fn generate_default_keys(store: &HeedAuthStore) -> Result<()> {
+    store.put_api_key(Key::default_chat())?;
     store.put_api_key(Key::default_admin())?;
     store.put_api_key(Key::default_search())?;
 

crates/meilisearch-auth/src/store.rs

@@ -125,6 +125,12 @@ impl HeedAuthStore {
                 Action::MetricsAll => {
                     actions.insert(Action::MetricsGet);
                 }
+                Action::ChatsAll => {
+                    actions.extend([Action::ChatsGet, Action::ChatsDelete]);
+                }
+                Action::ChatsSettingsAll => {
+                    actions.extend([Action::ChatsSettingsGet, Action::ChatsSettingsUpdate]);
+                }
                 other => {
                     actions.insert(*other);
                 }

crates/meilisearch-types/Cargo.toml

@@ -11,37 +11,38 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-actix-web = { version = "4.9.0", default-features = false }
-anyhow = "1.0.95"
-bumpalo = "3.16.0"
+actix-web = { version = "4.11.0", default-features = false }
+anyhow = "1.0.98"
+bumpalo = "3.18.1"
 bumparaw-collections = "0.1.4"
-convert_case = "0.6.0"
+byte-unit = { version = "5.1.6", features = ["serde"] }
+convert_case = "0.8.0"
 csv = "1.3.1"
 deserr = { version = "0.6.3", features = ["actix-web"] }
-either = { version = "1.13.0", features = ["serde"] }
+either = { version = "1.15.0", features = ["serde"] }
 enum-iterator = "2.1.0"
 file-store = { path = "../file-store" }
-flate2 = "1.0.35"
+flate2 = "1.1.2"
 fst = "0.4.7"
 memmap2 = "0.9.5"
 milli = { path = "../milli" }
-roaring = { version = "0.10.10", features = ["serde"] }
-rustc-hash = "2.1.0"
-serde = { version = "1.0.217", features = ["derive"] }
+roaring = { version = "0.10.12", features = ["serde"] }
+rustc-hash = "2.1.1"
+serde = { version = "1.0.219", features = ["derive"] }
 serde-cs = "0.2.4"
-serde_json = { version = "1.0.135", features = ["preserve_order"] }
-tar = "0.4.43"
-tempfile = "3.15.0"
-thiserror = "2.0.9"
-time = { version = "0.3.37", features = [
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
+tar = "0.4.44"
+tempfile = "3.20.0"
+thiserror = "2.0.12"
+time = { version = "0.3.41", features = [
     "serde-well-known",
     "formatting",
     "parsing",
     "macros",
 ] }
-tokio = "1.43"
-utoipa = { version = "5.3.1", features = ["macros"] }
-uuid = { version = "1.11.0", features = ["serde", "v4"] }
+tokio = "1.45"
+utoipa = { version = "5.4.0", features = ["macros"] }
+uuid = { version = "1.17.0", features = ["serde", "v4"] }
 
 [dev-dependencies]
 # fixed version due to format breakages in v1.40

crates/meilisearch-types/src/batch_view.rs

@@ -3,7 +3,7 @@ use serde::Serialize;
 use time::{Duration, OffsetDateTime};
 use utoipa::ToSchema;
 
-use crate::batches::{Batch, BatchId, BatchStats};
+use crate::batches::{Batch, BatchId, BatchStats, EmbedderStatsView};
 use crate::task_view::DetailsView;
 use crate::tasks::serialize_duration;
 
@@ -14,7 +14,7 @@ pub struct BatchView {
     pub uid: BatchId,
     pub progress: Option<ProgressView>,
     pub details: DetailsView,
-    pub stats: BatchStats,
+    pub stats: BatchStatsView,
     #[serde(serialize_with = "serialize_duration", default)]
     pub duration: Option<Duration>,
     #[serde(with = "time::serde::rfc3339", default)]
@@ -25,13 +25,26 @@ pub struct BatchView {
     pub batch_strategy: String,
 }
 
+#[derive(Debug, Clone, Serialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+#[schema(rename_all = "camelCase")]
+pub struct BatchStatsView {
+    #[serde(flatten)]
+    pub stats: BatchStats,
+    #[serde(skip_serializing_if = "EmbedderStatsView::skip_serializing", default)]
+    pub embedder_requests: EmbedderStatsView,
+}
+
 impl BatchView {
     pub fn from_batch(batch: &Batch) -> Self {
         Self {
             uid: batch.uid,
             progress: batch.progress.clone(),
             details: batch.details.clone(),
-            stats: batch.stats.clone(),
+            stats: BatchStatsView {
+                stats: batch.stats.clone(),
+                embedder_requests: batch.embedder_stats.clone(),
+            },
             duration: batch.finished_at.map(|finished_at| finished_at - batch.started_at),
             started_at: batch.started_at,
             finished_at: batch.finished_at,

crates/meilisearch-types/src/batches.rs

@@ -1,6 +1,6 @@
 use std::collections::BTreeMap;
 
-use milli::progress::ProgressView;
+use milli::progress::{EmbedderStats, ProgressView};
 use serde::{Deserialize, Serialize};
 use time::OffsetDateTime;
 use utoipa::ToSchema;
@@ -19,6 +19,8 @@ pub struct Batch {
     pub progress: Option<ProgressView>,
     pub details: DetailsView,
     pub stats: BatchStats,
+    #[serde(skip_serializing_if = "EmbedderStatsView::skip_serializing", default)]
+    pub embedder_stats: EmbedderStatsView,
 
     #[serde(with = "time::serde::rfc3339")]
     pub started_at: OffsetDateTime,
@@ -43,6 +45,7 @@ impl PartialEq for Batch {
             progress,
             details,
             stats,
+            embedder_stats,
             started_at,
             finished_at,
             enqueued_at,
@@ -53,6 +56,7 @@ impl PartialEq for Batch {
             && progress.is_none() == other.progress.is_none()
             && details == &other.details
             && stats == &other.stats
+            && embedder_stats == &other.embedder_stats
             && started_at == &other.started_at
             && finished_at == &other.finished_at
             && enqueued_at == &other.enqueued_at
@@ -83,3 +87,30 @@ pub struct BatchStats {
     #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
     pub internal_database_sizes: serde_json::Map<String, serde_json::Value>,
 }
+
+#[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+#[schema(rename_all = "camelCase")]
+pub struct EmbedderStatsView {
+    pub total: usize,
+    pub failed: usize,
+    #[serde(skip_serializing_if = "Option::is_none", default)]
+    pub last_error: Option<String>,
+}
+
+impl From<&EmbedderStats> for EmbedderStatsView {
+    fn from(stats: &EmbedderStats) -> Self {
+        let errors = stats.errors.read().unwrap_or_else(|p| p.into_inner());
+        Self {
+            total: stats.total_count.load(std::sync::atomic::Ordering::Relaxed),
+            failed: errors.1 as usize,
+            last_error: errors.0.clone(),
+        }
+    }
+}
+
+impl EmbedderStatsView {
+    pub fn skip_serializing(&self) -> bool {
+        self.total == 0 && self.failed == 0 && self.last_error.is_none()
+    }
+}

crates/meilisearch-types/src/deserr/mod.rs

@@ -4,9 +4,12 @@ use std::marker::PhantomData;
 use std::ops::ControlFlow;
 
 use deserr::errors::{JsonError, QueryParamError};
-use deserr::{take_cf_content, DeserializeError, IntoValue, MergeWithError, ValuePointerRef};
+use deserr::{
+    take_cf_content, DeserializeError, Deserr, IntoValue, MergeWithError, ValuePointerRef,
+};
+use milli::update::ChatSettings;
 
-use crate::error::deserr_codes::*;
+use crate::error::deserr_codes::{self, *};
 use crate::error::{
     Code, DeserrParseBoolError, DeserrParseIntError, ErrorCode, InvalidTaskDateError,
     ParseOffsetDateTimeError,
@@ -33,6 +36,7 @@ pub struct DeserrError<Format, C: Default + ErrorCode> {
     pub code: Code,
     _phantom: PhantomData<(Format, C)>,
 }
+
 impl<Format, C: Default + ErrorCode> DeserrError<Format, C> {
     pub fn new(msg: String, code: Code) -> Self {
         Self { msg, code, _phantom: PhantomData }
@@ -117,6 +121,16 @@ impl<C: Default + ErrorCode> DeserializeError for DeserrQueryParamError<C> {
     }
 }
 
+impl Deserr<DeserrError<DeserrJson, deserr_codes::InvalidSettingsIndexChat>> for ChatSettings {
+    fn deserialize_from_value<V: IntoValue>(
+        value: deserr::Value<V>,
+        location: ValuePointerRef,
+    ) -> Result<Self, DeserrError<DeserrJson, deserr_codes::InvalidSettingsIndexChat>> {
+        Deserr::<JsonError>::deserialize_from_value(value, location)
+            .map_err(|e| DeserrError::new(e.to_string(), InvalidSettingsIndexChat.error_code()))
+    }
+}
+
 pub fn immutable_field_error(field: &str, accepted: &[&str], code: Code) -> DeserrJsonError {
     let msg = format!(
         "Immutable field `{field}`: expected one of {}",

crates/meilisearch-types/src/error.rs

@@ -194,200 +194,229 @@ macro_rules! make_error_codes {
 
 // An exhaustive list of all the error codes used by meilisearch.
 make_error_codes! {
-ApiKeyAlreadyExists                   , InvalidRequest       , CONFLICT ;
-ApiKeyNotFound                        , InvalidRequest       , NOT_FOUND ;
-BadParameter                          , InvalidRequest       , BAD_REQUEST;
-BadRequest                            , InvalidRequest       , BAD_REQUEST;
-DatabaseSizeLimitReached              , Internal             , INTERNAL_SERVER_ERROR;
-DocumentNotFound                      , InvalidRequest       , NOT_FOUND;
-DumpAlreadyProcessing                 , InvalidRequest       , CONFLICT;
-DumpNotFound                          , InvalidRequest       , NOT_FOUND;
-DumpProcessFailed                     , Internal             , INTERNAL_SERVER_ERROR;
-DuplicateIndexFound                   , InvalidRequest       , BAD_REQUEST;
-ImmutableApiKeyActions                , InvalidRequest       , BAD_REQUEST;
-ImmutableApiKeyCreatedAt              , InvalidRequest       , BAD_REQUEST;
-ImmutableApiKeyExpiresAt              , InvalidRequest       , BAD_REQUEST;
-ImmutableApiKeyIndexes                , InvalidRequest       , BAD_REQUEST;
-ImmutableApiKeyKey                    , InvalidRequest       , BAD_REQUEST;
-ImmutableApiKeyUid                    , InvalidRequest       , BAD_REQUEST;
-ImmutableApiKeyUpdatedAt              , InvalidRequest       , BAD_REQUEST;
-ImmutableIndexCreatedAt               , InvalidRequest       , BAD_REQUEST;
-ImmutableIndexUid                     , InvalidRequest       , BAD_REQUEST;
-ImmutableIndexUpdatedAt               , InvalidRequest       , BAD_REQUEST;
-IndexAlreadyExists                    , InvalidRequest       , CONFLICT ;
-IndexCreationFailed                   , Internal             , INTERNAL_SERVER_ERROR;
-IndexNotFound                         , InvalidRequest       , NOT_FOUND;
-IndexPrimaryKeyAlreadyExists          , InvalidRequest       , BAD_REQUEST ;
-IndexPrimaryKeyMultipleCandidatesFound, InvalidRequest       , BAD_REQUEST;
-IndexPrimaryKeyNoCandidateFound       , InvalidRequest       , BAD_REQUEST ;
-Internal                              , Internal             , INTERNAL_SERVER_ERROR ;
-InvalidApiKey                         , Auth                 , FORBIDDEN ;
-InvalidApiKeyActions                  , InvalidRequest       , BAD_REQUEST ;
-InvalidApiKeyDescription              , InvalidRequest       , BAD_REQUEST ;
-InvalidApiKeyExpiresAt                , InvalidRequest       , BAD_REQUEST ;
-InvalidApiKeyIndexes                  , InvalidRequest       , BAD_REQUEST ;
-InvalidApiKeyLimit                    , InvalidRequest       , BAD_REQUEST ;
-InvalidApiKeyName                     , InvalidRequest       , BAD_REQUEST ;
-InvalidApiKeyOffset                   , InvalidRequest       , BAD_REQUEST ;
-InvalidApiKeyUid                      , InvalidRequest       , BAD_REQUEST ;
-InvalidContentType                    , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
-InvalidDocumentCsvDelimiter           , InvalidRequest       , BAD_REQUEST ;
-InvalidDocumentFields                 , InvalidRequest       , BAD_REQUEST ;
-InvalidDocumentRetrieveVectors        , InvalidRequest       , BAD_REQUEST ;
-MissingDocumentFilter                 , InvalidRequest       , BAD_REQUEST ;
-MissingDocumentEditionFunction        , InvalidRequest       , BAD_REQUEST ;
-InvalidDocumentFilter                 , InvalidRequest       , BAD_REQUEST ;
-InvalidDocumentGeoField               , InvalidRequest       , BAD_REQUEST ;
-InvalidVectorDimensions               , InvalidRequest       , BAD_REQUEST ;
-InvalidVectorsType                    , InvalidRequest       , BAD_REQUEST ;
-InvalidDocumentId                     , InvalidRequest       , BAD_REQUEST ;
-InvalidDocumentIds                    , InvalidRequest       , BAD_REQUEST ;
-InvalidDocumentLimit                  , InvalidRequest       , BAD_REQUEST ;
-InvalidDocumentOffset                 , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchEmbedder                 , InvalidRequest       , BAD_REQUEST ;
-InvalidSimilarEmbedder                , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchHybridQuery              , InvalidRequest       , BAD_REQUEST ;
-InvalidIndexLimit                     , InvalidRequest       , BAD_REQUEST ;
-InvalidIndexOffset                    , InvalidRequest       , BAD_REQUEST ;
-InvalidIndexPrimaryKey                , InvalidRequest       , BAD_REQUEST ;
-InvalidIndexUid                       , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchFacets              , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchFacetsByIndex       , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchFacetOrder          , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchFederated           , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchFederationOptions   , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchMaxValuesPerFacet   , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchMergeFacets         , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchQueryFacets         , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchQueryPagination     , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchQueryRankingRules   , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchQueryPosition       , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchRemote              , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchWeight              , InvalidRequest       , BAD_REQUEST ;
-InvalidNetworkRemotes                 , InvalidRequest       , BAD_REQUEST ;
-InvalidNetworkSelf                    , InvalidRequest       , BAD_REQUEST ;
-InvalidNetworkSearchApiKey            , InvalidRequest       , BAD_REQUEST ;
-InvalidNetworkUrl                     , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchAttributesToSearchOn     , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchAttributesToCrop         , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchAttributesToHighlight    , InvalidRequest       , BAD_REQUEST ;
-InvalidSimilarAttributesToRetrieve    , InvalidRequest       , BAD_REQUEST ;
-InvalidSimilarRetrieveVectors         , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchAttributesToRetrieve     , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchRankingScoreThreshold    , InvalidRequest       , BAD_REQUEST ;
-InvalidSimilarRankingScoreThreshold   , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchRetrieveVectors          , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchCropLength               , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchCropMarker               , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchFacets                   , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchSemanticRatio            , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchLocales                  , InvalidRequest       , BAD_REQUEST ;
-InvalidFacetSearchExhaustiveFacetCount, InvalidRequest       , BAD_REQUEST ;
-InvalidFacetSearchFacetName           , InvalidRequest       , BAD_REQUEST ;
-InvalidSimilarId                      , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchFilter                   , InvalidRequest       , BAD_REQUEST ;
-InvalidSimilarFilter                  , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchHighlightPostTag         , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchHighlightPreTag          , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchHitsPerPage              , InvalidRequest       , BAD_REQUEST ;
-InvalidSimilarLimit                   , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchLimit                    , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchMatchingStrategy         , InvalidRequest       , BAD_REQUEST ;
-InvalidSimilarOffset                  , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchOffset                   , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchPage                     , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchQ                        , InvalidRequest       , BAD_REQUEST ;
-InvalidFacetSearchQuery               , InvalidRequest       , BAD_REQUEST ;
-InvalidFacetSearchName                , InvalidRequest       , BAD_REQUEST ;
-FacetSearchDisabled                   , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchVector                   , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchShowMatchesPosition      , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchShowRankingScore         , InvalidRequest       , BAD_REQUEST ;
-InvalidSimilarShowRankingScore        , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchShowRankingScoreDetails  , InvalidRequest       , BAD_REQUEST ;
-InvalidSimilarShowRankingScoreDetails , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchSort                     , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchDistinct                 , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsDisplayedAttributes    , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsDistinctAttribute      , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsProximityPrecision     , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsFacetSearch            , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsPrefixSearch           , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsFaceting               , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsFilterableAttributes   , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsPagination             , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsSearchCutoffMs         , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsEmbedders              , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsRankingRules           , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsSearchableAttributes   , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsSortableAttributes     , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsStopWords              , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsNonSeparatorTokens     , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsSeparatorTokens        , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsDictionary             , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsSynonyms               , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsTypoTolerance          , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsLocalizedAttributes    , InvalidRequest       , BAD_REQUEST ;
-InvalidState                          , Internal             , INTERNAL_SERVER_ERROR ;
-InvalidStoreFile                      , Internal             , INTERNAL_SERVER_ERROR ;
-InvalidSwapDuplicateIndexFound        , InvalidRequest       , BAD_REQUEST ;
-InvalidSwapIndexes                    , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskAfterEnqueuedAt            , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskAfterFinishedAt            , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskAfterStartedAt             , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskBeforeEnqueuedAt           , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskBeforeFinishedAt           , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskBeforeStartedAt            , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskCanceledBy                 , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskFrom                       , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskLimit                      , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskReverse                    , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskStatuses                   , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskTypes                      , InvalidRequest       , BAD_REQUEST ;
-InvalidTaskUids                       , InvalidRequest       , BAD_REQUEST  ;
-InvalidBatchUids                      , InvalidRequest       , BAD_REQUEST  ;
-IoError                               , System               , UNPROCESSABLE_ENTITY;
-FeatureNotEnabled                     , InvalidRequest       , BAD_REQUEST ;
-MalformedPayload                      , InvalidRequest       , BAD_REQUEST ;
-MaxFieldsLimitExceeded                , InvalidRequest       , BAD_REQUEST ;
-MissingApiKeyActions                  , InvalidRequest       , BAD_REQUEST ;
-MissingApiKeyExpiresAt                , InvalidRequest       , BAD_REQUEST ;
-MissingApiKeyIndexes                  , InvalidRequest       , BAD_REQUEST ;
-MissingAuthorizationHeader            , Auth                 , UNAUTHORIZED ;
-MissingContentType                    , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
-MissingDocumentId                     , InvalidRequest       , BAD_REQUEST ;
-MissingFacetSearchFacetName           , InvalidRequest       , BAD_REQUEST ;
-MissingIndexUid                       , InvalidRequest       , BAD_REQUEST ;
-MissingMasterKey                      , Auth                 , UNAUTHORIZED ;
-MissingNetworkUrl                     , InvalidRequest       , BAD_REQUEST ;
-MissingPayload                        , InvalidRequest       , BAD_REQUEST ;
-MissingSearchHybrid                   , InvalidRequest       , BAD_REQUEST ;
-MissingSwapIndexes                    , InvalidRequest       , BAD_REQUEST ;
-MissingTaskFilters                    , InvalidRequest       , BAD_REQUEST ;
-NoSpaceLeftOnDevice                   , System               , UNPROCESSABLE_ENTITY;
-PayloadTooLarge                       , InvalidRequest       , PAYLOAD_TOO_LARGE ;
-RemoteBadResponse                     , System               , BAD_GATEWAY ;
-RemoteBadRequest                      , InvalidRequest       , BAD_REQUEST ;
-RemoteCouldNotSendRequest             , System               , BAD_GATEWAY ;
-RemoteInvalidApiKey                   , Auth                 , FORBIDDEN ;
-RemoteRemoteError                     , System               , BAD_GATEWAY ;
-RemoteTimeout                         , System               , BAD_GATEWAY ;
-TooManySearchRequests                 , System               , SERVICE_UNAVAILABLE ;
-TaskNotFound                          , InvalidRequest       , NOT_FOUND ;
-TaskFileNotFound                      , InvalidRequest       , NOT_FOUND ;
-BatchNotFound                         , InvalidRequest       , NOT_FOUND ;
-TooManyOpenFiles                      , System               , UNPROCESSABLE_ENTITY ;
-TooManyVectors                        , InvalidRequest       , BAD_REQUEST ;
-UnretrievableDocument                 , Internal             , BAD_REQUEST ;
-UnretrievableErrorCode                , InvalidRequest       , BAD_REQUEST ;
-UnsupportedMediaType                  , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
+ApiKeyAlreadyExists                            , InvalidRequest       , CONFLICT ;
+ApiKeyNotFound                                 , InvalidRequest       , NOT_FOUND ;
+BadParameter                                   , InvalidRequest       , BAD_REQUEST;
+BadRequest                                     , InvalidRequest       , BAD_REQUEST;
+DatabaseSizeLimitReached                       , Internal             , INTERNAL_SERVER_ERROR;
+DocumentNotFound                               , InvalidRequest       , NOT_FOUND;
+DumpAlreadyProcessing                          , InvalidRequest       , CONFLICT;
+DumpNotFound                                   , InvalidRequest       , NOT_FOUND;
+DumpProcessFailed                              , Internal             , INTERNAL_SERVER_ERROR;
+DuplicateIndexFound                            , InvalidRequest       , BAD_REQUEST;
+ImmutableApiKeyActions                         , InvalidRequest       , BAD_REQUEST;
+ImmutableApiKeyCreatedAt                       , InvalidRequest       , BAD_REQUEST;
+ImmutableApiKeyExpiresAt                       , InvalidRequest       , BAD_REQUEST;
+ImmutableApiKeyIndexes                         , InvalidRequest       , BAD_REQUEST;
+ImmutableApiKeyKey                             , InvalidRequest       , BAD_REQUEST;
+ImmutableApiKeyUid                             , InvalidRequest       , BAD_REQUEST;
+ImmutableApiKeyUpdatedAt                       , InvalidRequest       , BAD_REQUEST;
+ImmutableIndexCreatedAt                        , InvalidRequest       , BAD_REQUEST;
+ImmutableIndexUid                              , InvalidRequest       , BAD_REQUEST;
+ImmutableIndexUpdatedAt                        , InvalidRequest       , BAD_REQUEST;
+IndexAlreadyExists                             , InvalidRequest       , CONFLICT ;
+IndexCreationFailed                            , Internal             , INTERNAL_SERVER_ERROR;
+IndexNotFound                                  , InvalidRequest       , NOT_FOUND;
+IndexPrimaryKeyAlreadyExists                   , InvalidRequest       , BAD_REQUEST ;
+IndexPrimaryKeyMultipleCandidatesFound         , InvalidRequest       , BAD_REQUEST;
+IndexPrimaryKeyNoCandidateFound                , InvalidRequest       , BAD_REQUEST ;
+Internal                                       , Internal             , INTERNAL_SERVER_ERROR ;
+InvalidApiKey                                  , Auth                 , FORBIDDEN ;
+InvalidApiKeyActions                           , InvalidRequest       , BAD_REQUEST ;
+InvalidApiKeyDescription                       , InvalidRequest       , BAD_REQUEST ;
+InvalidApiKeyExpiresAt                         , InvalidRequest       , BAD_REQUEST ;
+InvalidApiKeyIndexes                           , InvalidRequest       , BAD_REQUEST ;
+InvalidApiKeyLimit                             , InvalidRequest       , BAD_REQUEST ;
+InvalidApiKeyName                              , InvalidRequest       , BAD_REQUEST ;
+InvalidApiKeyOffset                            , InvalidRequest       , BAD_REQUEST ;
+InvalidApiKeyUid                               , InvalidRequest       , BAD_REQUEST ;
+InvalidContentType                             , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
+InvalidDocumentCsvDelimiter                    , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentFields                          , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentRetrieveVectors                 , InvalidRequest       , BAD_REQUEST ;
+MissingDocumentFilter                          , InvalidRequest       , BAD_REQUEST ;
+MissingDocumentEditionFunction                 , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentFilter                          , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentGeoField                        , InvalidRequest       , BAD_REQUEST ;
+InvalidVectorDimensions                        , InvalidRequest       , BAD_REQUEST ;
+InvalidVectorsType                             , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentId                              , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentIds                             , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentLimit                           , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentOffset                          , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchEmbedder                          , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarEmbedder                         , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchHybridQuery                       , InvalidRequest       , BAD_REQUEST ;
+InvalidIndexLimit                              , InvalidRequest       , BAD_REQUEST ;
+InvalidIndexOffset                             , InvalidRequest       , BAD_REQUEST ;
+InvalidIndexPrimaryKey                         , InvalidRequest       , BAD_REQUEST ;
+InvalidIndexUid                                , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchFacets                       , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchFacetsByIndex                , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchFacetOrder                   , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchFederated                    , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchFederationOptions            , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchMaxValuesPerFacet            , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchMergeFacets                  , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchQueryFacets                  , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchQueryPagination              , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchQueryRankingRules            , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchQueryPosition                , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchRemote                       , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchWeight                       , InvalidRequest       , BAD_REQUEST ;
+InvalidNetworkRemotes                          , InvalidRequest       , BAD_REQUEST ;
+InvalidNetworkSelf                             , InvalidRequest       , BAD_REQUEST ;
+InvalidNetworkSearchApiKey                     , InvalidRequest       , BAD_REQUEST ;
+InvalidNetworkUrl                              , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchAttributesToSearchOn              , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchAttributesToCrop                  , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchAttributesToHighlight             , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarAttributesToRetrieve             , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarRetrieveVectors                  , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchAttributesToRetrieve              , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchRankingScoreThreshold             , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarRankingScoreThreshold            , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchRetrieveVectors                   , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchCropLength                        , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchCropMarker                        , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchFacets                            , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchSemanticRatio                     , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchLocales                           , InvalidRequest       , BAD_REQUEST ;
+InvalidFacetSearchExhaustiveFacetCount         , InvalidRequest       , BAD_REQUEST ;
+InvalidFacetSearchFacetName                    , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarId                               , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchFilter                            , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarFilter                           , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchHighlightPostTag                  , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchHighlightPreTag                   , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchHitsPerPage                       , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarLimit                            , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchLimit                             , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchMatchingStrategy                  , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarOffset                           , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchOffset                            , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchPage                              , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchQ                                 , InvalidRequest       , BAD_REQUEST ;
+InvalidFacetSearchQuery                        , InvalidRequest       , BAD_REQUEST ;
+InvalidFacetSearchName                         , InvalidRequest       , BAD_REQUEST ;
+FacetSearchDisabled                            , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchVector                            , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchMedia                             , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchShowMatchesPosition               , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchShowRankingScore                  , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarShowRankingScore                 , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchShowRankingScoreDetails           , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarShowRankingScoreDetails          , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchSort                              , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchDistinct                          , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchMediaAndVector                    , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsDisplayedAttributes             , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsDistinctAttribute               , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsProximityPrecision              , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsFacetSearch                     , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsPrefixSearch                    , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsFaceting                        , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsFilterableAttributes            , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsPagination                      , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsSearchCutoffMs                  , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsEmbedders                       , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsRankingRules                    , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsSearchableAttributes            , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsSortableAttributes              , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsStopWords                       , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsNonSeparatorTokens              , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsSeparatorTokens                 , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsDictionary                      , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsSynonyms                        , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsTypoTolerance                   , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsLocalizedAttributes             , InvalidRequest       , BAD_REQUEST ;
+InvalidState                                   , Internal             , INTERNAL_SERVER_ERROR ;
+InvalidStoreFile                               , Internal             , INTERNAL_SERVER_ERROR ;
+InvalidSwapDuplicateIndexFound                 , InvalidRequest       , BAD_REQUEST ;
+InvalidSwapIndexes                             , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskAfterEnqueuedAt                     , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskAfterFinishedAt                     , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskAfterStartedAt                      , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskBeforeEnqueuedAt                    , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskBeforeFinishedAt                    , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskBeforeStartedAt                     , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskCanceledBy                          , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskFrom                                , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskLimit                               , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskReverse                             , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskStatuses                            , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskTypes                               , InvalidRequest       , BAD_REQUEST ;
+InvalidTaskUids                                , InvalidRequest       , BAD_REQUEST  ;
+InvalidBatchUids                               , InvalidRequest       , BAD_REQUEST  ;
+IoError                                        , System               , UNPROCESSABLE_ENTITY;
+FeatureNotEnabled                              , InvalidRequest       , BAD_REQUEST ;
+MalformedPayload                               , InvalidRequest       , BAD_REQUEST ;
+MaxFieldsLimitExceeded                         , InvalidRequest       , BAD_REQUEST ;
+MissingApiKeyActions                           , InvalidRequest       , BAD_REQUEST ;
+MissingApiKeyExpiresAt                         , InvalidRequest       , BAD_REQUEST ;
+MissingApiKeyIndexes                           , InvalidRequest       , BAD_REQUEST ;
+MissingAuthorizationHeader                     , Auth                 , UNAUTHORIZED ;
+MissingContentType                             , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
+MissingDocumentId                              , InvalidRequest       , BAD_REQUEST ;
+MissingFacetSearchFacetName                    , InvalidRequest       , BAD_REQUEST ;
+MissingIndexUid                                , InvalidRequest       , BAD_REQUEST ;
+MissingMasterKey                               , Auth                 , UNAUTHORIZED ;
+MissingNetworkUrl                              , InvalidRequest       , BAD_REQUEST ;
+MissingPayload                                 , InvalidRequest       , BAD_REQUEST ;
+MissingSearchHybrid                            , InvalidRequest       , BAD_REQUEST ;
+MissingSwapIndexes                             , InvalidRequest       , BAD_REQUEST ;
+MissingTaskFilters                             , InvalidRequest       , BAD_REQUEST ;
+NoSpaceLeftOnDevice                            , System               , UNPROCESSABLE_ENTITY;
+PayloadTooLarge                                , InvalidRequest       , PAYLOAD_TOO_LARGE ;
+RemoteBadResponse                              , System               , BAD_GATEWAY ;
+RemoteBadRequest                               , InvalidRequest       , BAD_REQUEST ;
+RemoteCouldNotSendRequest                      , System               , BAD_GATEWAY ;
+RemoteInvalidApiKey                            , Auth                 , FORBIDDEN ;
+RemoteRemoteError                              , System               , BAD_GATEWAY ;
+RemoteTimeout                                  , System               , BAD_GATEWAY ;
+TooManySearchRequests                          , System               , SERVICE_UNAVAILABLE ;
+TaskNotFound                                   , InvalidRequest       , NOT_FOUND ;
+TaskFileNotFound                               , InvalidRequest       , NOT_FOUND ;
+BatchNotFound                                  , InvalidRequest       , NOT_FOUND ;
+TooManyOpenFiles                               , System               , UNPROCESSABLE_ENTITY ;
+TooManyVectors                                 , InvalidRequest       , BAD_REQUEST ;
+UnretrievableDocument                          , Internal             , BAD_REQUEST ;
+UnretrievableErrorCode                         , InvalidRequest       , BAD_REQUEST ;
+UnsupportedMediaType                           , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
 
 // Experimental features
-VectorEmbeddingError                  , InvalidRequest       , BAD_REQUEST ;
-NotFoundSimilarId                     , InvalidRequest       , BAD_REQUEST ;
-InvalidDocumentEditionContext         , InvalidRequest       , BAD_REQUEST ;
-InvalidDocumentEditionFunctionFilter  , InvalidRequest       , BAD_REQUEST ;
-EditDocumentsByFunctionError          , InvalidRequest       , BAD_REQUEST
+VectorEmbeddingError                           , InvalidRequest       , BAD_REQUEST ;
+NotFoundSimilarId                              , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentEditionContext                  , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentEditionFunctionFilter           , InvalidRequest       , BAD_REQUEST ;
+EditDocumentsByFunctionError                   , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsIndexChat                       , InvalidRequest       , BAD_REQUEST ;
+// Export
+InvalidExportUrl                               , InvalidRequest       , BAD_REQUEST ;
+InvalidExportApiKey                            , InvalidRequest       , BAD_REQUEST ;
+InvalidExportPayloadSize                       , InvalidRequest       , BAD_REQUEST ;
+InvalidExportIndexesPatterns                   , InvalidRequest       , BAD_REQUEST ;
+InvalidExportIndexFilter                       , InvalidRequest       , BAD_REQUEST ;
+InvalidExportIndexOverrideSettings             , InvalidRequest       , BAD_REQUEST ;
+// Experimental features - Chat Completions
+UnimplementedExternalFunctionCalling           , InvalidRequest       , NOT_IMPLEMENTED ;
+UnimplementedNonStreamingChatCompletions       , InvalidRequest       , NOT_IMPLEMENTED ;
+UnimplementedMultiChoiceChatCompletions        , InvalidRequest       , NOT_IMPLEMENTED ;
+ChatNotFound                                   , InvalidRequest       , NOT_FOUND   ;
+InvalidChatSettingDocumentTemplate             , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionOrgId                     , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionProjectId                 , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionApiVersion                , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionDeploymentId              , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionSource                    , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionBaseApi                   , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionApiKey                    , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionPrompts                   , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionSystemPrompt              , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionSearchDescriptionPrompt   , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionSearchQueryParamPrompt    , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionSearchIndexUidParamPrompt , InvalidRequest       , BAD_REQUEST ;
+InvalidChatCompletionPreQueryPrompt            , InvalidRequest       , BAD_REQUEST
 }
 
 impl ErrorCode for JoinError {
@@ -430,12 +459,14 @@ impl ErrorCode for milli::Error {
                     | UserError::InvalidSettingsDimensions { .. }
                     | UserError::InvalidUrl { .. }
                     | UserError::InvalidSettingsDocumentTemplateMaxBytes { .. }
+                    | UserError::InvalidChatSettingsDocumentTemplateMaxBytes
                     | UserError::InvalidPrompt(_)
                     | UserError::InvalidDisableBinaryQuantization { .. }
                     | UserError::InvalidSourceForNested { .. }
                     | UserError::MissingSourceForNested { .. }
                     | UserError::InvalidSettingsEmbedder { .. } => Code::InvalidSettingsEmbedders,
                     UserError::TooManyEmbedders(_) => Code::InvalidSettingsEmbedders,
+                    UserError::TooManyFragments(_) => Code::InvalidSettingsEmbedders,
                     UserError::InvalidPromptForEmbeddings(..) => Code::InvalidSettingsEmbedders,
                     UserError::NoPrimaryKeyCandidateFound => Code::IndexPrimaryKeyNoCandidateFound,
                     UserError::MultiplePrimaryKeyCandidatesFound { .. } => {

crates/meilisearch-types/src/features.rs

@@ -2,6 +2,14 @@ use std::collections::BTreeMap;
 
 use serde::{Deserialize, Serialize};
 
+use crate::error::{Code, ResponseError};
+
+pub const DEFAULT_CHAT_SYSTEM_PROMPT: &str = "You are a highly capable research assistant with access to powerful search tools. IMPORTANT INSTRUCTIONS:1. When answering questions, you MUST make multiple tool calls (at least 2-3) to gather comprehensive information.2. Use different search queries for each tool call - vary keywords, rephrase questions, and explore different semantic angles to ensure broad coverage.3. Always explicitly announce BEFORE making each tool call by saying: \"I'll search for [specific information] now.\"4. Combine information from ALL tool calls to provide complete, nuanced answers rather than relying on a single source.5. For complex topics, break down your research into multiple targeted queries rather than using a single generic search.";
+pub const DEFAULT_CHAT_SEARCH_DESCRIPTION_PROMPT: &str =
+    "Search the database for relevant JSON documents using an optional query.";
+pub const DEFAULT_CHAT_SEARCH_Q_PARAM_PROMPT: &str = "The search query string used to find relevant documents in the index. This should contain keywords or phrases that best represent what the user is looking for. More specific queries will yield more precise results.";
+pub const DEFAULT_CHAT_SEARCH_INDEX_UID_PARAM_PROMPT: &str = "The name of the index to search within. An index is a collection of documents organized for search. Selecting the right index ensures the most relevant results for the user query.";
+
 #[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)]
 #[serde(rename_all = "camelCase", default)]
 pub struct RuntimeTogglableFeatures {
@@ -12,6 +20,8 @@ pub struct RuntimeTogglableFeatures {
     pub network: bool,
     pub get_task_documents_route: bool,
     pub composite_embedders: bool,
+    pub chat_completions: bool,
+    pub multimodal: bool,
 }
 
 #[derive(Default, Debug, Clone, Copy)]
@@ -37,3 +47,215 @@ pub struct Network {
     #[serde(default)]
     pub remotes: BTreeMap<String, Remote>,
 }
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
+#[serde(rename_all = "camelCase")]
+pub struct ChatCompletionSettings {
+    pub source: ChatCompletionSource,
+    #[serde(default)]
+    pub org_id: Option<String>,
+    #[serde(default)]
+    pub project_id: Option<String>,
+    #[serde(default)]
+    pub api_version: Option<String>,
+    #[serde(default)]
+    pub deployment_id: Option<String>,
+    #[serde(default)]
+    pub base_url: Option<String>,
+    #[serde(default)]
+    pub api_key: Option<String>,
+    #[serde(default)]
+    pub prompts: ChatCompletionPrompts,
+}
+
+impl ChatCompletionSettings {
+    pub fn hide_secrets(&mut self) {
+        if let Some(api_key) = &mut self.api_key {
+            Self::hide_secret(api_key);
+        }
+    }
+
+    fn hide_secret(secret: &mut String) {
+        match secret.len() {
+            x if x < 10 => {
+                secret.replace_range(.., "XXX...");
+            }
+            x if x < 20 => {
+                secret.replace_range(2.., "XXXX...");
+            }
+            x if x < 30 => {
+                secret.replace_range(3.., "XXXXX...");
+            }
+            _x => {
+                secret.replace_range(5.., "XXXXXX...");
+            }
+        }
+    }
+
+    pub fn validate(&self) -> Result<(), ResponseError> {
+        use ChatCompletionSource::*;
+        match self {
+            Self { source: AzureOpenAi, base_url, deployment_id, api_version, .. } if base_url.is_none() || deployment_id.is_none() || api_version.is_none() => Err(ResponseError::from_msg(
+                "azureOpenAi requires setting a valid `baseUrl`, `deploymentId`, and `apiVersion`".to_string(),
+                Code::BadRequest,
+            )),
+            Self { source: VLlm, base_url, .. } if base_url.is_none() => Err(ResponseError::from_msg(
+                "vLlm requires setting a valid `baseUrl`".to_string(),
+                Code::BadRequest,
+            )),
+            _otherwise => Ok(()),
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, Copy, PartialEq, Eq, Default)]
+#[serde(rename_all = "camelCase")]
+pub enum ChatCompletionSource {
+    #[default]
+    OpenAi,
+    AzureOpenAi,
+    Mistral,
+    VLlm,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum SystemRole {
+    System,
+    Developer,
+}
+
+impl ChatCompletionSource {
+    pub fn system_role(&self, model: &str) -> SystemRole {
+        use ChatCompletionSource::*;
+        use SystemRole::*;
+        match self {
+            OpenAi if Self::old_openai_model(model) => System,
+            OpenAi => Developer,
+            AzureOpenAi if Self::old_openai_model(model) => System,
+            AzureOpenAi => Developer,
+            Mistral => System,
+            VLlm => System,
+        }
+    }
+
+    /// Returns true if the model is an old OpenAI model.
+    ///
+    /// Old OpenAI models use the system role while new ones use the developer role.
+    fn old_openai_model(model: &str) -> bool {
+        ["gpt-3.5", "gpt-4", "gpt-4.1", "gpt-4.5", "gpt-4o", "chatgpt-4o"].iter().any(|old| {
+            model.starts_with(old)
+                && model.chars().nth(old.chars().count()).is_none_or(|last| last == '-')
+        })
+    }
+
+    pub fn base_url(&self) -> Option<&'static str> {
+        use ChatCompletionSource::*;
+        match self {
+            OpenAi => Some("https://api.openai.com/v1/"),
+            Mistral => Some("https://api.mistral.ai/v1/"),
+            AzureOpenAi | VLlm => None,
+        }
+    }
+}
+
+#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)]
+#[serde(rename_all = "camelCase")]
+pub struct ChatCompletionPrompts {
+    pub system: String,
+    pub search_description: String,
+    pub search_q_param: String,
+    pub search_index_uid_param: String,
+}
+
+impl Default for ChatCompletionPrompts {
+    fn default() -> Self {
+        Self {
+            system: DEFAULT_CHAT_SYSTEM_PROMPT.to_string(),
+            search_description: DEFAULT_CHAT_SEARCH_DESCRIPTION_PROMPT.to_string(),
+            search_q_param: DEFAULT_CHAT_SEARCH_Q_PARAM_PROMPT.to_string(),
+            search_index_uid_param: DEFAULT_CHAT_SEARCH_INDEX_UID_PARAM_PROMPT.to_string(),
+        }
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    const ALL_OPENAI_MODELS_OLDINESS: &[(&str, bool)] = &[
+        ("gpt-4-0613", true),
+        ("gpt-4", true),
+        ("gpt-3.5-turbo", true),
+        ("gpt-4o-audio-preview-2025-06-03", true),
+        ("gpt-4.1-nano", true),
+        ("gpt-4o-realtime-preview-2025-06-03", true),
+        ("gpt-3.5-turbo-instruct", true),
+        ("gpt-3.5-turbo-instruct-0914", true),
+        ("gpt-4-1106-preview", true),
+        ("gpt-3.5-turbo-1106", true),
+        ("gpt-4-0125-preview", true),
+        ("gpt-4-turbo-preview", true),
+        ("gpt-3.5-turbo-0125", true),
+        ("gpt-4-turbo", true),
+        ("gpt-4-turbo-2024-04-09", true),
+        ("gpt-4o", true),
+        ("gpt-4o-2024-05-13", true),
+        ("gpt-4o-mini-2024-07-18", true),
+        ("gpt-4o-mini", true),
+        ("gpt-4o-2024-08-06", true),
+        ("chatgpt-4o-latest", true),
+        ("gpt-4o-realtime-preview-2024-10-01", true),
+        ("gpt-4o-audio-preview-2024-10-01", true),
+        ("gpt-4o-audio-preview", true),
+        ("gpt-4o-realtime-preview", true),
+        ("gpt-4o-realtime-preview-2024-12-17", true),
+        ("gpt-4o-audio-preview-2024-12-17", true),
+        ("gpt-4o-mini-realtime-preview-2024-12-17", true),
+        ("gpt-4o-mini-audio-preview-2024-12-17", true),
+        ("gpt-4o-mini-realtime-preview", true),
+        ("gpt-4o-mini-audio-preview", true),
+        ("gpt-4o-2024-11-20", true),
+        ("gpt-4.5-preview", true),
+        ("gpt-4.5-preview-2025-02-27", true),
+        ("gpt-4o-search-preview-2025-03-11", true),
+        ("gpt-4o-search-preview", true),
+        ("gpt-4o-mini-search-preview-2025-03-11", true),
+        ("gpt-4o-mini-search-preview", true),
+        ("gpt-4o-transcribe", true),
+        ("gpt-4o-mini-transcribe", true),
+        ("gpt-4o-mini-tts", true),
+        ("gpt-4.1-2025-04-14", true),
+        ("gpt-4.1", true),
+        ("gpt-4.1-mini-2025-04-14", true),
+        ("gpt-4.1-mini", true),
+        ("gpt-4.1-nano-2025-04-14", true),
+        ("gpt-3.5-turbo-16k", true),
+        //
+        // new models
+        ("o1-preview-2024-09-12", false),
+        ("o1-preview", false),
+        ("o1-mini-2024-09-12", false),
+        ("o1-mini", false),
+        ("o1-2024-12-17", false),
+        ("o1", false),
+        ("o3-mini", false),
+        ("o3-mini-2025-01-31", false),
+        ("o1-pro-2025-03-19", false),
+        ("o1-pro", false),
+        ("o3-2025-04-16", false),
+        ("o4-mini-2025-04-16", false),
+        ("o3", false),
+        ("o4-mini", false),
+    ];
+
+    #[test]
+    fn old_openai_models() {
+        for (name, is_old) in ALL_OPENAI_MODELS_OLDINESS.iter().copied() {
+            assert_eq!(
+                ChatCompletionSource::old_openai_model(name),
+                is_old,
+                "Model {name} is not considered old"
+            );
+        }
+    }
+}

crates/meilisearch-types/src/index_uid_pattern.rs

@@ -12,7 +12,7 @@ use crate::index_uid::{IndexUid, IndexUidFormatError};
 
 /// An index uid pattern is composed of only ascii alphanumeric characters, - and _, between 1 and 400
 /// bytes long and optionally ending with a *.
-#[derive(Serialize, Deserialize, Deserr, Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Serialize, Deserialize, Deserr, Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)]
 #[deserr(try_from(&String) = FromStr::from_str -> IndexUidPatternFormatError)]
 pub struct IndexUidPattern(String);
 

crates/meilisearch-types/src/keys.rs

@@ -53,7 +53,7 @@ pub struct CreateApiKey {
     #[schema(example = json!(["documents.add"]))]
     #[deserr(error = DeserrJsonError<InvalidApiKeyActions>, missing_field_error = DeserrJsonError::missing_api_key_actions)]
     pub actions: Vec<Action>,
-    /// A list of accesible indexes permitted for the key. `["*"]` for all indexes. The `*` character can be used as a wildcard when located at the last position. e.g. `products_*` to allow access to all indexes whose names start with `products_`.
+    /// A list of accessible indexes permitted for the key. `["*"]` for all indexes. The `*` character can be used as a wildcard when located at the last position. e.g. `products_*` to allow access to all indexes whose names start with `products_`.
     #[deserr(error = DeserrJsonError<InvalidApiKeyIndexes>, missing_field_error = DeserrJsonError::missing_api_key_indexes)]
     #[schema(value_type = Vec<String>, example = json!(["products"]))]
     pub indexes: Vec<IndexUidPattern>,
@@ -158,6 +158,21 @@ impl Key {
             updated_at: now,
         }
     }
+
+    pub fn default_chat() -> Self {
+        let now = OffsetDateTime::now_utc();
+        let uid = Uuid::new_v4();
+        Self {
+            name: Some("Default Chat API Key".to_string()),
+            description: Some("Use it to chat and search from the frontend".to_string()),
+            uid,
+            actions: vec![Action::ChatCompletions, Action::Search],
+            indexes: vec![IndexUidPattern::all()],
+            expires_at: None,
+            created_at: now,
+            updated_at: now,
+        }
+    }
 }
 
 fn parse_expiration_date(
@@ -302,12 +317,36 @@ pub enum Action {
     #[serde(rename = "experimental.update")]
     #[deserr(rename = "experimental.update")]
     ExperimentalFeaturesUpdate,
+    #[serde(rename = "export")]
+    #[deserr(rename = "export")]
+    Export,
     #[serde(rename = "network.get")]
     #[deserr(rename = "network.get")]
     NetworkGet,
     #[serde(rename = "network.update")]
     #[deserr(rename = "network.update")]
     NetworkUpdate,
+    #[serde(rename = "chatCompletions")]
+    #[deserr(rename = "chatCompletions")]
+    ChatCompletions,
+    #[serde(rename = "chats.*")]
+    #[deserr(rename = "chats.*")]
+    ChatsAll,
+    #[serde(rename = "chats.get")]
+    #[deserr(rename = "chats.get")]
+    ChatsGet,
+    #[serde(rename = "chats.delete")]
+    #[deserr(rename = "chats.delete")]
+    ChatsDelete,
+    #[serde(rename = "chatsSettings.*")]
+    #[deserr(rename = "chatsSettings.*")]
+    ChatsSettingsAll,
+    #[serde(rename = "chatsSettings.get")]
+    #[deserr(rename = "chatsSettings.get")]
+    ChatsSettingsGet,
+    #[serde(rename = "chatsSettings.update")]
+    #[deserr(rename = "chatsSettings.update")]
+    ChatsSettingsUpdate,
 }
 
 impl Action {
@@ -333,6 +372,13 @@ impl Action {
             SETTINGS_ALL => Some(Self::SettingsAll),
             SETTINGS_GET => Some(Self::SettingsGet),
             SETTINGS_UPDATE => Some(Self::SettingsUpdate),
+            CHAT_COMPLETIONS => Some(Self::ChatCompletions),
+            CHATS_ALL => Some(Self::ChatsAll),
+            CHATS_GET => Some(Self::ChatsGet),
+            CHATS_DELETE => Some(Self::ChatsDelete),
+            CHATS_SETTINGS_ALL => Some(Self::ChatsSettingsAll),
+            CHATS_SETTINGS_GET => Some(Self::ChatsSettingsGet),
+            CHATS_SETTINGS_UPDATE => Some(Self::ChatsSettingsUpdate),
             STATS_ALL => Some(Self::StatsAll),
             STATS_GET => Some(Self::StatsGet),
             METRICS_ALL => Some(Self::MetricsAll),
@@ -395,6 +441,16 @@ pub mod actions {
     pub const EXPERIMENTAL_FEATURES_GET: u8 = ExperimentalFeaturesGet.repr();
     pub const EXPERIMENTAL_FEATURES_UPDATE: u8 = ExperimentalFeaturesUpdate.repr();
 
+    pub const EXPORT: u8 = Export.repr();
+
     pub const NETWORK_GET: u8 = NetworkGet.repr();
     pub const NETWORK_UPDATE: u8 = NetworkUpdate.repr();
+
+    pub const CHAT_COMPLETIONS: u8 = ChatCompletions.repr();
+    pub const CHATS_ALL: u8 = ChatsAll.repr();
+    pub const CHATS_GET: u8 = ChatsGet.repr();
+    pub const CHATS_DELETE: u8 = ChatsDelete.repr();
+    pub const CHATS_SETTINGS_ALL: u8 = ChatsSettingsAll.repr();
+    pub const CHATS_SETTINGS_GET: u8 = ChatsSettingsGet.repr();
+    pub const CHATS_SETTINGS_UPDATE: u8 = ChatsSettingsUpdate.repr();
 }

crates/meilisearch-types/src/lib.rs

@@ -18,7 +18,7 @@ pub mod versioning;
 pub use milli::{heed, Index};
 use uuid::Uuid;
 pub use versioning::VERSION_FILE_NAME;
-pub use {milli, serde_cs};
+pub use {byte_unit, milli, serde_cs};
 
 pub type Document = serde_json::Map<String, serde_json::Value>;
 pub type InstanceUid = Uuid;

crates/meilisearch-types/src/settings.rs

@@ -9,9 +9,11 @@ use std::str::FromStr;
 use deserr::{DeserializeError, Deserr, ErrorKind, MergeWithError, ValuePointerRef};
 use fst::IntoStreamer;
 use milli::disabled_typos_terms::DisabledTyposTerms;
-use milli::index::{IndexEmbeddingConfig, PrefixSearch};
+use milli::index::PrefixSearch;
 use milli::proximity::ProximityPrecision;
+pub use milli::update::ChatSettings;
 use milli::update::Setting;
+use milli::vector::db::IndexEmbeddingConfig;
 use milli::{Criterion, CriterionError, FilterableAttributesRule, Index, DEFAULT_VALUES_PER_FACET};
 use serde::{Deserialize, Serialize, Serializer};
 use utoipa::ToSchema;
@@ -185,7 +187,7 @@ impl<E: DeserializeError> Deserr<E> for SettingEmbeddingSettings {
 /// Holds all the settings for an index. `T` can either be `Checked` if they represents settings
 /// whose validity is guaranteed, or `Unchecked` if they need to be validated. In the later case, a
 /// call to `check` will return a `Settings<Checked>` from a `Settings<Unchecked>`.
-#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, Deserr, ToSchema)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Deserr, ToSchema)]
 #[serde(
     deny_unknown_fields,
     rename_all = "camelCase",
@@ -199,72 +201,86 @@ pub struct Settings<T> {
     #[deserr(default, error = DeserrJsonError<InvalidSettingsDisplayedAttributes>)]
     #[schema(value_type = Option<Vec<String>>, example = json!(["id", "title", "description", "url"]))]
     pub displayed_attributes: WildcardSetting,
+
     /// Fields in which to search for matching query words sorted by order of importance.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsSearchableAttributes>)]
     #[schema(value_type = Option<Vec<String>>, example = json!(["title", "description"]))]
     pub searchable_attributes: WildcardSetting,
+
     /// Attributes to use for faceting and filtering. See [Filtering and Faceted Search](https://www.meilisearch.com/docs/learn/filtering_and_sorting/search_with_facet_filters).
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsFilterableAttributes>)]
     #[schema(value_type = Option<Vec<FilterableAttributesRule>>, example = json!(["release_date", "genre"]))]
     pub filterable_attributes: Setting<Vec<FilterableAttributesRule>>,
+
     /// Attributes to use when sorting search results.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsSortableAttributes>)]
     #[schema(value_type = Option<Vec<String>>, example = json!(["release_date"]))]
     pub sortable_attributes: Setting<BTreeSet<String>>,
+
     /// List of ranking rules sorted by order of importance. The order is customizable.
     /// [A list of ordered built-in ranking rules](https://www.meilisearch.com/docs/learn/relevancy/relevancy).
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsRankingRules>)]
     #[schema(value_type = Option<Vec<String>>, example = json!([RankingRuleView::Words, RankingRuleView::Typo, RankingRuleView::Proximity, RankingRuleView::Attribute, RankingRuleView::Exactness]))]
     pub ranking_rules: Setting<Vec<RankingRuleView>>,
+
     /// List of words ignored when present in search queries.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsStopWords>)]
     #[schema(value_type = Option<Vec<String>>, example = json!(["the", "a", "them", "their"]))]
     pub stop_words: Setting<BTreeSet<String>>,
+
     /// List of characters not delimiting where one term begins and ends.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsNonSeparatorTokens>)]
     #[schema(value_type = Option<Vec<String>>, example = json!([" ", "\n"]))]
     pub non_separator_tokens: Setting<BTreeSet<String>>,
+
     /// List of characters delimiting where one term begins and ends.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsSeparatorTokens>)]
     #[schema(value_type = Option<Vec<String>>, example = json!(["S"]))]
     pub separator_tokens: Setting<BTreeSet<String>>,
+
     /// List of strings Meilisearch should parse as a single term.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsDictionary>)]
     #[schema(value_type = Option<Vec<String>>, example = json!(["iPhone pro"]))]
     pub dictionary: Setting<BTreeSet<String>>,
+
     /// List of associated words treated similarly. A word associated to an array of word as synonyms.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsSynonyms>)]
     #[schema(value_type = Option<BTreeMap<String, Vec<String>>>, example = json!({ "he": ["she", "they", "them"], "phone": ["iPhone", "android"]}))]
     pub synonyms: Setting<BTreeMap<String, Vec<String>>>,
+
     /// Search returns documents with distinct (different) values of the given field.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsDistinctAttribute>)]
     #[schema(value_type = Option<String>, example = json!("sku"))]
     pub distinct_attribute: Setting<String>,
+
     /// Precision level when calculating the proximity ranking rule.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsProximityPrecision>)]
     #[schema(value_type = Option<String>, example = json!(ProximityPrecisionView::ByAttribute))]
     pub proximity_precision: Setting<ProximityPrecisionView>,
+
     /// Customize typo tolerance feature.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsTypoTolerance>)]
     #[schema(value_type = Option<TypoSettings>, example = json!({ "enabled": true, "disableOnAttributes": ["title"]}))]
     pub typo_tolerance: Setting<TypoSettings>,
+
     /// Faceting settings.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsFaceting>)]
     #[schema(value_type = Option<FacetingSettings>, example = json!({ "maxValuesPerFacet": 10, "sortFacetValuesBy": { "genre": FacetValuesSort::Count }}))]
     pub faceting: Setting<FacetingSettings>,
+
     /// Pagination settings.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsPagination>)]
@@ -276,24 +292,34 @@ pub struct Settings<T> {
     #[deserr(default, error = DeserrJsonError<InvalidSettingsEmbedders>)]
     #[schema(value_type = Option<BTreeMap<String, SettingEmbeddingSettings>>)]
     pub embedders: Setting<BTreeMap<String, SettingEmbeddingSettings>>,
+
     /// Maximum duration of a search query.
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsSearchCutoffMs>)]
     #[schema(value_type = Option<u64>, example = json!(50))]
     pub search_cutoff_ms: Setting<u64>,
+
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsLocalizedAttributes>)]
     #[schema(value_type = Option<Vec<LocalizedAttributesRuleView>>, example = json!(50))]
     pub localized_attributes: Setting<Vec<LocalizedAttributesRuleView>>,
+
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsFacetSearch>)]
     #[schema(value_type = Option<bool>, example = json!(true))]
     pub facet_search: Setting<bool>,
+
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsPrefixSearch>)]
     #[schema(value_type = Option<PrefixSearchSettings>, example = json!("Hemlo"))]
     pub prefix_search: Setting<PrefixSearchSettings>,
 
+    /// Customize the chat prompting.
+    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
+    #[deserr(default, error = DeserrJsonError<InvalidSettingsIndexChat>)]
+    #[schema(value_type = Option<ChatSettings>)]
+    pub chat: Setting<ChatSettings>,
+
     #[serde(skip)]
     #[deserr(skip)]
     pub _kind: PhantomData<T>,
@@ -359,6 +385,7 @@ impl Settings<Checked> {
             localized_attributes: Setting::Reset,
             facet_search: Setting::Reset,
             prefix_search: Setting::Reset,
+            chat: Setting::Reset,
             _kind: PhantomData,
         }
     }
@@ -385,6 +412,7 @@ impl Settings<Checked> {
             localized_attributes: localized_attributes_rules,
             facet_search,
             prefix_search,
+            chat,
             _kind,
         } = self;
 
@@ -409,6 +437,7 @@ impl Settings<Checked> {
             localized_attributes: localized_attributes_rules,
             facet_search,
             prefix_search,
+            chat,
             _kind: PhantomData,
         }
     }
@@ -459,6 +488,7 @@ impl Settings<Unchecked> {
             localized_attributes: self.localized_attributes,
             facet_search: self.facet_search,
             prefix_search: self.prefix_search,
+            chat: self.chat,
             _kind: PhantomData,
         }
     }
@@ -471,8 +501,11 @@ impl Settings<Unchecked> {
         let Setting::Set(mut configs) = self.embedders else { return Ok(self) };
         for (name, config) in configs.iter_mut() {
             let config_to_check = std::mem::take(config);
-            let checked_config =
-                milli::update::validate_embedding_settings(config_to_check.inner, name)?;
+            let checked_config = milli::update::validate_embedding_settings(
+                config_to_check.inner,
+                name,
+                milli::vector::settings::EmbeddingValidationContext::SettingsPartialUpdate,
+            )?;
             *config = SettingEmbeddingSettings { inner: checked_config };
         }
         self.embedders = Setting::Set(configs);
@@ -533,8 +566,9 @@ impl Settings<Unchecked> {
                     Setting::Set(this)
                 }
             },
-            prefix_search: other.prefix_search.or(self.prefix_search),
             facet_search: other.facet_search.or(self.facet_search),
+            prefix_search: other.prefix_search.or(self.prefix_search),
+            chat: other.chat.clone().or(self.chat.clone()),
             _kind: PhantomData,
         }
     }
@@ -573,6 +607,7 @@ pub fn apply_settings_to_builder(
         localized_attributes: localized_attributes_rules,
         facet_search,
         prefix_search,
+        chat,
         _kind,
     } = settings;
 
@@ -666,7 +701,7 @@ pub fn apply_settings_to_builder(
     match typo_tolerance {
         Setting::Set(ref value) => {
             match value.enabled {
-                Setting::Set(val) => builder.set_autorize_typos(val),
+                Setting::Set(val) => builder.set_authorize_typos(val),
                 Setting::Reset => builder.reset_authorize_typos(),
                 Setting::NotSet => (),
             }
@@ -720,6 +755,7 @@ pub fn apply_settings_to_builder(
             builder.reset_min_word_len_two_typos();
             builder.reset_exact_words();
             builder.reset_exact_attributes();
+            builder.reset_disable_on_numbers();
         }
         Setting::NotSet => (),
     }
@@ -783,6 +819,12 @@ pub fn apply_settings_to_builder(
         Setting::Reset => builder.reset_facet_search(),
         Setting::NotSet => (),
     }
+
+    match chat {
+        Setting::Set(chat) => builder.set_chat(chat.clone()),
+        Setting::Reset => builder.reset_chat(),
+        Setting::NotSet => (),
+    }
 }
 
 pub enum SecretPolicy {
@@ -873,6 +915,7 @@ pub fn settings(
     };
 
     let embedders: BTreeMap<_, _> = index
+        .embedding_configs()
         .embedding_configs(rtxn)?
         .into_iter()
         .map(|IndexEmbeddingConfig { name, config, .. }| {
@@ -880,14 +923,11 @@ pub fn settings(
         })
         .collect();
     let embedders = Setting::Set(embedders);
-
     let search_cutoff_ms = index.search_cutoff(rtxn)?;
-
     let localized_attributes_rules = index.localized_attributes_rules(rtxn)?;
-
     let prefix_search = index.prefix_search(rtxn)?.map(PrefixSearchSettings::from);
-
     let facet_search = index.facet_search(rtxn)?;
+    let chat = index.chat_config(rtxn).map(ChatSettings::from)?;
 
     let mut settings = Settings {
         displayed_attributes: match displayed_attributes {
@@ -925,14 +965,16 @@ pub fn settings(
             Some(rules) => Setting::Set(rules.into_iter().map(|r| r.into()).collect()),
             None => Setting::Reset,
         },
-        prefix_search: Setting::Set(prefix_search.unwrap_or_default()),
         facet_search: Setting::Set(facet_search),
+        prefix_search: Setting::Set(prefix_search.unwrap_or_default()),
+        chat: Setting::Set(chat),
         _kind: PhantomData,
     };
 
     if let SecretPolicy::HideSecrets = secret_policy {
         settings.hide_secrets()
     }
+
     Ok(settings)
 }
 
@@ -1154,6 +1196,7 @@ pub(crate) mod test {
             search_cutoff_ms: Setting::NotSet,
             facet_search: Setting::NotSet,
             prefix_search: Setting::NotSet,
+            chat: Setting::NotSet,
             _kind: PhantomData::<Unchecked>,
         };
 
@@ -1185,6 +1228,8 @@ pub(crate) mod test {
             search_cutoff_ms: Setting::NotSet,
             facet_search: Setting::NotSet,
             prefix_search: Setting::NotSet,
+            chat: Setting::NotSet,
+
             _kind: PhantomData::<Unchecked>,
         };
 

crates/meilisearch-types/src/task_view.rs

@@ -1,3 +1,6 @@
+use std::collections::BTreeMap;
+
+use byte_unit::UnitType;
 use milli::Object;
 use serde::{Deserialize, Serialize};
 use time::{Duration, OffsetDateTime};
@@ -6,9 +9,11 @@ use utoipa::ToSchema;
 use crate::batches::BatchId;
 use crate::error::ResponseError;
 use crate::settings::{Settings, Unchecked};
-use crate::tasks::{serialize_duration, Details, IndexSwap, Kind, Status, Task, TaskId};
+use crate::tasks::{
+    serialize_duration, Details, DetailsExportIndexSettings, IndexSwap, Kind, Status, Task, TaskId,
+};
 
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, ToSchema)]
+#[derive(Debug, Clone, PartialEq, Serialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
 #[schema(rename_all = "camelCase")]
 pub struct TaskView {
@@ -67,7 +72,7 @@ impl TaskView {
     }
 }
 
-#[derive(Default, Debug, PartialEq, Eq, Clone, Serialize, Deserialize, ToSchema)]
+#[derive(Default, Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)]
 #[serde(rename_all = "camelCase")]
 #[schema(rename_all = "camelCase")]
 pub struct DetailsView {
@@ -118,6 +123,15 @@ pub struct DetailsView {
     pub upgrade_from: Option<String>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub upgrade_to: Option<String>,
+    // exporting
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub url: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub api_key: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub payload_size: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub indexes: Option<BTreeMap<String, DetailsExportIndexSettings>>,
 }
 
 impl DetailsView {
@@ -238,6 +252,34 @@ impl DetailsView {
                     Some(left)
                 }
             },
+            url: match (self.url.clone(), other.url.clone()) {
+                (None, None) => None,
+                (None, Some(url)) | (Some(url), None) => Some(url),
+                // We should never be able to batch multiple exports at the same time.
+                // So we return the first one we encounter but that shouldn't be an issue anyway.
+                (Some(left), Some(_right)) => Some(left),
+            },
+            api_key: match (self.api_key.clone(), other.api_key.clone()) {
+                (None, None) => None,
+                (None, Some(key)) | (Some(key), None) => Some(key),
+                // We should never be able to batch multiple exports at the same time.
+                // So we return the first one we encounter but that shouldn't be an issue anyway.
+                (Some(left), Some(_right)) => Some(left),
+            },
+            payload_size: match (self.payload_size.clone(), other.payload_size.clone()) {
+                (None, None) => None,
+                (None, Some(size)) | (Some(size), None) => Some(size),
+                // We should never be able to batch multiple exports at the same time.
+                // So we return the first one we encounter but that shouldn't be an issue anyway.
+                (Some(left), Some(_right)) => Some(left),
+            },
+            indexes: match (self.indexes.clone(), other.indexes.clone()) {
+                (None, None) => None,
+                (None, Some(indexes)) | (Some(indexes), None) => Some(indexes),
+                // We should never be able to batch multiple exports at the same time.
+                // So we return the first one we encounter but that shouldn't be an issue anyway.
+                (Some(left), Some(_right)) => Some(left),
+            },
             // We want the earliest version
             upgrade_from: match (self.upgrade_from.clone(), other.upgrade_from.clone()) {
                 (None, None) => None,
@@ -327,6 +369,22 @@ impl From<Details> for DetailsView {
             Details::IndexSwap { swaps } => {
                 DetailsView { swaps: Some(swaps), ..Default::default() }
             }
+            Details::Export { url, api_key, payload_size, indexes } => DetailsView {
+                url: Some(url),
+                api_key: api_key.map(|mut api_key| {
+                    hide_secret(&mut api_key);
+                    api_key
+                }),
+                payload_size: payload_size
+                    .map(|ps| ps.get_appropriate_unit(UnitType::Both).to_string()),
+                indexes: Some(
+                    indexes
+                        .into_iter()
+                        .map(|(pattern, settings)| (pattern.to_string(), settings))
+                        .collect(),
+                ),
+                ..Default::default()
+            },
             Details::UpgradeDatabase { from, to } => DetailsView {
                 upgrade_from: Some(format!("v{}.{}.{}", from.0, from.1, from.2)),
                 upgrade_to: Some(format!("v{}.{}.{}", to.0, to.1, to.2)),
@@ -335,3 +393,21 @@ impl From<Details> for DetailsView {
         }
     }
 }
+
+// We definitely need to factorize the code to hide the secret key
+fn hide_secret(secret: &mut String) {
+    match secret.len() {
+        x if x < 10 => {
+            secret.replace_range(.., "XXX...");
+        }
+        x if x < 20 => {
+            secret.replace_range(2.., "XXXX...");
+        }
+        x if x < 30 => {
+            secret.replace_range(3.., "XXXXX...");
+        }
+        _x => {
+            secret.replace_range(5.., "XXXXXX...");
+        }
+    }
+}

crates/meilisearch-types/src/tasks.rs

@@ -1,19 +1,22 @@
 use core::fmt;
-use std::collections::HashSet;
+use std::collections::{BTreeMap, HashSet};
 use std::fmt::{Display, Write};
 use std::str::FromStr;
 
+use byte_unit::Byte;
 use enum_iterator::Sequence;
 use milli::update::IndexDocumentsMethod;
 use milli::Object;
 use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize, Serializer};
+use serde_json::Value;
 use time::{Duration, OffsetDateTime};
-use utoipa::ToSchema;
+use utoipa::{schema, ToSchema};
 use uuid::Uuid;
 
 use crate::batches::BatchId;
 use crate::error::ResponseError;
+use crate::index_uid_pattern::IndexUidPattern;
 use crate::keys::Key;
 use crate::settings::{Settings, Unchecked};
 use crate::{versioning, InstanceUid};
@@ -50,6 +53,7 @@ impl Task {
             | SnapshotCreation
             | TaskCancelation { .. }
             | TaskDeletion { .. }
+            | Export { .. }
             | UpgradeDatabase { .. }
             | IndexSwap { .. } => None,
             DocumentAdditionOrUpdate { index_uid, .. }
@@ -86,6 +90,7 @@ impl Task {
             | KindWithContent::TaskDeletion { .. }
             | KindWithContent::DumpCreation { .. }
             | KindWithContent::SnapshotCreation
+            | KindWithContent::Export { .. }
             | KindWithContent::UpgradeDatabase { .. } => None,
         }
     }
@@ -108,11 +113,11 @@ pub enum KindWithContent {
     },
     DocumentDeletionByFilter {
         index_uid: String,
-        filter_expr: serde_json::Value,
+        filter_expr: Value,
     },
     DocumentEdition {
         index_uid: String,
-        filter_expr: Option<serde_json::Value>,
+        filter_expr: Option<Value>,
         context: Option<milli::Object>,
         function: String,
     },
@@ -152,6 +157,12 @@ pub enum KindWithContent {
         instance_uid: Option<InstanceUid>,
     },
     SnapshotCreation,
+    Export {
+        url: String,
+        api_key: Option<String>,
+        payload_size: Option<Byte>,
+        indexes: BTreeMap<IndexUidPattern, ExportIndexSettings>,
+    },
     UpgradeDatabase {
         from: (u32, u32, u32),
     },
@@ -163,6 +174,13 @@ pub struct IndexSwap {
     pub indexes: (String, String),
 }
 
+#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]
+#[serde(rename_all = "camelCase")]
+pub struct ExportIndexSettings {
+    pub filter: Option<Value>,
+    pub override_settings: bool,
+}
+
 impl KindWithContent {
     pub fn as_kind(&self) -> Kind {
         match self {
@@ -180,6 +198,7 @@ impl KindWithContent {
             KindWithContent::TaskDeletion { .. } => Kind::TaskDeletion,
             KindWithContent::DumpCreation { .. } => Kind::DumpCreation,
             KindWithContent::SnapshotCreation => Kind::SnapshotCreation,
+            KindWithContent::Export { .. } => Kind::Export,
             KindWithContent::UpgradeDatabase { .. } => Kind::UpgradeDatabase,
         }
     }
@@ -192,6 +211,7 @@ impl KindWithContent {
             | SnapshotCreation
             | TaskCancelation { .. }
             | TaskDeletion { .. }
+            | Export { .. }
             | UpgradeDatabase { .. } => vec![],
             DocumentAdditionOrUpdate { index_uid, .. }
             | DocumentEdition { index_uid, .. }
@@ -269,6 +289,14 @@ impl KindWithContent {
             }),
             KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }),
             KindWithContent::SnapshotCreation => None,
+            KindWithContent::Export { url, api_key, payload_size, indexes } => {
+                Some(Details::Export {
+                    url: url.clone(),
+                    api_key: api_key.clone(),
+                    payload_size: *payload_size,
+                    indexes: indexes.iter().map(|(p, s)| (p.clone(), s.clone().into())).collect(),
+                })
+            }
             KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase {
                 from: (from.0, from.1, from.2),
                 to: (
@@ -335,6 +363,14 @@ impl KindWithContent {
             }),
             KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }),
             KindWithContent::SnapshotCreation => None,
+            KindWithContent::Export { url, api_key, payload_size, indexes } => {
+                Some(Details::Export {
+                    url: url.clone(),
+                    api_key: api_key.clone(),
+                    payload_size: *payload_size,
+                    indexes: indexes.iter().map(|(p, s)| (p.clone(), s.clone().into())).collect(),
+                })
+            }
             KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase {
                 from: *from,
                 to: (
@@ -383,6 +419,14 @@ impl From<&KindWithContent> for Option<Details> {
             }),
             KindWithContent::DumpCreation { .. } => Some(Details::Dump { dump_uid: None }),
             KindWithContent::SnapshotCreation => None,
+            KindWithContent::Export { url, api_key, payload_size, indexes } => {
+                Some(Details::Export {
+                    url: url.clone(),
+                    api_key: api_key.clone(),
+                    payload_size: *payload_size,
+                    indexes: indexes.iter().map(|(p, s)| (p.clone(), s.clone().into())).collect(),
+                })
+            }
             KindWithContent::UpgradeDatabase { from } => Some(Details::UpgradeDatabase {
                 from: *from,
                 to: (
@@ -499,6 +543,7 @@ pub enum Kind {
     TaskDeletion,
     DumpCreation,
     SnapshotCreation,
+    Export,
     UpgradeDatabase,
 }
 
@@ -516,6 +561,7 @@ impl Kind {
             | Kind::TaskCancelation
             | Kind::TaskDeletion
             | Kind::DumpCreation
+            | Kind::Export
             | Kind::UpgradeDatabase
             | Kind::SnapshotCreation => false,
         }
@@ -536,6 +582,7 @@ impl Display for Kind {
             Kind::TaskDeletion => write!(f, "taskDeletion"),
             Kind::DumpCreation => write!(f, "dumpCreation"),
             Kind::SnapshotCreation => write!(f, "snapshotCreation"),
+            Kind::Export => write!(f, "export"),
             Kind::UpgradeDatabase => write!(f, "upgradeDatabase"),
         }
     }
@@ -568,6 +615,8 @@ impl FromStr for Kind {
             Ok(Kind::DumpCreation)
         } else if kind.eq_ignore_ascii_case("snapshotCreation") {
             Ok(Kind::SnapshotCreation)
+        } else if kind.eq_ignore_ascii_case("export") {
+            Ok(Kind::Export)
         } else if kind.eq_ignore_ascii_case("upgradeDatabase") {
             Ok(Kind::UpgradeDatabase)
         } else {
@@ -597,7 +646,7 @@ impl fmt::Display for ParseTaskKindError {
 }
 impl std::error::Error for ParseTaskKindError {}
 
-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
 pub enum Details {
     DocumentAdditionOrUpdate {
         received_documents: u64,
@@ -643,12 +692,33 @@ pub enum Details {
     IndexSwap {
         swaps: Vec<IndexSwap>,
     },
+    Export {
+        url: String,
+        api_key: Option<String>,
+        payload_size: Option<Byte>,
+        indexes: BTreeMap<IndexUidPattern, DetailsExportIndexSettings>,
+    },
     UpgradeDatabase {
         from: (u32, u32, u32),
         to: (u32, u32, u32),
     },
 }
 
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize, ToSchema)]
+#[schema(rename_all = "camelCase")]
+pub struct DetailsExportIndexSettings {
+    #[serde(flatten)]
+    pub settings: ExportIndexSettings,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub matched_documents: Option<u64>,
+}
+
+impl From<ExportIndexSettings> for DetailsExportIndexSettings {
+    fn from(settings: ExportIndexSettings) -> Self {
+        DetailsExportIndexSettings { settings, matched_documents: None }
+    }
+}
+
 impl Details {
     pub fn to_failed(&self) -> Self {
         let mut details = self.clone();
@@ -667,6 +737,7 @@ impl Details {
             Self::SettingsUpdate { .. }
             | Self::IndexInfo { .. }
             | Self::Dump { .. }
+            | Self::Export { .. }
             | Self::UpgradeDatabase { .. }
             | Self::IndexSwap { .. } => (),
         }

crates/meilisearch/Cargo.toml

@@ -13,50 +13,50 @@ license.workspace = true
 default-run = "meilisearch"
 
 [dependencies]
-actix-cors = "0.7.0"
-actix-http = { version = "3.9.0", default-features = false, features = [
+actix-cors = "0.7.1"
+actix-http = { version = "3.11.0", default-features = false, features = [
     "compress-brotli",
     "compress-gzip",
     "rustls-0_23",
 ] }
 actix-utils = "3.0.1"
-actix-web = { version = "4.9.0", default-features = false, features = [
+actix-web = { version = "4.11.0", default-features = false, features = [
     "macros",
     "compress-brotli",
     "compress-gzip",
     "cookies",
     "rustls-0_23",
 ] }
-anyhow = { version = "1.0.95", features = ["backtrace"] }
-async-trait = "0.1.85"
-bstr = "1.11.3"
+anyhow = { version = "1.0.98", features = ["backtrace"] }
+bstr = "1.12.0"
 byte-unit = { version = "5.1.6", features = ["serde"] }
-bytes = "1.9.0"
-clap = { version = "4.5.24", features = ["derive", "env"] }
+bytes = "1.10.1"
+bumpalo = "3.18.1"
+clap = { version = "4.5.40", features = ["derive", "env"] }
 crossbeam-channel = "0.5.15"
 deserr = { version = "0.6.3", features = ["actix-web"] }
 dump = { path = "../dump" }
-either = "1.13.0"
+either = "1.15.0"
 file-store = { path = "../file-store" }
-flate2 = "1.0.35"
+flate2 = "1.1.2"
 fst = "0.4.7"
 futures = "0.3.31"
 futures-util = "0.3.31"
 index-scheduler = { path = "../index-scheduler" }
-indexmap = { version = "2.7.0", features = ["serde"] }
-is-terminal = "0.4.13"
+indexmap = { version = "2.9.0", features = ["serde"] }
+is-terminal = "0.4.16"
 itertools = "0.14.0"
-jsonwebtoken = "9.3.0"
+jsonwebtoken = "9.3.1"
 lazy_static = "1.5.0"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-mimalloc = { version = "0.1.43", default-features = false }
+mimalloc = { version = "0.1.47", default-features = false }
 mime = "0.3.17"
-num_cpus = "1.16.0"
+num_cpus = "1.17.0"
 obkv = "0.3.0"
-once_cell = "1.20.2"
-ordered-float = "4.6.0"
-parking_lot = "0.12.3"
+once_cell = "1.21.3"
+ordered-float = "5.0.0"
+parking_lot = "0.12.4"
 permissive-json-pointer = { path = "../permissive-json-pointer" }
 pin-project-lite = "0.2.16"
 platform-dirs = "0.3.0"
@@ -64,44 +64,44 @@ prometheus = { version = "0.14.0", features = ["process"] }
 rand = "0.8.5"
 rayon = "1.10.0"
 regex = "1.11.1"
-reqwest = { version = "0.12.12", features = [
+reqwest = { version = "0.12.20", features = [
     "rustls-tls",
     "json",
 ], default-features = false }
-rustls = { version = "0.23.20", features = ["ring"], default-features = false }
-rustls-pki-types = { version = "1.10.1", features = ["alloc"] }
+rustls = { version = "0.23.28", features = ["ring"], default-features = false }
+rustls-pki-types = { version = "1.12.0", features = ["alloc"] }
 rustls-pemfile = "2.2.0"
-segment = { version = "0.2.5" }
-serde = { version = "1.0.217", features = ["derive"] }
-serde_json = { version = "1.0.135", features = ["preserve_order"] }
-sha2 = "0.10.8"
+segment = { version = "0.2.6" }
+serde = { version = "1.0.219", features = ["derive"] }
+serde_json = { version = "1.0.140", features = ["preserve_order"] }
+sha2 = "0.10.9"
 siphasher = "1.0.1"
 slice-group-by = "0.3.1"
-static-files = { version = "0.2.4", optional = true }
-sysinfo = "0.33.1"
-tar = "0.4.43"
-tempfile = "3.15.0"
-thiserror = "2.0.9"
-time = { version = "0.3.37", features = [
+static-files = { version = "0.2.5", optional = true }
+sysinfo = "0.35.2"
+tar = "0.4.44"
+tempfile = "3.20.0"
+thiserror = "2.0.12"
+time = { version = "0.3.41", features = [
     "serde-well-known",
     "formatting",
     "parsing",
     "macros",
 ] }
-tokio = { version = "1.43.1", features = ["full"] }
-toml = "0.8.19"
-uuid = { version = "1.11.0", features = ["serde", "v4"] }
+tokio = { version = "1.45.1", features = ["full"] }
+toml = "0.8.23"
+uuid = { version = "1.17.0", features = ["serde", "v4"] }
 serde_urlencoded = "0.7.1"
 termcolor = "1.4.1"
 url = { version = "2.5.4", features = ["serde"] }
 tracing = "0.1.41"
 tracing-subscriber = { version = "0.3.19", features = ["json"] }
 tracing-trace = { version = "0.1.0", path = "../tracing-trace" }
-tracing-actix-web = "0.7.15"
+tracing-actix-web = "0.7.18"
 build-info = { version = "1.7.0", path = "../build-info" }
-roaring = "0.10.10"
+roaring = "0.10.12"
 mopa-maintained = "0.2.3"
-utoipa = { version = "5.3.1", features = [
+utoipa = { version = "5.4.0", features = [
     "actix_extras",
     "macros",
     "non_strict_integers",
@@ -111,32 +111,35 @@ utoipa = { version = "5.3.1", features = [
     "openapi_extensions",
 ] }
 utoipa-scalar = { version = "0.3.0", optional = true, features = ["actix-web"] }
+async-openai = { git = "https://github.com/meilisearch/async-openai", branch = "better-error-handling" }
+secrecy = "0.10.3"
+actix-web-lab = { version = "0.24.1", default-features = false }
 
 [dev-dependencies]
 actix-rt = "2.10.0"
-brotli = "6.0.0"
+brotli = "8.0.1"
 # fixed version due to format breakages in v1.40
-insta = "=1.39.0"
+insta = { version = "=1.39.0", features = ["redactions"] }
 manifest-dir-macros = "0.1.18"
 maplit = "1.0.2"
 meili-snap = { path = "../meili-snap" }
 temp-env = "0.3.6"
 urlencoding = "2.1.3"
-wiremock = "0.6.2"
+wiremock = "0.6.3"
 yaup = "0.3.1"
 
 [build-dependencies]
-anyhow = { version = "1.0.95", optional = true }
-cargo_toml = { version = "0.21.0", optional = true }
+anyhow = { version = "1.0.98", optional = true }
+cargo_toml = { version = "0.22.1", optional = true }
 hex = { version = "0.4.3", optional = true }
-reqwest = { version = "0.12.12", features = [
+reqwest = { version = "0.12.20", features = [
     "blocking",
     "rustls-tls",
 ], default-features = false, optional = true }
 sha-1 = { version = "0.10.1", optional = true }
-static-files = { version = "0.2.4", optional = true }
-tempfile = { version = "3.15.0", optional = true }
-zip = { version = "2.3.0", optional = true }
+static-files = { version = "0.2.5", optional = true }
+tempfile = { version = "3.20.0", optional = true }
+zip = { version = "4.1.0", optional = true }
 
 [features]
 default = ["meilisearch-types/all-tokenizations", "mini-dashboard"]
@@ -166,5 +169,5 @@ german = ["meilisearch-types/german"]
 turkish = ["meilisearch-types/turkish"]
 
 [package.metadata.mini-dashboard]
-assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.19/build.zip"
-sha1 = "7974430d5277c97f67cf6e95eec6faaac2788834"
+assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.20/build.zip"
+sha1 = "82a7ddd7bf14bb5323c3d235d2b62892a98b6a59"

crates/meilisearch/src/analytics/segment_analytics.rs

@@ -197,10 +197,13 @@ struct Infos {
     experimental_max_number_of_batched_tasks: usize,
     experimental_limit_batched_tasks_total_size: u64,
     experimental_network: bool,
+    experimental_multimodal: bool,
+    experimental_chat_completions: bool,
     experimental_get_task_documents_route: bool,
     experimental_composite_embedders: bool,
     experimental_embedding_cache_entries: usize,
     experimental_no_snapshot_compaction: bool,
+    experimental_no_edition_2024_for_settings: bool,
     gpu_enabled: bool,
     db_path: bool,
     import_dump: bool,
@@ -285,8 +288,12 @@ impl Infos {
             ScheduleSnapshot::Enabled(interval) => Some(interval),
         };
 
-        let IndexerOpts { max_indexing_memory, max_indexing_threads, skip_index_budget: _ } =
-            indexer_options;
+        let IndexerOpts {
+            max_indexing_memory,
+            max_indexing_threads,
+            skip_index_budget: _,
+            experimental_no_edition_2024_for_settings,
+        } = indexer_options;
 
         let RuntimeTogglableFeatures {
             metrics,
@@ -296,6 +303,8 @@ impl Infos {
             network,
             get_task_documents_route,
             composite_embedders,
+            chat_completions,
+            multimodal,
         } = features;
 
         // We're going to override every sensible information.
@@ -314,6 +323,8 @@ impl Infos {
             experimental_enable_logs_route: experimental_enable_logs_route | logs_route,
             experimental_reduce_indexing_memory_usage,
             experimental_network: network,
+            experimental_chat_completions: chat_completions,
+            experimental_multimodal: multimodal,
             experimental_get_task_documents_route: get_task_documents_route,
             experimental_composite_embedders: composite_embedders,
             experimental_embedding_cache_entries,
@@ -347,6 +358,7 @@ impl Infos {
             ssl_require_auth,
             ssl_resumption,
             ssl_tickets,
+            experimental_no_edition_2024_for_settings,
         }
     }
 }

crates/meilisearch/src/error.rs

@@ -76,8 +76,10 @@ pub enum MeilisearchHttpError {
     DocumentFormat(#[from] DocumentFormatError),
     #[error(transparent)]
     Join(#[from] JoinError),
-    #[error("Invalid request: missing `hybrid` parameter when `vector` is present.")]
+    #[error("Invalid request: missing `hybrid` parameter when `vector` or `media` are present.")]
     MissingSearchHybrid,
+    #[error("Invalid request: both `media` and `vector` parameters are present.")]
+    MediaAndVector,
 }
 
 impl MeilisearchHttpError {
@@ -111,6 +113,7 @@ impl ErrorCode for MeilisearchHttpError {
             MeilisearchHttpError::DocumentFormat(e) => e.error_code(),
             MeilisearchHttpError::Join(_) => Code::Internal,
             MeilisearchHttpError::MissingSearchHybrid => Code::MissingSearchHybrid,
+            MeilisearchHttpError::MediaAndVector => Code::InvalidSearchMediaAndVector,
             MeilisearchHttpError::FederationOptionsInNonFederatedRequest(_) => {
                 Code::InvalidMultiSearchFederationOptions
             }

crates/meilisearch/src/extractors/authentication/mod.rs

@@ -4,6 +4,7 @@ use std::marker::PhantomData;
 use std::ops::Deref;
 use std::pin::Pin;
 
+use actix_web::http::header::AUTHORIZATION;
 use actix_web::web::Data;
 use actix_web::FromRequest;
 pub use error::AuthenticationError;
@@ -94,36 +95,44 @@ impl<P: Policy + 'static, D: 'static + Clone> FromRequest for GuardedData<P, D>
         _payload: &mut actix_web::dev::Payload,
     ) -> Self::Future {
         match req.app_data::<Data<AuthController>>().cloned() {
-            Some(auth) => match req
-                .headers()
-                .get("Authorization")
-                .map(|type_token| type_token.to_str().unwrap_or_default().splitn(2, ' '))
-            {
-                Some(mut type_token) => match type_token.next() {
-                    Some("Bearer") => {
-                        // TODO: find a less hardcoded way?
-                        let index = req.match_info().get("index_uid");
-                        match type_token.next() {
-                            Some(token) => Box::pin(Self::auth_bearer(
-                                auth,
-                                token.to_string(),
-                                index.map(String::from),
-                                req.app_data::<D>().cloned(),
-                            )),
-                            None => Box::pin(err(AuthenticationError::InvalidToken.into())),
-                        }
-                    }
-                    _otherwise => {
-                        Box::pin(err(AuthenticationError::MissingAuthorizationHeader.into()))
-                    }
-                },
-                None => Box::pin(Self::auth_token(auth, req.app_data::<D>().cloned())),
+            Some(auth) => match extract_token_from_request(req) {
+                Ok(Some(token)) => {
+                    // TODO: find a less hardcoded way?
+                    let index = req.match_info().get("index_uid");
+                    Box::pin(Self::auth_bearer(
+                        auth,
+                        token.to_string(),
+                        index.map(String::from),
+                        req.app_data::<D>().cloned(),
+                    ))
+                }
+                Ok(None) => Box::pin(Self::auth_token(auth, req.app_data::<D>().cloned())),
+                Err(e) => Box::pin(err(e.into())),
             },
             None => Box::pin(err(AuthenticationError::IrretrievableState.into())),
         }
     }
 }
 
+pub fn extract_token_from_request(
+    req: &actix_web::HttpRequest,
+) -> Result<Option<&str>, AuthenticationError> {
+    match req
+        .headers()
+        .get(AUTHORIZATION)
+        .map(|type_token| type_token.to_str().unwrap_or_default().splitn(2, ' '))
+    {
+        Some(mut type_token) => match type_token.next() {
+            Some("Bearer") => match type_token.next() {
+                Some(token) => Ok(Some(token)),
+                None => Err(AuthenticationError::InvalidToken),
+            },
+            _otherwise => Err(AuthenticationError::MissingAuthorizationHeader),
+        },
+        None => Ok(None),
+    }
+}
+
 pub trait Policy {
     fn authenticate(
         auth: Data<AuthController>,
@@ -299,8 +308,8 @@ pub mod policies {
             auth: &AuthController,
             token: &str,
         ) -> Result<TenantTokenOutcome, AuthError> {
-            // Only search action can be accessed by a tenant token.
-            if A != actions::SEARCH {
+            // Only search and chat actions can be accessed by a tenant token.
+            if A != actions::SEARCH && A != actions::CHAT_COMPLETIONS {
                 return Ok(TenantTokenOutcome::NotATenantToken);
             }
 

crates/meilisearch/src/lib.rs

@@ -37,7 +37,10 @@ use index_scheduler::{IndexScheduler, IndexSchedulerOptions};
 use meilisearch_auth::{open_auth_store_env, AuthController};
 use meilisearch_types::milli::constants::VERSION_MAJOR;
 use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
-use meilisearch_types::milli::update::{IndexDocumentsConfig, IndexDocumentsMethod};
+use meilisearch_types::milli::progress::{EmbedderStats, Progress};
+use meilisearch_types::milli::update::{
+    default_thread_pool_and_threads, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig,
+};
 use meilisearch_types::settings::apply_settings_to_builder;
 use meilisearch_types::tasks::KindWithContent;
 use meilisearch_types::versioning::{
@@ -461,6 +464,7 @@ fn import_dump(
     index_scheduler: &mut IndexScheduler,
     auth: &mut AuthController,
 ) -> Result<(), anyhow::Error> {
+    let progress = Progress::default();
     let reader = File::open(dump_path)?;
     let mut dump_reader = dump::DumpReader::open(reader)?;
 
@@ -494,19 +498,37 @@ fn import_dump(
         keys.push(key);
     }
 
-    // 3. Import the runtime features and network
+    // 3. Import the `ChatCompletionSettings`s.
+    for result in dump_reader.chat_completions_settings()? {
+        let (name, settings) = result?;
+        index_scheduler.put_chat_settings(&name, &settings)?;
+    }
+
+    // 4. Import the runtime features and network
     let features = dump_reader.features()?.unwrap_or_default();
     index_scheduler.put_runtime_features(features)?;
 
     let network = dump_reader.network()?.cloned().unwrap_or_default();
     index_scheduler.put_network(network)?;
 
-    let indexer_config = index_scheduler.indexer_config();
+    // 4.1 Use all cpus to process dump if `max_indexing_threads` not configured
+    let backup_config;
+    let base_config = index_scheduler.indexer_config();
+
+    let indexer_config = if base_config.max_threads.is_none() {
+        let (thread_pool, _) = default_thread_pool_and_threads();
+
+        let _config = IndexerConfig { thread_pool, ..*base_config };
+        backup_config = _config;
+        &backup_config
+    } else {
+        base_config
+    };
 
     // /!\ The tasks must be imported AFTER importing the indexes or else the scheduler might
     // try to process tasks while we're trying to import the indexes.
 
-    // 4. Import the indexes.
+    // 5. Import the indexes.
     for index_reader in dump_reader.indexes()? {
         let mut index_reader = index_reader?;
         let metadata = index_reader.metadata();
@@ -519,20 +541,20 @@ fn import_dump(
         let mut wtxn = index.write_txn()?;
 
         let mut builder = milli::update::Settings::new(&mut wtxn, &index, indexer_config);
-        // 4.1 Import the primary key if there is one.
+        // 5.1 Import the primary key if there is one.
         if let Some(ref primary_key) = metadata.primary_key {
             builder.set_primary_key(primary_key.to_string());
         }
 
-        // 4.2 Import the settings.
+        // 5.2 Import the settings.
         tracing::info!("Importing the settings.");
         let settings = index_reader.settings()?;
         apply_settings_to_builder(&settings, &mut builder);
-        builder
-            .execute(|indexing_step| tracing::debug!("update: {:?}", indexing_step), || false)?;
+        let embedder_stats: Arc<EmbedderStats> = Default::default();
+        builder.execute(&|| false, &progress, embedder_stats.clone())?;
 
-        // 4.3 Import the documents.
-        // 4.3.1 We need to recreate the grenad+obkv format accepted by the index.
+        // 5.3 Import the documents.
+        // 5.3.1 We need to recreate the grenad+obkv format accepted by the index.
         tracing::info!("Importing the documents.");
         let file = tempfile::tempfile()?;
         let mut builder = DocumentsBatchBuilder::new(BufWriter::new(file));
@@ -543,11 +565,11 @@ fn import_dump(
         // This flush the content of the batch builder.
         let file = builder.into_inner()?.into_inner()?;
 
-        // 4.3.2 We feed it to the milli index.
+        // 5.3.2 We feed it to the milli index.
         let reader = BufReader::new(file);
         let reader = DocumentsBatchReader::from_reader(reader)?;
 
-        let embedder_configs = index.embedding_configs(&wtxn)?;
+        let embedder_configs = index.embedding_configs().embedding_configs(&wtxn)?;
         let embedders = index_scheduler.embedders(uid.to_string(), embedder_configs)?;
 
         let builder = milli::update::IndexDocuments::new(
@@ -560,6 +582,7 @@ fn import_dump(
             },
             |indexing_step| tracing::trace!("update: {:?}", indexing_step),
             || false,
+            &embedder_stats,
         )?;
 
         let builder = builder.with_embedders(embedders);
@@ -574,15 +597,15 @@ fn import_dump(
         index_scheduler.refresh_index_stats(&uid)?;
     }
 
-    // 5. Import the queue
+    // 6. Import the queue
     let mut index_scheduler_dump = index_scheduler.register_dumped_task()?;
-    // 5.1. Import the batches
+    // 6.1. Import the batches
     for ret in dump_reader.batches()? {
         let batch = ret?;
         index_scheduler_dump.register_dumped_batch(batch)?;
     }
 
-    // 5.2. Import the tasks
+    // 6.2. Import the tasks
     for ret in dump_reader.tasks()? {
         let (task, file) = ret?;
         index_scheduler_dump.register_dumped_task(task, file)?;

crates/meilisearch/src/metrics.rs

@@ -15,6 +15,33 @@ lazy_static! {
         "Meilisearch number of degraded search requests"
     ))
     .expect("Can't create a metric");
+    pub static ref MEILISEARCH_CHAT_SEARCH_REQUESTS: IntCounterVec = register_int_counter_vec!(
+        opts!(
+            "meilisearch_chat_search_requests",
+            "Meilisearch number of search requests performed by the chat route itself"
+        ),
+        &["type"]
+    )
+    .expect("Can't create a metric");
+    pub static ref MEILISEARCH_CHAT_PROMPT_TOKENS_USAGE: IntCounterVec = register_int_counter_vec!(
+        opts!("meilisearch_chat_prompt_tokens_usage", "Meilisearch Chat Prompt Tokens Usage"),
+        &["workspace", "model"]
+    )
+    .expect("Can't create a metric");
+    pub static ref MEILISEARCH_CHAT_COMPLETION_TOKENS_USAGE: IntCounterVec =
+        register_int_counter_vec!(
+            opts!(
+                "meilisearch_chat_completion_tokens_usage",
+                "Meilisearch Chat Completion Tokens Usage"
+            ),
+            &["workspace", "model"]
+        )
+        .expect("Can't create a metric");
+    pub static ref MEILISEARCH_CHAT_TOTAL_TOKENS_USAGE: IntCounterVec = register_int_counter_vec!(
+        opts!("meilisearch_chat_total_tokens_usage", "Meilisearch Chat Total Tokens Usage"),
+        &["workspace", "model"]
+    )
+    .expect("Can't create a metric");
     pub static ref MEILISEARCH_DB_SIZE_BYTES: IntGauge =
         register_int_gauge!(opts!("meilisearch_db_size_bytes", "Meilisearch DB Size In Bytes"))
             .expect("Can't create a metric");

crates/meilisearch/src/option.rs

@@ -53,6 +53,8 @@ const MEILI_EXPERIMENTAL_DUMPLESS_UPGRADE: &str = "MEILI_EXPERIMENTAL_DUMPLESS_U
 const MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS: &str = "MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS";
 const MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE: &str = "MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE";
 const MEILI_EXPERIMENTAL_CONTAINS_FILTER: &str = "MEILI_EXPERIMENTAL_CONTAINS_FILTER";
+const MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_SETTINGS: &str =
+    "MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_SETTINGS";
 const MEILI_EXPERIMENTAL_ENABLE_METRICS: &str = "MEILI_EXPERIMENTAL_ENABLE_METRICS";
 const MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE: &str = "MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE";
 const MEILI_EXPERIMENTAL_DROP_SEARCH_AFTER: &str = "MEILI_EXPERIMENTAL_DROP_SEARCH_AFTER";
@@ -62,7 +64,7 @@ const MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: &str =
 const MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS: &str =
     "MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS";
 const MEILI_EXPERIMENTAL_LIMIT_BATCHED_TASKS_TOTAL_SIZE: &str =
-    "MEILI_EXPERIMENTAL_LIMIT_BATCHED_TASKS_SIZE";
+    "MEILI_EXPERIMENTAL_LIMIT_BATCHED_TASKS_TOTAL_SIZE";
 const MEILI_EXPERIMENTAL_EMBEDDING_CACHE_ENTRIES: &str =
     "MEILI_EXPERIMENTAL_EMBEDDING_CACHE_ENTRIES";
 const MEILI_EXPERIMENTAL_NO_SNAPSHOT_COMPACTION: &str = "MEILI_EXPERIMENTAL_NO_SNAPSHOT_COMPACTION";
@@ -749,22 +751,43 @@ pub struct IndexerOpts {
     #[clap(skip)]
     #[serde(skip)]
     pub skip_index_budget: bool,
+
+    /// Experimental no edition 2024 for settings feature. For more information,
+    /// see: <https://github.com/orgs/meilisearch/discussions/847>
+    ///
+    /// Enables the experimental no edition 2024 for settings feature.
+    #[clap(long, env = MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_SETTINGS)]
+    #[serde(default)]
+    pub experimental_no_edition_2024_for_settings: bool,
 }
 
 impl IndexerOpts {
     /// Exports the values to their corresponding env vars if they are not set.
     pub fn export_to_env(self) {
-        let IndexerOpts { max_indexing_memory, max_indexing_threads, skip_index_budget: _ } = self;
+        let IndexerOpts {
+            max_indexing_memory,
+            max_indexing_threads,
+            skip_index_budget: _,
+            experimental_no_edition_2024_for_settings,
+        } = self;
         if let Some(max_indexing_memory) = max_indexing_memory.0 {
             export_to_env_if_not_present(
                 MEILI_MAX_INDEXING_MEMORY,
                 max_indexing_memory.to_string(),
             );
         }
-        export_to_env_if_not_present(
-            MEILI_MAX_INDEXING_THREADS,
-            max_indexing_threads.0.to_string(),
-        );
+        if let Some(max_indexing_threads) = max_indexing_threads.0 {
+            export_to_env_if_not_present(
+                MEILI_MAX_INDEXING_THREADS,
+                max_indexing_threads.to_string(),
+            );
+        }
+        if experimental_no_edition_2024_for_settings {
+            export_to_env_if_not_present(
+                MEILI_EXPERIMENTAL_NO_EDITION_2024_FOR_SETTINGS,
+                experimental_no_edition_2024_for_settings.to_string(),
+            );
+        }
     }
 }
 
@@ -772,18 +795,23 @@ impl TryFrom<&IndexerOpts> for IndexerConfig {
     type Error = anyhow::Error;
 
     fn try_from(other: &IndexerOpts) -> Result<Self, Self::Error> {
-        let thread_pool = ThreadPoolNoAbortBuilder::new()
-            .thread_name(|index| format!("indexing-thread:{index}"))
-            .num_threads(*other.max_indexing_threads)
+        let thread_pool = ThreadPoolNoAbortBuilder::new_for_indexing()
+            .num_threads(other.max_indexing_threads.unwrap_or_else(|| num_cpus::get() / 2))
             .build()?;
 
         Ok(Self {
+            thread_pool,
             log_every_n: Some(DEFAULT_LOG_EVERY_N),
             max_memory: other.max_indexing_memory.map(|b| b.as_u64() as usize),
-            thread_pool: Some(thread_pool),
+            max_threads: *other.max_indexing_threads,
             max_positions_per_attributes: None,
             skip_index_budget: other.skip_index_budget,
-            ..Default::default()
+            experimental_no_edition_2024_for_settings: other
+                .experimental_no_edition_2024_for_settings,
+            chunk_compression_type: Default::default(),
+            chunk_compression_level: Default::default(),
+            documents_chunk_size: Default::default(),
+            max_nb_chunks: Default::default(),
         })
     }
 }
@@ -843,31 +871,31 @@ fn total_memory_bytes() -> Option<u64> {
     }
 }
 
-#[derive(Debug, Clone, Copy, Deserialize, Serialize)]
-pub struct MaxThreads(usize);
+#[derive(Default, Debug, Clone, Copy, Deserialize, Serialize)]
+pub struct MaxThreads(Option<usize>);
 
 impl FromStr for MaxThreads {
     type Err = ParseIntError;
 
-    fn from_str(s: &str) -> Result<Self, Self::Err> {
-        usize::from_str(s).map(Self)
-    }
-}
-
-impl Default for MaxThreads {
-    fn default() -> Self {
-        MaxThreads(num_cpus::get() / 2)
+    fn from_str(s: &str) -> Result<MaxThreads, Self::Err> {
+        if s.is_empty() || s == "unlimited" {
+            return Ok(MaxThreads::default());
+        }
+        usize::from_str(s).map(Some).map(MaxThreads)
     }
 }
 
 impl fmt::Display for MaxThreads {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        write!(f, "{}", self.0)
+        match self.0 {
+            Some(threads) => write!(f, "{}", threads),
+            None => write!(f, "unlimited"),
+        }
     }
 }
 
 impl Deref for MaxThreads {
-    type Target = usize;
+    type Target = Option<usize>;
 
     fn deref(&self) -> &Self::Target {
         &self.0

crates/meilisearch/src/routes/chats/chat_completion_analytics.rs

@@ -0,0 +1,135 @@
+use std::collections::BinaryHeap;
+
+use serde_json::{json, Value};
+
+use crate::analytics::Aggregate;
+
+#[derive(Default)]
+pub struct ChatCompletionAggregator {
+    // requests
+    total_received: usize,
+    total_succeeded: usize,
+    time_spent: BinaryHeap<usize>,
+
+    // chat completion specific metrics
+    total_messages: usize,
+    total_streamed_requests: usize,
+    total_non_streamed_requests: usize,
+
+    // model usage tracking
+    models_used: std::collections::HashMap<String, usize>,
+}
+
+impl ChatCompletionAggregator {
+    pub fn from_request(model: &str, message_count: usize, is_stream: bool) -> Self {
+        let mut models_used = std::collections::HashMap::new();
+        models_used.insert(model.to_string(), 1);
+
+        Self {
+            total_received: 1,
+            total_succeeded: 0,
+            time_spent: BinaryHeap::new(),
+
+            total_messages: message_count,
+            total_streamed_requests: if is_stream { 1 } else { 0 },
+            total_non_streamed_requests: if is_stream { 0 } else { 1 },
+
+            models_used,
+        }
+    }
+
+    pub fn succeed(&mut self, time_spent: std::time::Duration) {
+        self.total_succeeded += 1;
+        self.time_spent.push(time_spent.as_millis() as usize);
+    }
+}
+
+impl Aggregate for ChatCompletionAggregator {
+    fn event_name(&self) -> &'static str {
+        "Chat Completion POST"
+    }
+
+    fn aggregate(mut self: Box<Self>, new: Box<Self>) -> Box<Self> {
+        let Self {
+            total_received,
+            total_succeeded,
+            mut time_spent,
+            total_messages,
+            total_streamed_requests,
+            total_non_streamed_requests,
+            models_used,
+            ..
+        } = *new;
+
+        // Aggregate time spent
+        self.time_spent.append(&mut time_spent);
+
+        // Aggregate counters
+        self.total_received = self.total_received.saturating_add(total_received);
+        self.total_succeeded = self.total_succeeded.saturating_add(total_succeeded);
+        self.total_messages = self.total_messages.saturating_add(total_messages);
+        self.total_streamed_requests =
+            self.total_streamed_requests.saturating_add(total_streamed_requests);
+        self.total_non_streamed_requests =
+            self.total_non_streamed_requests.saturating_add(total_non_streamed_requests);
+
+        // Aggregate model usage
+        for (model, count) in models_used {
+            *self.models_used.entry(model).or_insert(0) += count;
+        }
+
+        self
+    }
+
+    fn into_event(self: Box<Self>) -> Value {
+        let Self {
+            total_received,
+            total_succeeded,
+            time_spent,
+            total_messages,
+            total_streamed_requests,
+            total_non_streamed_requests,
+            models_used,
+            ..
+        } = *self;
+
+        // Compute time statistics
+        let time_spent: Vec<usize> = time_spent.into_sorted_vec();
+        let (max_time, min_time, avg_time) = if time_spent.is_empty() {
+            (0, 0, 0)
+        } else {
+            let max_time = time_spent.last().unwrap_or(&0);
+            let min_time = time_spent.first().unwrap_or(&0);
+            let sum: usize = time_spent.iter().sum();
+            let avg_time = sum / time_spent.len();
+            (*max_time, *min_time, avg_time)
+        };
+
+        // Compute average messages per request
+        let avg_messages_per_request =
+            if total_received > 0 { total_messages as f64 / total_received as f64 } else { 0.0 };
+
+        // Compute streaming vs non-streaming proportions
+        let streaming_ratio = if total_received > 0 {
+            total_streamed_requests as f64 / total_received as f64
+        } else {
+            0.0
+        };
+
+        json!({
+            "total_received": total_received,
+            "total_succeeded": total_succeeded,
+            "time_spent": {
+                "max": max_time,
+                "min": min_time,
+                "avg": avg_time
+            },
+            "total_messages": total_messages,
+            "avg_messages_per_request": avg_messages_per_request,
+            "total_streamed_requests": total_streamed_requests,
+            "total_non_streamed_requests": total_non_streamed_requests,
+            "streaming_ratio": streaming_ratio,
+            "models_used": models_used,
+        })
+    }
+}

crates/meilisearch/src/routes/chats/chat_completions.rs

@@ -0,0 +1,804 @@
+use std::collections::HashMap;
+use std::fmt::Write as _;
+use std::mem;
+use std::ops::ControlFlow;
+use std::time::Duration;
+
+use actix_web::web::{self, Data};
+use actix_web::{Either, HttpRequest, HttpResponse, Responder};
+use actix_web_lab::sse::{Event, Sse};
+use async_openai::types::{
+    ChatCompletionMessageToolCall, ChatCompletionMessageToolCallChunk,
+    ChatCompletionRequestAssistantMessageArgs, ChatCompletionRequestDeveloperMessage,
+    ChatCompletionRequestDeveloperMessageContent, ChatCompletionRequestMessage,
+    ChatCompletionRequestSystemMessage, ChatCompletionRequestSystemMessageContent,
+    ChatCompletionRequestToolMessage, ChatCompletionRequestToolMessageContent,
+    ChatCompletionStreamOptions, ChatCompletionStreamResponseDelta, ChatCompletionToolArgs,
+    ChatCompletionToolType, CreateChatCompletionRequest, CreateChatCompletionStreamResponse,
+    FinishReason, FunctionCall, FunctionCallStream, FunctionObjectArgs,
+};
+use async_openai::Client;
+use bumpalo::Bump;
+use futures::StreamExt;
+use index_scheduler::IndexScheduler;
+use meilisearch_auth::AuthController;
+use meilisearch_types::error::{Code, ResponseError};
+use meilisearch_types::features::{
+    ChatCompletionPrompts as DbChatCompletionPrompts,
+    ChatCompletionSource as DbChatCompletionSource, SystemRole,
+};
+use meilisearch_types::keys::actions;
+use meilisearch_types::milli::index::ChatConfig;
+use meilisearch_types::milli::{all_obkv_to_json, obkv_to_json, TimeBudget};
+use meilisearch_types::{Document, Index};
+use serde::Deserialize;
+use serde_json::json;
+use tokio::runtime::Handle;
+use tokio::sync::mpsc::error::SendError;
+
+use super::chat_completion_analytics::ChatCompletionAggregator;
+use super::config::Config;
+use super::errors::{MistralError, OpenAiOutsideError, StreamErrorEvent};
+use super::utils::format_documents;
+use super::{
+    ChatsParam, MEILI_APPEND_CONVERSATION_MESSAGE_NAME, MEILI_SEARCH_IN_INDEX_FUNCTION_NAME,
+    MEILI_SEARCH_PROGRESS_NAME, MEILI_SEARCH_SOURCES_NAME,
+};
+use crate::analytics::Analytics;
+use crate::error::MeilisearchHttpError;
+use crate::extractors::authentication::policies::ActionPolicy;
+use crate::extractors::authentication::{extract_token_from_request, GuardedData, Policy as _};
+use crate::metrics::{
+    MEILISEARCH_CHAT_COMPLETION_TOKENS_USAGE, MEILISEARCH_CHAT_PROMPT_TOKENS_USAGE,
+    MEILISEARCH_CHAT_SEARCH_REQUESTS, MEILISEARCH_CHAT_TOTAL_TOKENS_USAGE,
+    MEILISEARCH_DEGRADED_SEARCH_REQUESTS,
+};
+use crate::routes::chats::utils::SseEventSender;
+use crate::routes::indexes::search::search_kind;
+use crate::search::{add_search_rules, prepare_search, search_from_kind, SearchQuery};
+use crate::search_queue::SearchQueue;
+
+pub fn configure(cfg: &mut web::ServiceConfig) {
+    cfg.service(web::resource("").route(web::post().to(chat)));
+}
+
+/// Get a chat completion
+async fn chat(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::CHAT_COMPLETIONS }>, Data<IndexScheduler>>,
+    auth_ctrl: web::Data<AuthController>,
+    chats_param: web::Path<ChatsParam>,
+    req: HttpRequest,
+    search_queue: web::Data<SearchQueue>,
+    web::Json(chat_completion): web::Json<CreateChatCompletionRequest>,
+    analytics: web::Data<Analytics>,
+) -> impl Responder {
+    let ChatsParam { workspace_uid } = chats_param.into_inner();
+
+    if chat_completion.stream.unwrap_or(false) {
+        Either::Right(
+            streamed_chat(
+                index_scheduler,
+                auth_ctrl,
+                search_queue,
+                &workspace_uid,
+                req,
+                chat_completion,
+                analytics,
+            )
+            .await,
+        )
+    } else {
+        Either::Left(
+            non_streamed_chat(
+                index_scheduler,
+                auth_ctrl,
+                search_queue,
+                &workspace_uid,
+                req,
+                chat_completion,
+                analytics,
+            )
+            .await,
+        )
+    }
+}
+
+#[derive(Default, Debug, Clone, Copy)]
+pub struct FunctionSupport {
+    /// Defines if we can call the _meiliSearchProgress function
+    /// to inform the front-end about what we are searching for.
+    report_progress: bool,
+    /// Defines if we can call the _meiliSearchSources function
+    /// to inform the front-end about the sources of the search.
+    report_sources: bool,
+    /// Defines if we can call the _meiliAppendConversationMessage
+    /// function to provide the messages to append into the conversation.
+    append_to_conversation: bool,
+}
+
+/// Setup search tool in chat completion request
+fn setup_search_tool(
+    index_scheduler: &Data<IndexScheduler>,
+    filters: &meilisearch_auth::AuthFilter,
+    chat_completion: &mut CreateChatCompletionRequest,
+    prompts: &DbChatCompletionPrompts,
+    system_role: SystemRole,
+) -> Result<FunctionSupport, ResponseError> {
+    let tools = chat_completion.tools.get_or_insert_default();
+    for tool in &tools[..] {
+        match tool.function.name.as_str() {
+            MEILI_SEARCH_IN_INDEX_FUNCTION_NAME => {
+                return Err(ResponseError::from_msg(
+                    format!("{MEILI_SEARCH_IN_INDEX_FUNCTION_NAME} function is already defined."),
+                    Code::BadRequest,
+                ));
+            }
+            MEILI_SEARCH_PROGRESS_NAME
+            | MEILI_SEARCH_SOURCES_NAME
+            | MEILI_APPEND_CONVERSATION_MESSAGE_NAME => (),
+            external_function_name => {
+                return Err(ResponseError::from_msg(
+                    format!("{external_function_name}: External functions are not supported yet."),
+                    Code::UnimplementedExternalFunctionCalling,
+                ));
+            }
+        }
+    }
+
+    // Remove internal tools used for front-end notifications as they should be hidden from the LLM.
+    let mut report_progress = false;
+    let mut report_sources = false;
+    let mut append_to_conversation = false;
+    tools.retain(|tool| {
+        match tool.function.name.as_str() {
+            MEILI_SEARCH_PROGRESS_NAME => {
+                report_progress = true;
+                false
+            }
+            MEILI_SEARCH_SOURCES_NAME => {
+                report_sources = true;
+                false
+            }
+            MEILI_APPEND_CONVERSATION_MESSAGE_NAME => {
+                append_to_conversation = true;
+                false
+            }
+            _ => true, // keep other tools
+        }
+    });
+
+    let mut index_uids = Vec::new();
+    let mut function_description = prompts.search_description.clone();
+    index_scheduler.try_for_each_index::<_, ()>(|name, index| {
+        // Make sure to skip unauthorized indexes
+        if !filters.is_index_authorized(name) {
+            return Ok(());
+        }
+
+        let rtxn = index.read_txn()?;
+        let chat_config = index.chat_config(&rtxn)?;
+        let index_description = chat_config.description;
+        let _ = writeln!(&mut function_description, "\n\n - {name}: {index_description}\n");
+        index_uids.push(name.to_string());
+
+        Ok(())
+    })?;
+
+    let tool = ChatCompletionToolArgs::default()
+        .r#type(ChatCompletionToolType::Function)
+        .function(
+            FunctionObjectArgs::default()
+                .name(MEILI_SEARCH_IN_INDEX_FUNCTION_NAME)
+                .description(&function_description)
+                .parameters(json!({
+                    "type": "object",
+                    "properties": {
+                        "index_uid": {
+                            "type": "string",
+                            "enum": index_uids,
+                            "description": prompts.search_index_uid_param,
+                        },
+                        "q": {
+                            // Unfortunately, Mistral does not support an array of types, here.
+                            // "type": ["string", "null"],
+                            "type": "string",
+                            "description": prompts.search_q_param,
+                        }
+                    },
+                    "required": ["index_uid", "q"],
+                    "additionalProperties": false,
+                }))
+                .strict(true)
+                .build()
+                .unwrap(),
+        )
+        .build()
+        .unwrap();
+
+    tools.push(tool);
+
+    let system_message = match system_role {
+        SystemRole::System => {
+            ChatCompletionRequestMessage::System(ChatCompletionRequestSystemMessage {
+                content: ChatCompletionRequestSystemMessageContent::Text(prompts.system.clone()),
+                name: None,
+            })
+        }
+        SystemRole::Developer => {
+            ChatCompletionRequestMessage::Developer(ChatCompletionRequestDeveloperMessage {
+                content: ChatCompletionRequestDeveloperMessageContent::Text(prompts.system.clone()),
+                name: None,
+            })
+        }
+    };
+    chat_completion.messages.insert(0, system_message);
+
+    Ok(FunctionSupport { report_progress, report_sources, append_to_conversation })
+}
+
+/// Process search request and return formatted results
+async fn process_search_request(
+    index_scheduler: &GuardedData<
+        ActionPolicy<{ actions::CHAT_COMPLETIONS }>,
+        Data<IndexScheduler>,
+    >,
+    auth_ctrl: web::Data<AuthController>,
+    search_queue: &web::Data<SearchQueue>,
+    auth_token: &str,
+    index_uid: String,
+    q: Option<String>,
+) -> Result<(Index, Vec<Document>, String), ResponseError> {
+    let index = index_scheduler.index(&index_uid)?;
+    let rtxn = index.static_read_txn()?;
+    let ChatConfig { description: _, prompt: _, search_parameters } = index.chat_config(&rtxn)?;
+    let mut query = SearchQuery { q, ..SearchQuery::from(search_parameters) };
+    let auth_filter = ActionPolicy::<{ actions::SEARCH }>::authenticate(
+        auth_ctrl,
+        auth_token,
+        Some(index_uid.as_str()),
+    )?;
+
+    // Tenant token search_rules.
+    if let Some(search_rules) = auth_filter.get_index_search_rules(&index_uid) {
+        add_search_rules(&mut query.filter, search_rules);
+    }
+    let search_kind =
+        search_kind(&query, index_scheduler.get_ref(), index_uid.to_string(), &index)?;
+
+    let permit = search_queue.try_get_search_permit().await?;
+    let features = index_scheduler.features();
+    let index_cloned = index.clone();
+    let output = tokio::task::spawn_blocking(move || -> Result<_, ResponseError> {
+        let time_budget = match index_cloned
+            .search_cutoff(&rtxn)
+            .map_err(|e| MeilisearchHttpError::from_milli(e, Some(index_uid.clone())))?
+        {
+            Some(cutoff) => TimeBudget::new(Duration::from_millis(cutoff)),
+            None => TimeBudget::default(),
+        };
+
+        let (search, _is_finite_pagination, _max_total_hits, _offset) =
+            prepare_search(&index_cloned, &rtxn, &query, &search_kind, time_budget, features)?;
+
+        search_from_kind(index_uid, search_kind, search)
+            .map(|(search_results, _)| (rtxn, search_results))
+            .map_err(ResponseError::from)
+    })
+    .await;
+    permit.drop().await;
+
+    let output = output?;
+    let mut documents = Vec::new();
+    if let Ok((ref rtxn, ref search_result)) = output {
+        MEILISEARCH_CHAT_SEARCH_REQUESTS.with_label_values(&["internal"]).inc();
+        if search_result.degraded {
+            MEILISEARCH_DEGRADED_SEARCH_REQUESTS.inc();
+        }
+
+        let fields_ids_map = index.fields_ids_map(rtxn)?;
+        let displayed_fields = index.displayed_fields_ids(rtxn)?;
+        for &document_id in &search_result.documents_ids {
+            let obkv = index.document(rtxn, document_id)?;
+            let document = match displayed_fields {
+                Some(ref fields) => obkv_to_json(fields, &fields_ids_map, obkv)?,
+                None => all_obkv_to_json(obkv, &fields_ids_map)?,
+            };
+            documents.push(document);
+        }
+    }
+
+    let (rtxn, search_result) = output?;
+    let render_alloc = Bump::new();
+    let formatted = format_documents(&rtxn, &index, &render_alloc, search_result.documents_ids)?;
+    let text = formatted.join("\n");
+    drop(rtxn);
+
+    Ok((index, documents, text))
+}
+
+#[allow(unreachable_code, unused_variables)] // will be correctly implemented in the future
+async fn non_streamed_chat(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::CHAT_COMPLETIONS }>, Data<IndexScheduler>>,
+    auth_ctrl: web::Data<AuthController>,
+    search_queue: web::Data<SearchQueue>,
+    workspace_uid: &str,
+    req: HttpRequest,
+    chat_completion: CreateChatCompletionRequest,
+    analytics: web::Data<Analytics>,
+) -> Result<HttpResponse, ResponseError> {
+    index_scheduler.features().check_chat_completions("using the /chats chat completions route")?;
+
+    // Create analytics aggregator
+    let aggregate = ChatCompletionAggregator::from_request(
+        &chat_completion.model,
+        chat_completion.messages.len(),
+        false, // non_streamed_chat is not streaming
+    );
+    let start_time = std::time::Instant::now();
+
+    if let Some(n) = chat_completion.n.filter(|&n| n != 1) {
+        return Err(ResponseError::from_msg(
+            format!("You tried to specify n = {n} but only single choices are supported (n = 1)."),
+            Code::UnimplementedMultiChoiceChatCompletions,
+        ));
+    }
+
+    return Err(ResponseError::from_msg(
+        "Non-streamed chat completions is not implemented".to_string(),
+        Code::UnimplementedNonStreamingChatCompletions,
+    ));
+
+    let filters = index_scheduler.filters();
+    let chat_settings = match index_scheduler.chat_settings(workspace_uid).unwrap() {
+        Some(settings) => settings,
+        None => {
+            return Err(ResponseError::from_msg(
+                format!("Chat `{workspace_uid}` not found"),
+                Code::ChatNotFound,
+            ))
+        }
+    };
+
+    let config = Config::new(&chat_settings);
+    let client = Client::with_config(config);
+    let auth_token = extract_token_from_request(&req)?.unwrap();
+    let system_role = chat_settings.source.system_role(&chat_completion.model);
+    // TODO do function support later
+    let _function_support = setup_search_tool(
+        &index_scheduler,
+        filters,
+        &mut chat_completion,
+        &chat_settings.prompts,
+        system_role,
+    )?;
+
+    let mut response;
+    loop {
+        response = client.chat().create(chat_completion.clone()).await.unwrap();
+
+        let choice = &mut response.choices[0];
+        match choice.finish_reason {
+            Some(FinishReason::ToolCalls) => {
+                let tool_calls = mem::take(&mut choice.message.tool_calls).unwrap_or_default();
+
+                let (meili_calls, other_calls): (Vec<_>, Vec<_>) = tool_calls
+                    .into_iter()
+                    .partition(|call| call.function.name == MEILI_SEARCH_IN_INDEX_FUNCTION_NAME);
+
+                chat_completion.messages.push(
+                    ChatCompletionRequestAssistantMessageArgs::default()
+                        .tool_calls(meili_calls.clone())
+                        .build()
+                        .unwrap()
+                        .into(),
+                );
+
+                for call in meili_calls {
+                    let result = match serde_json::from_str(&call.function.arguments) {
+                        Ok(SearchInIndexParameters { index_uid, q }) => process_search_request(
+                            &index_scheduler,
+                            auth_ctrl.clone(),
+                            &search_queue,
+                            auth_token,
+                            index_uid,
+                            q,
+                        )
+                        .await
+                        .map_err(|e| e.to_string()),
+                        Err(err) => Err(err.to_string()),
+                    };
+
+                    // TODO report documents sources later
+                    let answer = match result {
+                        Ok((_, _documents, text)) => text,
+                        Err(err) => err,
+                    };
+
+                    chat_completion.messages.push(ChatCompletionRequestMessage::Tool(
+                        ChatCompletionRequestToolMessage {
+                            tool_call_id: call.id.clone(),
+                            content: ChatCompletionRequestToolMessageContent::Text(answer),
+                        },
+                    ));
+                }
+
+                // Let the client call other tools by themselves
+                if !other_calls.is_empty() {
+                    response.choices[0].message.tool_calls = Some(other_calls);
+                    break;
+                }
+            }
+            _ => break,
+        }
+    }
+
+    // Record success in analytics
+    let mut aggregate = aggregate;
+    aggregate.succeed(start_time.elapsed());
+    analytics.publish(aggregate, &req);
+
+    Ok(HttpResponse::Ok().json(response))
+}
+
+async fn streamed_chat(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::CHAT_COMPLETIONS }>, Data<IndexScheduler>>,
+    auth_ctrl: web::Data<AuthController>,
+    search_queue: web::Data<SearchQueue>,
+    workspace_uid: &str,
+    req: HttpRequest,
+    mut chat_completion: CreateChatCompletionRequest,
+    analytics: web::Data<Analytics>,
+) -> Result<impl Responder, ResponseError> {
+    index_scheduler.features().check_chat_completions("using the /chats chat completions route")?;
+    let filters = index_scheduler.filters();
+
+    if let Some(n) = chat_completion.n.filter(|&n| n != 1) {
+        return Err(ResponseError::from_msg(
+            format!("You tried to specify n = {n} but only single choices are supported (n = 1)."),
+            Code::UnimplementedMultiChoiceChatCompletions,
+        ));
+    }
+
+    let chat_settings = match index_scheduler.chat_settings(workspace_uid)? {
+        Some(settings) => settings,
+        None => {
+            return Err(ResponseError::from_msg(
+                format!("Chat `{workspace_uid}` not found"),
+                Code::ChatNotFound,
+            ))
+        }
+    };
+
+    // Create analytics aggregator
+    let mut aggregate = ChatCompletionAggregator::from_request(
+        &chat_completion.model,
+        chat_completion.messages.len(),
+        true, // streamed_chat is always streaming
+    );
+    let start_time = std::time::Instant::now();
+
+    let config = Config::new(&chat_settings);
+    let auth_token = extract_token_from_request(&req)?.unwrap().to_string();
+    let system_role = chat_settings.source.system_role(&chat_completion.model);
+    let function_support = setup_search_tool(
+        &index_scheduler,
+        filters,
+        &mut chat_completion,
+        &chat_settings.prompts,
+        system_role,
+    )?;
+
+    tracing::debug!("Conversation function support: {function_support:?}");
+
+    let (tx, rx) = tokio::sync::mpsc::channel(10);
+    let tx = SseEventSender::new(tx);
+    let workspace_uid = workspace_uid.to_string();
+    let _join_handle = Handle::current().spawn(async move {
+        let client = Client::with_config(config.clone());
+        let mut global_tool_calls = HashMap::<u32, Call>::new();
+
+        // Limit the number of internal calls to satisfy the search requests of the LLM
+        for _ in 0..20 {
+            let output = run_conversation(
+                &index_scheduler,
+                &auth_ctrl,
+                &workspace_uid,
+                &search_queue,
+                &auth_token,
+                &client,
+                chat_settings.source,
+                &mut chat_completion,
+                &tx,
+                &mut global_tool_calls,
+                function_support,
+            );
+
+            match output.await {
+                Ok(ControlFlow::Continue(())) => (),
+                Ok(ControlFlow::Break(_finish_reason)) => break,
+                // If the connection is closed we must stop
+                Err(SendError(_)) => return,
+            }
+        }
+
+        let _ = tx.stop().await;
+    });
+
+    // Record success in analytics after the stream is set up
+    aggregate.succeed(start_time.elapsed());
+    analytics.publish(aggregate, &req);
+
+    Ok(Sse::from_infallible_receiver(rx).with_retry_duration(Duration::from_secs(10)))
+}
+
+/// Updates the chat completion with the new messages, streams the LLM tokens,
+/// and report progress and errors.
+#[allow(clippy::too_many_arguments)]
+async fn run_conversation<C: async_openai::config::Config>(
+    index_scheduler: &GuardedData<
+        ActionPolicy<{ actions::CHAT_COMPLETIONS }>,
+        Data<IndexScheduler>,
+    >,
+    auth_ctrl: &web::Data<AuthController>,
+    workspace_uid: &str,
+    search_queue: &web::Data<SearchQueue>,
+    auth_token: &str,
+    client: &Client<C>,
+    source: DbChatCompletionSource,
+    chat_completion: &mut CreateChatCompletionRequest,
+    tx: &SseEventSender,
+    global_tool_calls: &mut HashMap<u32, Call>,
+    function_support: FunctionSupport,
+) -> Result<ControlFlow<Option<FinishReason>, ()>, SendError<Event>> {
+    use DbChatCompletionSource::*;
+
+    let mut finish_reason = None;
+    chat_completion.stream_options = match source {
+        OpenAi | AzureOpenAi => Some(ChatCompletionStreamOptions { include_usage: true }),
+        Mistral | VLlm => None,
+    };
+
+    // safety: unwrap: can only happens if `stream` was set to `false`
+    let mut response = client.chat().create_stream(chat_completion.clone()).await.unwrap();
+    while let Some(result) = response.next().await {
+        match result {
+            Ok(resp) => {
+                if let Some(usage) = resp.usage.as_ref() {
+                    MEILISEARCH_CHAT_PROMPT_TOKENS_USAGE
+                        .with_label_values(&[workspace_uid, &chat_completion.model])
+                        .inc_by(usage.prompt_tokens as u64);
+                    MEILISEARCH_CHAT_COMPLETION_TOKENS_USAGE
+                        .with_label_values(&[workspace_uid, &chat_completion.model])
+                        .inc_by(usage.completion_tokens as u64);
+                    MEILISEARCH_CHAT_TOTAL_TOKENS_USAGE
+                        .with_label_values(&[workspace_uid, &chat_completion.model])
+                        .inc_by(usage.total_tokens as u64);
+                }
+                let choice = match resp.choices.first() {
+                    Some(choice) => choice,
+                    None => break,
+                };
+                finish_reason = choice.finish_reason;
+
+                let ChatCompletionStreamResponseDelta { ref tool_calls, .. } = &choice.delta;
+
+                match tool_calls {
+                    Some(tool_calls) => {
+                        for chunk in tool_calls {
+                            let ChatCompletionMessageToolCallChunk {
+                                index,
+                                id,
+                                r#type: _,
+                                function,
+                            } = chunk;
+                            let FunctionCallStream { name, arguments } = function.as_ref().unwrap();
+
+                            global_tool_calls
+                                .entry(*index)
+                                .and_modify(|call| {
+                                    if call.is_internal() {
+                                        call.append(arguments.as_ref().unwrap())
+                                    }
+                                })
+                                .or_insert_with(|| {
+                                    if name.as_deref() == Some(MEILI_SEARCH_IN_INDEX_FUNCTION_NAME)
+                                    {
+                                        Call::Internal {
+                                            id: id.as_ref().unwrap().clone(),
+                                            function_name: name.as_ref().unwrap().clone(),
+                                            arguments: arguments.as_ref().unwrap().clone(),
+                                        }
+                                    } else {
+                                        Call::External
+                                    }
+                                });
+                        }
+                    }
+                    None => {
+                        if !global_tool_calls.is_empty() {
+                            let (meili_calls, _other_calls): (Vec<_>, Vec<_>) =
+                                mem::take(global_tool_calls)
+                                    .into_values()
+                                    .flat_map(|call| match call {
+                                        Call::Internal { id, function_name: name, arguments } => {
+                                            Some(ChatCompletionMessageToolCall {
+                                                id,
+                                                r#type: Some(ChatCompletionToolType::Function),
+                                                function: FunctionCall { name, arguments },
+                                            })
+                                        }
+                                        Call::External => None,
+                                    })
+                                    .partition(|call| {
+                                        call.function.name == MEILI_SEARCH_IN_INDEX_FUNCTION_NAME
+                                    });
+
+                            chat_completion.messages.push(
+                                ChatCompletionRequestAssistantMessageArgs::default()
+                                    .tool_calls(meili_calls.clone())
+                                    .build()
+                                    .unwrap()
+                                    .into(),
+                            );
+
+                            handle_meili_tools(
+                                index_scheduler,
+                                auth_ctrl,
+                                search_queue,
+                                auth_token,
+                                tx,
+                                meili_calls,
+                                chat_completion,
+                                &resp,
+                                function_support,
+                            )
+                            .await?;
+                        } else {
+                            tx.forward_response(&resp).await?;
+                        }
+                    }
+                }
+            }
+            Err(error) => {
+                let result = match source {
+                    DbChatCompletionSource::Mistral => {
+                        StreamErrorEvent::from_openai_error::<MistralError>(error).await
+                    }
+                    _ => StreamErrorEvent::from_openai_error::<OpenAiOutsideError>(error).await,
+                };
+                let error = result.unwrap_or_else(StreamErrorEvent::from_reqwest_error);
+                tx.send_error(&error).await?;
+                return Ok(ControlFlow::Break(None));
+            }
+        }
+    }
+
+    // We must stop if the finish reason is not something we can solve with Meilisearch
+    match finish_reason {
+        Some(FinishReason::ToolCalls) => Ok(ControlFlow::Continue(())),
+        otherwise => Ok(ControlFlow::Break(otherwise)),
+    }
+}
+
+#[allow(clippy::too_many_arguments)]
+async fn handle_meili_tools(
+    index_scheduler: &GuardedData<
+        ActionPolicy<{ actions::CHAT_COMPLETIONS }>,
+        Data<IndexScheduler>,
+    >,
+    auth_ctrl: &web::Data<AuthController>,
+    search_queue: &web::Data<SearchQueue>,
+    auth_token: &str,
+    tx: &SseEventSender,
+    meili_calls: Vec<ChatCompletionMessageToolCall>,
+    chat_completion: &mut CreateChatCompletionRequest,
+    resp: &CreateChatCompletionStreamResponse,
+    FunctionSupport { report_progress, report_sources, append_to_conversation, .. }: FunctionSupport,
+) -> Result<(), SendError<Event>> {
+    for call in meili_calls {
+        if report_progress {
+            tx.report_search_progress(
+                resp.clone(),
+                &call.id,
+                &call.function.name,
+                &call.function.arguments,
+            )
+            .await?;
+        }
+
+        if append_to_conversation {
+            tx.append_tool_call_conversation_message(
+                resp.clone(),
+                call.id.clone(),
+                call.function.name.clone(),
+                call.function.arguments.clone(),
+            )
+            .await?;
+        }
+
+        let mut error = None;
+
+        let result = match serde_json::from_str(&call.function.arguments) {
+            Ok(SearchInIndexParameters { index_uid, q }) => match process_search_request(
+                index_scheduler,
+                auth_ctrl.clone(),
+                search_queue,
+                auth_token,
+                index_uid,
+                q,
+            )
+            .await
+            {
+                Ok(output) => Ok(output),
+                Err(err) => {
+                    let error_text = format!("the search tool call failed with {err}");
+                    error = Some(err);
+                    Err(error_text)
+                }
+            },
+            Err(err) => Err(err.to_string()),
+        };
+
+        let answer = match result {
+            Ok((_index, documents, text)) => {
+                if report_sources {
+                    tx.report_sources(resp.clone(), &call.id, &documents).await?;
+                }
+                text
+            }
+            Err(err) => err,
+        };
+
+        let tool = ChatCompletionRequestMessage::Tool(ChatCompletionRequestToolMessage {
+            tool_call_id: call.id.clone(),
+            content: ChatCompletionRequestToolMessageContent::Text(answer),
+        });
+
+        if append_to_conversation {
+            tx.append_conversation_message(resp.clone(), &tool).await?;
+        }
+
+        chat_completion.messages.push(tool);
+
+        if let Some(error) = error {
+            tx.send_error(&StreamErrorEvent::from_response_error(error)).await?;
+        }
+    }
+
+    Ok(())
+}
+
+/// The structure used to aggregate the function calls to make.
+#[derive(Debug)]
+enum Call {
+    /// Tool calls to tools that must be managed by Meilisearch internally.
+    /// Typically the search functions.
+    Internal { id: String, function_name: String, arguments: String },
+    /// Tool calls that we track but only to know that its not our functions.
+    /// We return the function calls as-is to the end-user.
+    External,
+}
+
+impl Call {
+    fn is_internal(&self) -> bool {
+        matches!(self, Call::Internal { .. })
+    }
+
+    /// # Panics
+    ///
+    /// - if called on external calls
+    fn append(&mut self, more: &str) {
+        match self {
+            Call::Internal { arguments, .. } => arguments.push_str(more),
+            Call::External => panic!("Cannot append argument chunks to an external function"),
+        }
+    }
+}
+
+#[derive(Deserialize)]
+struct SearchInIndexParameters {
+    /// The index uid to search in.
+    index_uid: String,
+    /// The query parameter to use.
+    q: Option<String>,
+}

crates/meilisearch/src/routes/chats/config.rs

@@ -0,0 +1,88 @@
+use async_openai::config::{AzureConfig, OpenAIConfig};
+use meilisearch_types::features::ChatCompletionSettings as DbChatSettings;
+use reqwest::header::HeaderMap;
+use secrecy::SecretString;
+
+#[derive(Debug, Clone)]
+pub enum Config {
+    OpenAiCompatible(OpenAIConfig),
+    AzureOpenAiCompatible(AzureConfig),
+}
+
+impl Config {
+    pub fn new(chat_settings: &DbChatSettings) -> Self {
+        use meilisearch_types::features::ChatCompletionSource::*;
+        match chat_settings.source {
+            OpenAi | Mistral | VLlm => {
+                let mut config = OpenAIConfig::default();
+                if let Some(org_id) = chat_settings.org_id.as_ref() {
+                    config = config.with_org_id(org_id);
+                }
+                if let Some(project_id) = chat_settings.project_id.as_ref() {
+                    config = config.with_project_id(project_id);
+                }
+                if let Some(api_key) = chat_settings.api_key.as_ref() {
+                    config = config.with_api_key(api_key);
+                }
+                let base_url = chat_settings.base_url.as_deref();
+                if let Some(base_url) = chat_settings.source.base_url().or(base_url) {
+                    config = config.with_api_base(base_url);
+                }
+                Self::OpenAiCompatible(config)
+            }
+            AzureOpenAi => {
+                let mut config = AzureConfig::default();
+                if let Some(version) = chat_settings.api_version.as_ref() {
+                    config = config.with_api_version(version);
+                }
+                if let Some(deployment_id) = chat_settings.deployment_id.as_ref() {
+                    config = config.with_deployment_id(deployment_id);
+                }
+                if let Some(api_key) = chat_settings.api_key.as_ref() {
+                    config = config.with_api_key(api_key);
+                }
+                if let Some(base_url) = chat_settings.base_url.as_ref() {
+                    config = config.with_api_base(base_url);
+                }
+                Self::AzureOpenAiCompatible(config)
+            }
+        }
+    }
+}
+
+impl async_openai::config::Config for Config {
+    fn headers(&self) -> HeaderMap {
+        match self {
+            Config::OpenAiCompatible(config) => config.headers(),
+            Config::AzureOpenAiCompatible(config) => config.headers(),
+        }
+    }
+
+    fn url(&self, path: &str) -> String {
+        match self {
+            Config::OpenAiCompatible(config) => config.url(path),
+            Config::AzureOpenAiCompatible(config) => config.url(path),
+        }
+    }
+
+    fn query(&self) -> Vec<(&str, &str)> {
+        match self {
+            Config::OpenAiCompatible(config) => config.query(),
+            Config::AzureOpenAiCompatible(config) => config.query(),
+        }
+    }
+
+    fn api_base(&self) -> &str {
+        match self {
+            Config::OpenAiCompatible(config) => config.api_base(),
+            Config::AzureOpenAiCompatible(config) => config.api_base(),
+        }
+    }
+
+    fn api_key(&self) -> &SecretString {
+        match self {
+            Config::OpenAiCompatible(config) => config.api_key(),
+            Config::AzureOpenAiCompatible(config) => config.api_key(),
+        }
+    }
+}