Move the default pagination limit into a const

Move to Rust v1.91.1

.github/ISSUE_TEMPLATE/new_feature_issue.md

@@ -24,6 +24,11 @@ TBD
 - [ ] If not, add the `no db change` label to your PR, and you're good to merge.
 - [ ] If yes, add the `db change` label to your PR. You'll receive a message explaining you what to do.
 
+### Reminders when adding features
+
+- [ ] Write unit tests using insta
+- [ ] Write declarative integration tests in [workloads/tests](https://github.com/meilisearch/meilisearch/tree/main/workloads/test). Specify the routes to call and then call `cargo xtask test workloads/tests/YOUR_TEST.json --update-responses` so that responses are automatically filled.
+
 ### Reminders when modifying the API
 
 - [ ] Update the openAPI file with utoipa:

.github/workflows/bench-manual.yml

@@ -18,7 +18,7 @@ jobs:
     timeout-minutes: 180 # 3h
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
+      - uses: dtolnay/rust-toolchain@1.91.1
         with:
           profile: minimal
 

.github/workflows/bench-pr.yml

@@ -66,9 +66,7 @@ jobs:
           fetch-depth: 0 # fetch full history to be able to get main commit sha
           ref: ${{ steps.comment-branch.outputs.head_ref }}
 
-      - uses: dtolnay/rust-toolchain@1.89
-        with:
-          profile: minimal
+      - uses: dtolnay/rust-toolchain@1.91.1
 
       - name: Run benchmarks on PR ${{ github.event.issue.id }}
         run: |

.github/workflows/bench-push-indexing.yml

@@ -12,9 +12,7 @@ jobs:
     timeout-minutes: 180 # 3h
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
-        with:
-          profile: minimal
+      - uses: dtolnay/rust-toolchain@1.91.1
 
       # Run benchmarks
       - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch main - Commit ${{ github.sha }}

.github/workflows/benchmarks-manual.yml

@@ -18,7 +18,7 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
+      - uses: dtolnay/rust-toolchain@1.91.1
         with:
           profile: minimal
 

.github/workflows/benchmarks-pr.yml

@@ -44,7 +44,7 @@ jobs:
             exit 1
           fi
 
-      - uses: dtolnay/rust-toolchain@1.89
+      - uses: dtolnay/rust-toolchain@1.91.1
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-indexing.yml

@@ -16,7 +16,7 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
+      - uses: dtolnay/rust-toolchain@1.91.1
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-search-geo.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
+      - uses: dtolnay/rust-toolchain@1.91.1
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-search-songs.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
+      - uses: dtolnay/rust-toolchain@1.91.1
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-search-wiki.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
+      - uses: dtolnay/rust-toolchain@1.91.1
         with:
           profile: minimal
 

.github/workflows/db-change-comments.yml

@@ -19,6 +19,7 @@ env:
 
     - [ ] Detail the change to the DB format and why they are forward compatible
     - [ ] Forward-compatibility: A database created before this PR and using the features touched by this PR was able to be opened by a Meilisearch produced by the code of this PR.
+    - [ ] Declarative test: add a [declarative test containing a dumpless upgrade](https://github.com/meilisearch/meilisearch/blob/main/TESTING.md#typical-usage)
 
 
     ## This PR makes breaking changes
@@ -35,8 +36,7 @@ env:
       - [ ] Write the code to go from the old database to the new one
         - If the change happened in milli, the upgrade function should be written and called [here](https://github.com/meilisearch/meilisearch/blob/3fd86e8d76d7d468b0095d679adb09211ca3b6c0/crates/milli/src/update/upgrade/mod.rs#L24-L47)
         - If the change happened in the index-scheduler, we've never done it yet, but the right place to do it should be [here](https://github.com/meilisearch/meilisearch/blob/3fd86e8d76d7d468b0095d679adb09211ca3b6c0/crates/index-scheduler/src/scheduler/process_upgrade/mod.rs#L13)
-      - [ ] Write an integration test [here](https://github.com/meilisearch/meilisearch/blob/main/crates/meilisearch/tests/upgrade/mod.rs) ensuring you can read the old database, upgrade to the new database, and read the new database as expected
-    
+      - [ ] Declarative test: add a [declarative test containing a dumpless upgrade](https://github.com/meilisearch/meilisearch/blob/main/TESTING.md#typical-usage)
 
 jobs:
   add-comment:

.github/workflows/flaky-tests.yml

@@ -3,7 +3,7 @@ name: Look for flaky tests
 on:
   workflow_dispatch:
   schedule:
-    - cron: '0 4 * * *' # Every day at 4:00AM
+    - cron: "0 4 * * *" # Every day at 4:00AM
 
 jobs:
   flaky:
@@ -13,11 +13,17 @@ jobs:
       image: ubuntu:22.04
     steps:
       - uses: actions/checkout@v5
+      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+        run: |
+          sudo rm -rf "/opt/ghc" || true
+          sudo rm -rf "/usr/share/dotnet" || true
+          sudo rm -rf "/usr/local/lib/android" || true
+          sudo rm -rf "/usr/local/share/boost" || true
       - name: Install needed dependencies
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - uses: dtolnay/rust-toolchain@1.89
+      - uses: dtolnay/rust-toolchain@1.91.1
       - name: Install cargo-flaky
         run: cargo install cargo-flaky
       - name: Run cargo flaky in the dumps

.github/workflows/fuzzer-indexing.yml

@@ -12,9 +12,7 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
-        with:
-          profile: minimal
+      - uses: dtolnay/rust-toolchain@1.91.1
 
       # Run benchmarks
       - name: Run the fuzzer

.github/workflows/publish-apt-brew-pkg.yml

@@ -25,7 +25,13 @@ jobs:
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - uses: dtolnay/rust-toolchain@1.89
+      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+        run: |
+          sudo rm -rf "/opt/ghc" || true
+          sudo rm -rf "/usr/share/dotnet" || true
+          sudo rm -rf "/usr/local/lib/android" || true
+          sudo rm -rf "/usr/local/share/boost" || true
+      - uses: dtolnay/rust-toolchain@1.91.1
       - name: Install cargo-deb
         run: cargo install cargo-deb
       - uses: actions/checkout@v5

.github/workflows/publish-docker-images.yml

@@ -208,8 +208,8 @@ jobs:
           done
           cosign sign --yes ${images}
 
-      # /!\ Don't touch this without checking with Cloud team
-      - name: Send CI information to Cloud team
+      # /!\ Don't touch this without checking with engineers working on the Cloud code base on #discussion-engineering Slack channel
+      - name: Notify meilisearch-cloud 
         # Do not send if nightly build (i.e. 'schedule' or 'workflow_dispatch' event)
         if: ${{ (github.event_name == 'push') && (matrix.edition == 'enterprise') }}
         uses: peter-evans/repository-dispatch@v3
@@ -218,3 +218,14 @@ jobs:
           repository: meilisearch/meilisearch-cloud
           event-type: cloud-docker-build
           client-payload: '{ "meilisearch_version": "${{ github.ref_name }}", "stable": "${{ steps.check-tag-format.outputs.stable }}" }'
+
+      # /!\ Don't touch this without checking with integration team members on #discussion-integrations Slack channel 
+      - name: Notify meilisearch-kubernetes
+        # Do not send if nightly build (i.e. 'schedule' or 'workflow_dispatch' event), or if not stable
+        if: ${{ github.event_name == 'push' && matrix.edition == 'community' && steps.check-tag-format.outputs.stable == 'true' }}
+        uses: peter-evans/repository-dispatch@v3
+        with:
+          token: ${{ secrets.MEILI_BOT_GH_PAT }}
+          repository: meilisearch/meilisearch-kubernetes
+          event-type: meilisearch-release
+          client-payload: '{ "version": "${{ github.ref_name }}" }'

.github/workflows/publish-release-assets.yml

@@ -76,7 +76,7 @@ jobs:
     needs: check-version
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
+      - uses: dtolnay/rust-toolchain@1.91.1
       - name: Build
         run: cargo build --release --locked ${{ matrix.feature-flag }} ${{ matrix.extra-args }}
       # No need to upload binaries for dry run (cron or workflow_dispatch)

.github/workflows/test-suite.yml

@@ -19,31 +19,36 @@ jobs:
     runs-on: ${{ matrix.runner }}
     strategy:
       matrix:
-        runner: [ubuntu-24.04, ubuntu-24.04-arm]
+        runner: [ubuntu-22.04, ubuntu-22.04-arm]
         features: ["", "--features enterprise"]
-    container:
-      # Use ubuntu-22.04 to compile with glibc 2.35
-      image: ubuntu:22.04
     steps:
       - uses: actions/checkout@v5
-      - name: Install needed dependencies
+      - name: check free space before
+        run: df -h
+      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
         run: |
-          apt-get update && apt-get install -y curl
-          apt-get install build-essential -y
+          sudo rm -rf "/opt/ghc" || true
+          sudo rm -rf "/usr/share/dotnet" || true
+          sudo rm -rf "/usr/local/lib/android" || true
+          sudo rm -rf "/usr/local/share/boost" || true
+      - name: check free space after
+        run: df -h
       - name: Setup test with Rust stable
-        uses: dtolnay/rust-toolchain@1.89
+        uses: dtolnay/rust-toolchain@1.91.1
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.8.0
-      - name: Run cargo check without any default features
+        with:
+          key: ${{ matrix.features }}
+      - name: Run cargo build without any default features
         uses: actions-rs/cargo@v1
         with:
           command: build
-          args: --locked --release --no-default-features --all
+          args: --locked --no-default-features --all
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --locked --release --all ${{ matrix.features }}
+          args: --locked --all ${{ matrix.features }}
 
   test-others:
     name: Tests on ${{ matrix.os }}
@@ -53,53 +58,56 @@ jobs:
       matrix:
         os: [macos-14, windows-2022]
         features: ["", "--features enterprise"]
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/checkout@v5
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.8.0
-      - uses: dtolnay/rust-toolchain@1.89
-      - name: Run cargo check without any default features
+      - uses: dtolnay/rust-toolchain@1.91.1
+      - name: Run cargo build without any default features
         uses: actions-rs/cargo@v1
         with:
           command: build
-          args: --locked --release --no-default-features --all
+          args: --locked --no-default-features --all
       - name: Run cargo test
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --locked --release --all ${{ matrix.features }}
+          args: --locked --all ${{ matrix.features }}
 
   test-all-features:
     name: Tests almost all features
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-22.04 to compile with glibc 2.35
-      image: ubuntu:22.04
+    runs-on: ubuntu-22.04
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/checkout@v5
-      - name: Install needed dependencies
+      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
         run: |
-          apt-get update
-          apt-get install --assume-yes build-essential curl
-      - uses: dtolnay/rust-toolchain@1.89
+          sudo rm -rf "/opt/ghc" || true
+          sudo rm -rf "/usr/share/dotnet" || true
+          sudo rm -rf "/usr/local/lib/android" || true
+          sudo rm -rf "/usr/local/share/boost" || true
+      - uses: dtolnay/rust-toolchain@1.91.1
       - name: Run cargo build with almost all features
         run: |
-          cargo build --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda,test-ollama)"
+          cargo build --workspace --locked --features "$(cargo xtask list-features --exclude-feature cuda,test-ollama)"
       - name: Run cargo test with almost all features
         run: |
-          cargo test --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda,test-ollama)"
+          cargo test --workspace --locked --features "$(cargo xtask list-features --exclude-feature cuda,test-ollama)"
 
   ollama-ubuntu:
     name: Test with Ollama
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        features: ["", "--features enterprise"]
+    runs-on: ubuntu-22.04
     env:
       MEILI_TEST_OLLAMA_SERVER: "http://localhost:11434"
     steps:
       - uses: actions/checkout@v5
+      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+        run: |
+          sudo rm -rf "/opt/ghc" || true
+          sudo rm -rf "/usr/share/dotnet" || true
+          sudo rm -rf "/usr/local/lib/android" || true
+          sudo rm -rf "/usr/local/share/boost" || true
       - name: Install Ollama
         run: |
           curl -fsSL https://ollama.com/install.sh | sudo -E sh
@@ -123,21 +131,21 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --locked --release --all --features test-ollama ollama ${{ matrix.features }}
+          args: --locked -p meilisearch --features test-ollama ollama
 
   test-disabled-tokenization:
     name: Test disabled tokenization
-    runs-on: ubuntu-latest
-    container:
-      image: ubuntu:22.04
+    runs-on: ubuntu-22.04
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/checkout@v5
-      - name: Install needed dependencies
+      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
         run: |
-          apt-get update
-          apt-get install --assume-yes build-essential curl
-      - uses: dtolnay/rust-toolchain@1.89
+          sudo rm -rf "/opt/ghc" || true
+          sudo rm -rf "/usr/share/dotnet" || true
+          sudo rm -rf "/usr/local/lib/android" || true
+          sudo rm -rf "/usr/local/share/boost" || true
+      - uses: dtolnay/rust-toolchain@1.91.1
       - name: Run cargo tree without default features and check lindera is not present
         run: |
           if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -qz lindera; then
@@ -148,35 +156,39 @@ jobs:
         run: |
           cargo tree -f '{p} {f}' -e normal | grep lindera -qz
 
-  # We run tests in debug also, to make sure that the debug_assertions are hit
-  test-debug:
-    name: Run tests in debug
+  build:
+    name: Build in release
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v5
+      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+        run: |
+          sudo rm -rf "/opt/ghc" || true
+          sudo rm -rf "/usr/share/dotnet" || true
+          sudo rm -rf "/usr/local/lib/android" || true
+          sudo rm -rf "/usr/local/share/boost" || true
+      - uses: dtolnay/rust-toolchain@1.91.1
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v2.8.0
+      - name: Build
+        run: cargo build --release --locked --target x86_64-unknown-linux-gnu
+
+  clippy:
+    name: Run Clippy
     runs-on: ubuntu-22.04
     strategy:
       matrix:
         features: ["", "--features enterprise"]
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.8.0
-      - name: Run tests in debug
-        uses: actions-rs/cargo@v1
+      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+        run: |
+          sudo rm -rf "/opt/ghc" || true
+          sudo rm -rf "/usr/share/dotnet" || true
+          sudo rm -rf "/usr/local/lib/android" || true
+          sudo rm -rf "/usr/local/share/boost" || true
+      - uses: dtolnay/rust-toolchain@1.91.1
         with:
-          command: test
-          args: --locked --all ${{ matrix.features }}
-
-  clippy:
-    name: Run Clippy
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        features: ["", "--features enterprise"]
-    steps:
-      - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
-        with:
-          profile: minimal
           components: clippy
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.8.0
@@ -188,14 +200,17 @@ jobs:
 
   fmt:
     name: Run Rustfmt
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
+      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+        run: |
+          sudo rm -rf "/opt/ghc" || true
+          sudo rm -rf "/usr/share/dotnet" || true
+          sudo rm -rf "/usr/local/lib/android" || true
+          sudo rm -rf "/usr/local/share/boost" || true
+      - uses: dtolnay/rust-toolchain@1.91.1
         with:
-          profile: minimal
-          toolchain: nightly-2024-07-09
-          override: true
           components: rustfmt
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.8.0
@@ -206,3 +221,23 @@ jobs:
         run: |
           echo -ne "\n" > crates/benchmarks/benches/datasets_paths.rs
           cargo fmt --all -- --check
+
+  declarative-tests:
+    name: Run declarative tests
+    runs-on: ubuntu-22.04-arm
+    permissions:
+      contents: read
+    steps:
+      - uses: actions/checkout@v5
+      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+        run: |
+          sudo rm -rf "/opt/ghc" || true
+          sudo rm -rf "/usr/share/dotnet" || true
+          sudo rm -rf "/usr/local/lib/android" || true
+          sudo rm -rf "/usr/local/share/boost" || true
+      - uses: dtolnay/rust-toolchain@1.91.1
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v2.8.0
+      - name: Run declarative tests
+        run: |
+          cargo xtask test workloads/tests/*.json

.github/workflows/update-cargo-toml-version.yml

@@ -18,9 +18,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v5
-      - uses: dtolnay/rust-toolchain@1.89
-        with:
-          profile: minimal
+      - name: Clean space as per https://github.com/actions/virtual-environments/issues/709
+        run: |
+          sudo rm -rf "/opt/ghc" || true
+          sudo rm -rf "/usr/share/dotnet" || true
+          sudo rm -rf "/usr/local/lib/android" || true
+          sudo rm -rf "/usr/local/share/boost" || true
+      - uses: dtolnay/rust-toolchain@1.91.1
       - name: Install sd
         run: cargo install sd
       - name: Update Cargo.toml file

BENCHMARKS.md

@@ -124,6 +124,7 @@ They are JSON files with the following structure (comments are not actually supp
 {
   // Name of the workload. Must be unique to the workload, as it will be used to group results on the dashboard.
   "name": "hackernews.ndjson_1M,no-threads",
+  "type": "bench",
   // Number of consecutive runs of the commands that should be performed.
   // Each run uses a fresh instance of Meilisearch and a fresh database.
   // Each run produces its own report file.

Cargo.lock

@@ -580,7 +580,7 @@ source = "git+https://github.com/meilisearch/bbqueue#e8af4a4bccc8eb36b2b0442c4a9
 
 [[package]]
 name = "benchmarks"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "anyhow",
  "bumpalo",
@@ -790,11 +790,11 @@ dependencies = [
 
 [[package]]
 name = "build-info"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "anyhow",
  "time",
- "vergen-git2",
+ "vergen-gitcl",
 ]
 
 [[package]]
@@ -1786,7 +1786,7 @@ dependencies = [
 
 [[package]]
 name = "dump"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "anyhow",
  "big_s",
@@ -2018,7 +2018,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
 
 [[package]]
 name = "file-store"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "tempfile",
  "thiserror 2.0.17",
@@ -2040,7 +2040,7 @@ dependencies = [
 
 [[package]]
 name = "filter-parser"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "insta",
  "levenshtein_automata",
@@ -2068,7 +2068,7 @@ dependencies = [
 
 [[package]]
 name = "flatten-serde-json"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "criterion",
  "serde_json",
@@ -2231,7 +2231,7 @@ dependencies = [
 
 [[package]]
 name = "fuzzers"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "arbitrary",
  "bumpalo",
@@ -2604,19 +2604,6 @@ version = "0.32.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
 
-[[package]]
-name = "git2"
-version = "0.20.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2deb07a133b1520dc1a5690e9bd08950108873d7ed5de38dcc74d3b5ebffa110"
-dependencies = [
- "bitflags 2.10.0",
- "libc",
- "libgit2-sys",
- "log",
- "url",
-]
-
 [[package]]
 name = "glob"
 version = "0.3.3"
@@ -2711,9 +2698,9 @@ dependencies = [
 
 [[package]]
 name = "hannoy"
-version = "0.0.9-nested-rtxns-2"
+version = "0.1.0-nested-rtxns"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "06eda090938d9dcd568c8c2a5de383047ed9191578ebf4a342d2975d16e621f2"
+checksum = "be82bf3f2108ddc8885e3d306fcd7f4692066bfe26065ca8b42ba417f3c26dd1"
 dependencies = [
  "bytemuck",
  "byteorder",
@@ -3198,7 +3185,7 @@ dependencies = [
 
 [[package]]
 name = "index-scheduler"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "anyhow",
  "backoff",
@@ -3460,7 +3447,7 @@ dependencies = [
 
 [[package]]
 name = "json-depth-checker"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "criterion",
  "serde_json",
@@ -3557,18 +3544,6 @@ dependencies = [
  "rle-decode-fast",
 ]
 
-[[package]]
-name = "libgit2-sys"
-version = "0.18.2+1.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1c42fe03df2bd3c53a3a9c7317ad91d80c81cd1fb0caec8d7cc4cd2bfa10c222"
-dependencies = [
- "cc",
- "libc",
- "libz-sys",
- "pkg-config",
-]
-
 [[package]]
 name = "libloading"
 version = "0.8.9"
@@ -3626,18 +3601,6 @@ dependencies = [
  "zlib-rs",
 ]
 
-[[package]]
-name = "libz-sys"
-version = "1.1.22"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b70e7a7df205e92a1a4cd9aaae7898dac0aa555503cc0a649494d0d60e7651d"
-dependencies = [
- "cc",
- "libc",
- "pkg-config",
- "vcpkg",
-]
-
 [[package]]
 name = "lindera"
 version = "0.43.3"
@@ -3974,7 +3937,7 @@ checksum = "ae960838283323069879657ca3de837e9f7bbb4c7bf6ea7f1b290d5e9476d2e0"
 
 [[package]]
 name = "meili-snap"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "insta",
  "md5 0.8.0",
@@ -3985,7 +3948,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "actix-cors",
  "actix-http",
@@ -4083,7 +4046,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-auth"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "base64 0.22.1",
  "enum-iterator",
@@ -4102,7 +4065,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-types"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "actix-web",
  "anyhow",
@@ -4137,7 +4100,7 @@ dependencies = [
 
 [[package]]
 name = "meilitool"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "anyhow",
  "clap",
@@ -4171,7 +4134,7 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "arroy",
  "bbqueue",
@@ -4750,7 +4713,7 @@ checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
 
 [[package]]
 name = "permissive-json-pointer"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "big_s",
  "serde_json",
@@ -6072,6 +6035,20 @@ name = "similar"
 version = "2.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
+dependencies = [
+ "bstr",
+ "unicode-segmentation",
+]
+
+[[package]]
+name = "similar-asserts"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b5b441962c817e33508847a22bd82f03a30cff43642dc2fae8b050566121eb9a"
+dependencies = [
+ "console",
+ "similar",
+]
 
 [[package]]
 name = "simple_asn1"
@@ -7105,12 +7082,6 @@ version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
 
-[[package]]
-name = "vcpkg"
-version = "0.2.15"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
-
 [[package]]
 name = "vergen"
 version = "9.0.6"
@@ -7124,14 +7095,13 @@ dependencies = [
 ]
 
 [[package]]
-name = "vergen-git2"
-version = "1.0.7"
+name = "vergen-gitcl"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4f6ee511ec45098eabade8a0750e76eec671e7fb2d9360c563911336bea9cac1"
+checksum = "b9dfc1de6eb2e08a4ddf152f1b179529638bedc0ea95e6d667c014506377aefe"
 dependencies = [
  "anyhow",
  "derive_builder",
- "git2",
  "rustversion",
  "time",
  "vergen",
@@ -7783,7 +7753,7 @@ dependencies = [
 
 [[package]]
 name = "xtask"
-version = "1.28.1"
+version = "1.29.0"
 dependencies = [
  "anyhow",
  "build-info",
@@ -7792,9 +7762,11 @@ dependencies = [
  "futures-core",
  "futures-util",
  "reqwest",
+ "semver",
  "serde",
  "serde_json",
  "sha2",
+ "similar-asserts",
  "sysinfo",
  "time",
  "tokio",

Cargo.toml

@@ -23,7 +23,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.28.1"
+version = "1.29.0"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",

TESTING.md

@@ -0,0 +1,326 @@
+# Declarative tests
+
+Declarative tests ensure that Meilisearch features remain stable across versions.
+
+While we already have unit tests, those are run against **temporary databases** that are created fresh each time and therefore never risk corruption.
+
+Declarative tests instead **simulate the lifetime of a database**: they chain together commands and requests to change the binary, verifying that database state and API responses remain consistent.
+
+## Basic example
+
+```jsonc
+{
+  "type": "test",
+  "name": "api-keys",
+  "binary": { // the first command will run on the binary following this specification.
+    "source": "release", // get the binary as a release from GitHub
+    "version": "1.19.0", // version to fetch
+    "edition": "community" // edition to fetch
+  },
+  "commands": []
+}
+```
+
+This example defines a no-op test (it does nothing).
+
+If the file is saved at `workloads/tests/example.json`, you can run it with:
+
+```bash
+cargo xtask test workloads/tests/example.json
+```
+
+## Commands
+
+Commands represent API requests sent to Meilisearch endpoints during a test.
+
+They are executed sequentially, and their responses can be validated to ensure consistent behavior across upgrades.
+
+```jsonc
+
+{
+  "route": "keys",
+  "method": "POST",
+  "body": {
+    "inline": {
+      "actions": [
+        "search",
+        "documents.add"
+      ],
+      "description": "Test API Key",
+      "expiresAt": null,
+      "indexes": [ "movies" ]
+    }
+  }
+}
+```
+
+This command issues a `POST /keys` request, creating an API key with permissions to search and add documents in the `movies` index.
+
+### Using assets in commands
+
+To keep tests concise and reusable, you can define **assets** at the root of the workload file.
+
+Assets are external data sources (such as datasets) that are cached between runs, making tests faster and easier to read.
+
+```jsonc
+{
+  "type": "test",
+  "name": "movies",
+  "binary": {
+    "source": "release",
+    "version": "1.19.0",
+    "edition": "community"
+  },
+  "assets": {
+    "movies.json": {
+      "local_location": null,
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/movies.json",
+      "sha256": "5b6e4cb660bc20327776e8a33ea197b43d9ec84856710ead1cc87ab24df77de1"
+    }
+  },
+  "commands": [
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "asset": "movies.json"
+      }
+    }
+  ]
+}
+```
+
+In this example:
+- The `movies.json` dataset is defined as an asset, pointing to a remote URL.
+- The SHA-256 checksum ensures integrity.
+- The `POST /indexes/movies/documents` command uses this asset as the request body.
+
+This makes the test much cleaner than inlining a large dataset directly into the command.
+
+For asset handling, please refer to the [declarative benchmarks documentation](/BENCHMARKS.md#adding-new-assets).
+
+### Asserting responses
+
+Commands can specify both the **expected status code** and the **expected response body**.
+
+```jsonc
+{
+  "route": "indexes/movies/documents",
+  "method": "POST",
+  "body": {
+    "asset": "movies.json"
+  },
+  "expectedStatus": 202,
+  "expectedResponse": {
+    "enqueuedAt": "[timestamp]", // Set to a bracketed string to ignore the value
+    "indexUid": "movies",
+    "status": "enqueued",
+    "taskUid": 1,
+    "type": "documentAdditionOrUpdate"
+  },
+  "synchronous": "WaitForTask"
+}
+```
+
+Manually writing `expectedResponse` fields can be tedious.
+
+Instead, you can let the test runner populate them automatically:
+
+```bash
+# Run the workload to populate expected fields. Only adds the missing ones, doesn't change existing data
+cargo xtask test workloads/tests/example.json --add-missing-responses
+
+# OR
+
+# Run the workload to populate expected fields. Updates all fields including existing ones
+cargo xtask test workloads/tests/example.json --update-responses
+```
+
+This workflow is recommended:
+
+1. Write the test without expected fields.
+2. Run it with `--add-missing-responses` to capture the actual responses.
+3. Review and commit the generated expectations.
+
+## Changing binary
+
+It is possible to insert an instruction to change the current Meilisearch instance from one binary specification to another during a test.
+
+When executed, such an instruction will:
+1. Stop the current Meilisearch instance.
+2. Fetch the binary specified by the instruction.
+3. Restart the server with the specified binary on the same database.
+
+```jsonc
+{
+  "type": "test",
+  "name": "movies",
+  "binary": {
+    "source": "release",
+    "version": "1.19.0", // start with version v1.19.0
+    "edition": "community"
+  },
+  "assets": {
+    "movies.json": {
+      "local_location": null,
+      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/movies.json",
+      "sha256": "5b6e4cb660bc20327776e8a33ea197b43d9ec84856710ead1cc87ab24df77de1"
+    }
+  },
+  "commands": [
+    // setup some data
+    {
+      "route": "indexes/movies/documents",
+      "method": "POST",
+      "body": {
+        "asset": "movies.json"
+      }
+    },
+    // switch binary to v1.24.0
+    {
+      "binary": {
+        "source": "release",
+        "version": "1.24.0",
+        "edition": "community"
+      }
+    }
+  ]
+}
+```
+
+### Typical Usage
+
+In most cases, the change binary instruction will be used to update a database.
+
+- **Set up** some data using commands on an older version.
+- **Upgrade** to the latest version.
+- **Assert** that the data and API behavior remain correct after the upgrade.
+
+To properly test the dumpless upgrade, one should typically:
+
+1. Open the database without processing the update task: Use a `binary` instruction to switch to the desired version, passing `--experimental-dumpless-upgrade` and `--experimental-max-number-of-batched-tasks=0` as extra CLI arguments
+2. Check that the search, stats and task queue still work.
+3. Open the database and process the update task: Use a `binary` instruction to switch to the desired version, passing `--experimental-dumpless-upgrade` as the extra CLI argument. Use a `health` command to wait for the upgrade task to finish.
+4. Check that the indexing, search, stats, and task queue still work.
+
+```jsonc
+{
+  "type": "test",
+  "name": "movies",
+  "binary": {
+    "source": "release",
+    "version": "1.12.0",
+    "edition": "community"
+  },
+  "commands": [
+    // 0. Run commands to populate the database
+    {
+      // ..
+    },
+    // 1. Open the database with new MS without processing the update task
+    {
+      "binary": {
+        "source": "build", // build the binary from the sources in the current git repository
+        "edition": "community",
+        "extraCliArgs": [
+          "--experimental-dumpless-upgrade", // allows to open with a newer MS
+          "--experimental-max-number-of-batched-tasks=0" // prevent processing of the update task
+        ]
+      }
+    },
+    // 2. Check the search etc.
+    {
+      // ..
+    },
+    // 3. Open the database with new MS and processing the update task
+    {
+      "binary": {
+        "source": "build", // build the binary from the sources in the current git repository
+        "edition": "community",
+        "extraCliArgs": [
+          "--experimental-dumpless-upgrade" // allows to open with a newer MS
+          // no `--experimental-max-number-of-batched-tasks=0`
+        ]
+      }
+    },
+    // 4. Check the indexing, search, etc.
+    {
+      // ..
+    }
+  ]
+}
+```
+
+This ensures backward compatibility: databases created with older Meilisearch versions should remain functional and consistent after an upgrade.
+
+## Variables
+
+Sometimes a command needs to use a value returned by a **previous response**.
+These values can be captured and reused using the register field.
+
+```jsonc
+{
+  "route": "keys",
+  "method": "POST",
+  "body": {
+    "inline": {
+        "actions": [
+        "search",
+        "documents.add"
+        ],
+        "description": "Test API Key",
+        "expiresAt": null,
+        "indexes": [ "movies" ]
+    }
+  },
+  "expectedResponse": {
+      "key": "c6f64630bad2996b1f675007c8800168e14adf5d6a7bb1a400a6d2b158050eaf",
+      // ...
+  },
+  "register": {
+    "key": "/key"
+  },
+  "synchronous": "WaitForResponse"
+}
+```
+
+The `register` field captures the value at the JSON path `/key` from the response.
+Paths follow the **JavaScript Object Notation Pointer (RFC 6901)** format.
+Registered variables are available for all subsequent commands.
+
+Registered variables can be referenced by wrapping their name in double curly braces:
+
+In the route/path:
+
+```jsonc
+{
+  "route": "tasks/{{ task_id }}",
+  "method": "GET"
+}
+```
+
+In the request body:
+
+```jsonc
+{
+  "route": "indexes/movies/documents",
+  "method": "PATCH",
+  "body": {
+    "inline": {
+      "id": "{{ document_id }}",
+      "overview": "Shazam turns evil and the world is in danger.",
+    }
+  }
+}
+```
+
+Or they can be referenced by their name (**without curly braces**) as an API key:
+
+```jsonc
+{
+  "route": "indexes/movies/documents",
+  "method": "POST",
+  "body": { /* ... */ },
+  "apiKeyVariable": "key" // The **content** of the key variable will be used as an API key
+}
+```

crates/benchmarks/benches/indexing.rs

@@ -21,6 +21,10 @@ use roaring::RoaringBitmap;
 #[global_allocator]
 static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
+fn no_cancel() -> bool {
+    false
+}
+
 const BENCHMARK_ITERATION: usize = 10;
 
 fn setup_dir(path: impl AsRef<Path>) {
@@ -65,7 +69,7 @@ fn setup_settings<'t>(
     let sortable_fields = sortable_fields.iter().map(|s| s.to_string()).collect();
     builder.set_sortable_fields(sortable_fields);
 
-    builder.execute(&|| false, &Progress::default(), Default::default()).unwrap();
+    builder.execute(&no_cancel, &Progress::default(), Default::default()).unwrap();
 }
 
 fn setup_index_with_settings(
@@ -152,7 +156,7 @@ fn indexing_songs_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -168,7 +172,7 @@ fn indexing_songs_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -220,7 +224,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -236,7 +240,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -266,7 +270,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -282,7 +286,7 @@ fn reindexing_songs_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -336,7 +340,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -352,7 +356,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -414,7 +418,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -430,7 +434,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -460,7 +464,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -476,7 +480,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -502,7 +506,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -518,7 +522,7 @@ fn indexing_songs_in_three_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -571,7 +575,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -587,7 +591,7 @@ fn indexing_songs_without_faceted_numbers(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -639,7 +643,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -655,7 +659,7 @@ fn indexing_songs_without_faceted_fields(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -707,7 +711,7 @@ fn indexing_wiki(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -723,7 +727,7 @@ fn indexing_wiki(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -774,7 +778,7 @@ fn reindexing_wiki(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -790,7 +794,7 @@ fn reindexing_wiki(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -820,7 +824,7 @@ fn reindexing_wiki(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -836,7 +840,7 @@ fn reindexing_wiki(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -889,7 +893,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -905,7 +909,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -967,7 +971,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -983,7 +987,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1014,7 +1018,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1030,7 +1034,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1057,7 +1061,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1073,7 +1077,7 @@ fn indexing_wiki_in_three_batches(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1125,7 +1129,7 @@ fn indexing_movies_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1141,7 +1145,7 @@ fn indexing_movies_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1192,7 +1196,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1208,7 +1212,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1238,7 +1242,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1254,7 +1258,7 @@ fn reindexing_movies_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1307,7 +1311,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1323,7 +1327,7 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1372,7 +1376,7 @@ fn delete_documents_from_ids(index: Index, document_ids_to_delete: Vec<RoaringBi
             Some(primary_key),
             &document_changes,
             RuntimeEmbedders::default(),
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -1422,7 +1426,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1438,7 +1442,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1468,7 +1472,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1484,7 +1488,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1510,7 +1514,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1526,7 +1530,7 @@ fn indexing_movies_in_three_batches(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1601,7 +1605,7 @@ fn indexing_nested_movies_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1617,7 +1621,7 @@ fn indexing_nested_movies_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1693,7 +1697,7 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1709,7 +1713,7 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1777,7 +1781,7 @@ fn indexing_nested_movies_without_faceted_fields(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1793,7 +1797,7 @@ fn indexing_nested_movies_without_faceted_fields(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1845,7 +1849,7 @@ fn indexing_geo(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1861,7 +1865,7 @@ fn indexing_geo(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1912,7 +1916,7 @@ fn reindexing_geo(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1928,7 +1932,7 @@ fn reindexing_geo(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -1958,7 +1962,7 @@ fn reindexing_geo(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -1974,7 +1978,7 @@ fn reindexing_geo(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )
@@ -2027,7 +2031,7 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) {
                         &rtxn,
                         None,
                         &mut new_fields_ids_map,
-                        &|| false,
+                        &no_cancel,
                         Progress::default(),
                         None,
                     )
@@ -2043,7 +2047,7 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) {
                     primary_key,
                     &document_changes,
                     RuntimeEmbedders::default(),
-                    &|| false,
+                    &no_cancel,
                     &Progress::default(),
                     &Default::default(),
                 )

crates/build-info/Cargo.toml

@@ -15,4 +15,4 @@ time = { version = "0.3.44", features = ["parsing"] }
 
 [build-dependencies]
 anyhow = "1.0.100"
-vergen-git2 = "1.0.7"
+vergen-gitcl = "1.0.8"

crates/build-info/build.rs

@@ -15,7 +15,7 @@ fn emit_git_variables() -> anyhow::Result<()> {
     // Note: any code that needs VERGEN_ environment variables should take care to define them manually in the Dockerfile and pass them
     // in the corresponding GitHub workflow (publish_docker.yml).
     // This is due to the Dockerfile building the binary outside of the git directory.
-    let mut builder = vergen_git2::Git2Builder::default();
+    let mut builder = vergen_gitcl::GitclBuilder::default();
 
     builder.branch(true);
     builder.commit_timestamp(true);
@@ -25,5 +25,5 @@ fn emit_git_variables() -> anyhow::Result<()> {
 
     let git2 = builder.build()?;
 
-    vergen_git2::Emitter::default().fail_on_error().add_instructions(&git2)?.emit()
+    vergen_gitcl::Emitter::default().fail_on_error().add_instructions(&git2)?.emit()
 }

crates/build-info/src/main.rs

@@ -0,0 +1,6 @@
+use build_info::BuildInfo;
+
+fn main() {
+    let info = BuildInfo::from_build();
+    dbg!(info);
+}

crates/dump/src/reader/v2/settings.rs

@@ -107,19 +107,14 @@ impl Settings<Unchecked> {
     }
 }
 
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Default, Debug, Clone, PartialEq)]
 pub enum Setting<T> {
     Set(T),
     Reset,
+    #[default]
     NotSet,
 }
 
-impl<T> Default for Setting<T> {
-    fn default() -> Self {
-        Self::NotSet
-    }
-}
-
 impl<T> Setting<T> {
     pub const fn is_not_set(&self) -> bool {
         matches!(self, Self::NotSet)

crates/dump/src/reader/v3/settings.rs

@@ -161,19 +161,14 @@ pub struct Facets {
     pub min_level_size: Option<NonZeroUsize>,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Default, Debug, Clone, PartialEq, Eq)]
 pub enum Setting<T> {
     Set(T),
     Reset,
+    #[default]
     NotSet,
 }
 
-impl<T> Default for Setting<T> {
-    fn default() -> Self {
-        Self::NotSet
-    }
-}
-
 impl<T> Setting<T> {
     pub fn map<U, F>(self, f: F) -> Setting<U>
     where

crates/dump/src/reader/v4/meta.rs

@@ -1,9 +1,7 @@
 use std::fmt::{self, Display, Formatter};
-use std::marker::PhantomData;
 use std::str::FromStr;
 
-use serde::de::Visitor;
-use serde::{Deserialize, Deserializer};
+use serde::Deserialize;
 use uuid::Uuid;
 
 use super::settings::{Settings, Unchecked};
@@ -82,59 +80,3 @@ impl Display for IndexUidFormatError {
 }
 
 impl std::error::Error for IndexUidFormatError {}
-
-/// A type that tries to match either a star (*) or
-/// any other thing that implements `FromStr`.
-#[derive(Debug)]
-#[cfg_attr(test, derive(serde::Serialize))]
-pub enum StarOr<T> {
-    Star,
-    Other(T),
-}
-
-impl<'de, T, E> Deserialize<'de> for StarOr<T>
-where
-    T: FromStr<Err = E>,
-    E: Display,
-{
-    fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
-    where
-        D: Deserializer<'de>,
-    {
-        /// Serde can't differentiate between `StarOr::Star` and `StarOr::Other` without a tag.
-        /// Simply using `#[serde(untagged)]` + `#[serde(rename="*")]` will lead to attempting to
-        /// deserialize everything as a `StarOr::Other`, including "*".
-        /// [`#[serde(other)]`](https://serde.rs/variant-attrs.html#other) might have helped but is
-        /// not supported on untagged enums.
-        struct StarOrVisitor<T>(PhantomData<T>);
-
-        impl<T, FE> Visitor<'_> for StarOrVisitor<T>
-        where
-            T: FromStr<Err = FE>,
-            FE: Display,
-        {
-            type Value = StarOr<T>;
-
-            fn expecting(&self, formatter: &mut Formatter) -> std::fmt::Result {
-                formatter.write_str("a string")
-            }
-
-            fn visit_str<SE>(self, v: &str) -> Result<Self::Value, SE>
-            where
-                SE: serde::de::Error,
-            {
-                match v {
-                    "*" => Ok(StarOr::Star),
-                    v => {
-                        let other = FromStr::from_str(v).map_err(|e: T::Err| {
-                            SE::custom(format!("Invalid `other` value: {}", e))
-                        })?;
-                        Ok(StarOr::Other(other))
-                    }
-                }
-            }
-        }
-
-        deserializer.deserialize_str(StarOrVisitor(PhantomData))
-    }
-}

crates/dump/src/reader/v4/settings.rs

@@ -192,19 +192,14 @@ pub struct Facets {
     pub min_level_size: Option<NonZeroUsize>,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Copy)]
+#[derive(Default, Debug, Clone, PartialEq, Eq, Copy)]
 pub enum Setting<T> {
     Set(T),
     Reset,
+    #[default]
     NotSet,
 }
 
-impl<T> Default for Setting<T> {
-    fn default() -> Self {
-        Self::NotSet
-    }
-}
-
 impl<T> Setting<T> {
     pub fn set(self) -> Option<T> {
         match self {

crates/dump/src/reader/v5/settings.rs

@@ -47,20 +47,15 @@ pub struct Settings<T> {
     pub _kind: PhantomData<T>,
 }
 
-#[derive(Debug, Clone, PartialEq, Eq, Copy)]
+#[derive(Default, Debug, Clone, PartialEq, Eq, Copy)]
 #[cfg_attr(test, derive(serde::Serialize))]
 pub enum Setting<T> {
     Set(T),
     Reset,
+    #[default]
     NotSet,
 }
 
-impl<T> Default for Setting<T> {
-    fn default() -> Self {
-        Self::NotSet
-    }
-}
-
 impl<T> Setting<T> {
     pub fn set(self) -> Option<T> {
         match self {

crates/dump/src/reader/v5/tasks.rs

@@ -322,7 +322,7 @@ impl From<Task> for TaskView {
             _ => None,
         });
 
-        let duration = finished_at.zip(started_at).map(|(tf, ts)| (tf - ts));
+        let duration = finished_at.zip(started_at).map(|(tf, ts)| tf - ts);
 
         Self {
             uid: id,

crates/index-scheduler/src/insta_snapshot.rs

@@ -6,7 +6,7 @@ use meilisearch_types::heed::types::{SerdeBincode, SerdeJson, Str};
 use meilisearch_types::heed::{Database, RoTxn};
 use meilisearch_types::milli::{CboRoaringBitmapCodec, RoaringBitmapCodec, BEU32};
 use meilisearch_types::tasks::{Details, Kind, Status, Task};
-use meilisearch_types::versioning;
+use meilisearch_types::versioning::{self, VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
 use roaring::RoaringBitmap;
 
 use crate::index_mapper::IndexMapper;
@@ -320,8 +320,12 @@ fn snapshot_details(d: &Details) -> String {
             format!("{{ url: {url:?}, api_key: {api_key:?}, payload_size: {payload_size:?}, indexes: {indexes:?} }}")
         }
         Details::UpgradeDatabase { from, to } => {
+            if to == &(VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) {
+                format!("{{ from: {from:?}, to: [current version] }}")
+            } else {
                 format!("{{ from: {from:?}, to: {to:?} }}")
             }
+        }
         Details::IndexCompaction { index_uid, pre_compaction_size, post_compaction_size } => {
             format!("{{ index_uid: {index_uid:?}, pre_compaction_size: {pre_compaction_size:?}, post_compaction_size: {post_compaction_size:?} }}")
         }
@@ -400,7 +404,21 @@ pub fn snapshot_batch(batch: &Batch) -> String {
 
     snap.push('{');
     snap.push_str(&format!("uid: {uid}, "));
-    snap.push_str(&format!("details: {}, ", serde_json::to_string(details).unwrap()));
+    let details = if let Some(upgrade_to) = &details.upgrade_to {
+        if upgrade_to.as_str()
+            == format!("v{VERSION_MAJOR}.{VERSION_MINOR}.{VERSION_PATCH}").as_str()
+        {
+            let mut details = details.clone();
+
+            details.upgrade_to = Some("[current version]".into());
+            serde_json::to_string(&details).unwrap()
+        } else {
+            serde_json::to_string(details).unwrap()
+        }
+    } else {
+        serde_json::to_string(details).unwrap()
+    };
+    snap.push_str(&format!("details: {details}, "));
     snap.push_str(&format!("stats: {}, ", serde_json::to_string(&stats).unwrap()));
     if !embedder_stats.skip_serializing() {
         snap.push_str(&format!(

crates/index-scheduler/src/queue/batches.rs

@@ -502,13 +502,11 @@ impl Queue {
             *before_finished_at,
         )?;
 
-        if let Some(limit) = limit {
         batches = if query.reverse.unwrap_or_default() {
-                batches.into_iter().take(*limit as usize).collect()
+            batches.into_iter().take(*limit).collect()
         } else {
-                batches.into_iter().rev().take(*limit as usize).collect()
+            batches.into_iter().rev().take(*limit).collect()
         };
-        }
 
         Ok(batches)
     }
@@ -602,11 +600,8 @@ impl Queue {
             Box::new(batches.into_iter().rev()) as Box<dyn Iterator<Item = u32>>
         };
 
-        let batches = self.batches.get_existing_batches(
-            rtxn,
-            batches.take(query.limit.unwrap_or(u32::MAX) as usize),
-            processing,
-        )?;
+        let batches =
+            self.batches.get_existing_batches(rtxn, batches.take(query.limit), processing)?;
 
         Ok((batches, total))
     }

crates/index-scheduler/src/queue/batches_test.rs

@@ -28,21 +28,21 @@ fn query_batches_from_and_limit() {
 
     let proc = index_scheduler.processing_tasks.read().unwrap().clone();
     let rtxn = index_scheduler.env.read_txn().unwrap();
-    let query = Query { limit: Some(0), ..Default::default() };
+    let query = Query { limit: 0, ..Default::default() };
     let (batches, _) = index_scheduler
         .queue
         .get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default(), &proc)
         .unwrap();
     snapshot!(snapshot_bitmap(&batches), @"[]");
 
-    let query = Query { limit: Some(1), ..Default::default() };
+    let query = Query { limit: 1, ..Default::default() };
     let (batches, _) = index_scheduler
         .queue
         .get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default(), &proc)
         .unwrap();
     snapshot!(snapshot_bitmap(&batches), @"[2,]");
 
-    let query = Query { limit: Some(2), ..Default::default() };
+    let query = Query { limit: 2, ..Default::default() };
     let (batches, _) = index_scheduler
         .queue
         .get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default(), &proc)
@@ -63,14 +63,14 @@ fn query_batches_from_and_limit() {
         .unwrap();
     snapshot!(snapshot_bitmap(&batches), @"[0,1,2,]");
 
-    let query = Query { from: Some(1), limit: Some(1), ..Default::default() };
+    let query = Query { from: Some(1), limit: 1, ..Default::default() };
     let (batches, _) = index_scheduler
         .queue
         .get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default(), &proc)
         .unwrap();
     snapshot!(snapshot_bitmap(&batches), @"[1,]");
 
-    let query = Query { from: Some(1), limit: Some(2), ..Default::default() };
+    let query = Query { from: Some(1), limit: 2, ..Default::default() };
     let (batches, _) = index_scheduler
         .queue
         .get_batch_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default(), &proc)

crates/index-scheduler/src/queue/mod.rs

@@ -31,6 +31,9 @@ use crate::{Error, IndexSchedulerOptions, Result, TaskId};
 
 /// The number of database used by queue itself
 const NUMBER_OF_DATABASES: u32 = 1;
+/// The default limit for pagination
+const DEFAULT_LIMIT: usize = 20;
+
 /// Database const names for the `IndexScheduler`.
 mod db_name {
     pub const BATCH_TO_TASKS_MAPPING: &str = "batch-to-tasks-mapping";
@@ -40,11 +43,11 @@ mod db_name {
 ///
 /// An empty/default query (where each field is set to `None`) matches all tasks.
 /// Each non-null field restricts the set of tasks further.
-#[derive(Default, Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Query {
-    /// The maximum number of tasks to be matched
-    pub limit: Option<u32>,
-    /// The minimum [task id](`meilisearch_types::tasks::Task::uid`) to be matched
+    /// The maximum number of tasks to be matched. Defaults to 20.
+    pub limit: usize,
+    /// The minimum [task id](`meilisearch_types::tasks::Task::uid`) to be matched. Defaults to 0.
     pub from: Option<u32>,
     /// The order used to return the tasks. By default the newest tasks are returned first and the boolean is `false`.
     pub reverse: Option<bool>,
@@ -83,32 +86,29 @@ pub struct Query {
     pub after_finished_at: Option<OffsetDateTime>,
 }
 
-impl Query {
-    /// Return `true` if every field of the query is set to `None`, such that the query
-    /// matches all tasks.
-    pub fn is_empty(&self) -> bool {
-        matches!(
-            self,
-            Query {
-                limit: None,
-                from: None,
-                reverse: None,
-                uids: None,
-                batch_uids: None,
-                statuses: None,
-                types: None,
-                index_uids: None,
-                canceled_by: None,
-                before_enqueued_at: None,
-                after_enqueued_at: None,
-                before_started_at: None,
-                after_started_at: None,
-                before_finished_at: None,
-                after_finished_at: None,
+impl Default for Query {
+    fn default() -> Self {
+        Self {
+            limit: DEFAULT_LIMIT,
+            from: Default::default(),
+            reverse: Default::default(),
+            uids: Default::default(),
+            batch_uids: Default::default(),
+            statuses: Default::default(),
+            types: Default::default(),
+            index_uids: Default::default(),
+            canceled_by: Default::default(),
+            before_enqueued_at: Default::default(),
+            after_enqueued_at: Default::default(),
+            before_started_at: Default::default(),
+            after_started_at: Default::default(),
+            before_finished_at: Default::default(),
+            after_finished_at: Default::default(),
+        }
     }
-        )
 }
 
+impl Query {
     /// Add an [index id](meilisearch_types::tasks::Task::index_uid) to the list of permitted indexes.
     pub fn with_index(self, index_uid: String) -> Self {
         let mut index_vec = self.index_uids.unwrap_or_default();
@@ -119,7 +119,7 @@ impl Query {
     // Removes the `from` and `limit` restrictions from the query.
     // Useful to get the total number of tasks matching a filter.
     pub fn without_limits(self) -> Self {
-        Query { limit: None, from: None, ..self }
+        Query { limit: usize::MAX, from: None, ..self }
     }
 }
 

crates/index-scheduler/src/queue/tasks.rs

@@ -465,13 +465,11 @@ impl Queue {
             *before_finished_at,
         )?;
 
-        if let Some(limit) = limit {
         tasks = if query.reverse.unwrap_or_default() {
-                tasks.into_iter().take(*limit as usize).collect()
+            tasks.into_iter().take(*limit).collect()
         } else {
-                tasks.into_iter().rev().take(*limit as usize).collect()
+            tasks.into_iter().rev().take(*limit).collect()
         };
-        }
 
         Ok(tasks)
     }
@@ -529,9 +527,7 @@ impl Queue {
         } else {
             Box::new(tasks.into_iter().rev()) as Box<dyn Iterator<Item = u32>>
         };
-        let tasks = self
-            .tasks
-            .get_existing_tasks(rtxn, tasks.take(query.limit.unwrap_or(u32::MAX) as usize))?;
+        let tasks = self.tasks.get_existing_tasks(rtxn, tasks.take(query.limit))?;
 
         let ProcessingTasks { batch, processing, progress: _ } = processing_tasks;
 

crates/index-scheduler/src/queue/tasks_test.rs

@@ -28,21 +28,21 @@ fn query_tasks_from_and_limit() {
 
     let rtxn = index_scheduler.env.read_txn().unwrap();
     let processing = index_scheduler.processing_tasks.read().unwrap();
-    let query = Query { limit: Some(0), ..Default::default() };
+    let query = Query { limit: 0, ..Default::default() };
     let (tasks, _) = index_scheduler
         .queue
         .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default(), &processing)
         .unwrap();
     snapshot!(snapshot_bitmap(&tasks), @"[]");
 
-    let query = Query { limit: Some(1), ..Default::default() };
+    let query = Query { limit: 1, ..Default::default() };
     let (tasks, _) = index_scheduler
         .queue
         .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default(), &processing)
         .unwrap();
     snapshot!(snapshot_bitmap(&tasks), @"[2,]");
 
-    let query = Query { limit: Some(2), ..Default::default() };
+    let query = Query { limit: 2, ..Default::default() };
     let (tasks, _) = index_scheduler
         .queue
         .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default(), &processing)
@@ -63,14 +63,14 @@ fn query_tasks_from_and_limit() {
         .unwrap();
     snapshot!(snapshot_bitmap(&tasks), @"[0,1,2,]");
 
-    let query = Query { from: Some(1), limit: Some(1), ..Default::default() };
+    let query = Query { from: Some(1), limit: 1, ..Default::default() };
     let (tasks, _) = index_scheduler
         .queue
         .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default(), &processing)
         .unwrap();
     snapshot!(snapshot_bitmap(&tasks), @"[1,]");
 
-    let query = Query { from: Some(1), limit: Some(2), ..Default::default() };
+    let query = Query { from: Some(1), limit: 2, ..Default::default() };
     let (tasks, _) = index_scheduler
         .queue
         .get_task_ids_from_authorized_indexes(&rtxn, &query, &AuthFilter::default(), &processing)

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/after_processing_everything.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 28, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, batch_uid: 1, status: succeeded, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, batch_uid: 2, status: succeeded, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 3 {uid: 3, batch_uid: 3, status: failed, error: ResponseError { code: 200, message: "Index `doggo` already exists.", error_code: "index_already_exists", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#index_already_exists" }, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -57,7 +57,7 @@ girafo: { number_of_documents: 0, field_distribution: {} }
 [timestamp] [4,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.28.1"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"[current version]"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 1 {uid: 1, details: {"primaryKey":"mouse"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"catto":1}}, stop reason: "created batch containing only task with id 1 of type `indexCreation` that cannot be batched with any other task.", }
 2 {uid: 2, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 2 of type `indexCreation` that cannot be batched with any other task.", }
 3 {uid: 3, details: {"primaryKey":"bone"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"indexCreation":1},"indexUids":{"doggo":1}}, stop reason: "created batch containing only task with id 3 of type `indexCreation` that cannot be batched with any other task.", }

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/register_automatic_upgrade_task.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 28, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/registered_a_task_while_the_upgrade_task_is_enqueued.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: (1, 28, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, status: enqueued, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 ----------------------------------------------------------------------
 ### Status:

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 28, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 ----------------------------------------------------------------------
 ### Status:
@@ -37,7 +37,7 @@ catto [1,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.28.1"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"[current version]"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_failed_again.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: (1, 28, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: failed, error: ResponseError { code: 200, message: "Planned failure for tests.", error_code: "internal", error_type: "internal", error_link: "https://docs.meilisearch.com/errors#internal" }, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 ----------------------------------------------------------------------
@@ -40,7 +40,7 @@ doggo [2,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.28.1"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"[current version]"}, stats: {"totalNbTasks":1,"status":{"failed":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/scheduler/snapshots/test_failure.rs/upgrade_failure/upgrade_task_succeeded.snap

@@ -6,7 +6,7 @@ source: crates/index-scheduler/src/scheduler/test_failure.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: (1, 28, 1) }, kind: UpgradeDatabase { from: (1, 12, 0) }}
+0 {uid: 0, batch_uid: 0, status: succeeded, details: { from: (1, 12, 0), to: [current version] }, kind: UpgradeDatabase { from: (1, 12, 0) }}
 1 {uid: 1, status: enqueued, details: { primary_key: Some("mouse"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "catto", primary_key: Some("mouse") }}
 2 {uid: 2, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
 3 {uid: 3, status: enqueued, details: { primary_key: Some("bone"), old_new_uid: None, new_index_uid: None }, kind: IndexCreation { index_uid: "doggo", primary_key: Some("bone") }}
@@ -43,7 +43,7 @@ doggo [2,3,]
 [timestamp] [0,]
 ----------------------------------------------------------------------
 ### All Batches:
-0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"v1.28.1"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
+0 {uid: 0, details: {"upgradeFrom":"v1.12.0","upgradeTo":"[current version]"}, stats: {"totalNbTasks":1,"status":{"succeeded":1},"types":{"upgradeDatabase":1},"indexUids":{}}, stop reason: "stopped after the last task of type `upgradeDatabase` because they cannot be batched with tasks of any other type.", }
 ----------------------------------------------------------------------
 ### Batch to tasks mapping:
 0 [0,]

crates/index-scheduler/src/upgrade/mod.rs

@@ -1,7 +1,7 @@
 use anyhow::bail;
 use meilisearch_types::heed::{Env, RwTxn, WithoutTls};
 use meilisearch_types::tasks::{Details, KindWithContent, Status, Task};
-use meilisearch_types::versioning::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
+use meilisearch_types::versioning;
 use time::OffsetDateTime;
 use tracing::info;
 
@@ -9,83 +9,82 @@ use crate::queue::TaskQueue;
 use crate::versioning::Versioning;
 
 trait UpgradeIndexScheduler {
-    fn upgrade(
-        &self,
-        env: &Env<WithoutTls>,
-        wtxn: &mut RwTxn,
-        original: (u32, u32, u32),
-    ) -> anyhow::Result<()>;
-    fn target_version(&self) -> (u32, u32, u32);
+    fn upgrade(&self, env: &Env<WithoutTls>, wtxn: &mut RwTxn) -> anyhow::Result<()>;
+    /// Whether the migration should be applied, depending on the initial version of the index scheduler before
+    /// any migration was applied
+    fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool;
+    /// A progress-centric description of the migration
+    fn description(&self) -> &'static str;
 }
 
+/// Upgrade the index scheduler to the binary version.
+///
+/// # Warning
+///
+/// The current implementation uses a single wtxn to the index scheduler for the whole duration of the upgrade.
+/// If migrations start taking take a long time, it might prevent tasks from being registered.
+/// If this issue manifests, then it can be mitigated by adding a `fn target_version` to `UpgradeIndexScheduler`,
+/// to be able to write intermediate versions and drop the wtxn between applying migrations.
 pub fn upgrade_index_scheduler(
     env: &Env<WithoutTls>,
     versioning: &Versioning,
-    from: (u32, u32, u32),
-    to: (u32, u32, u32),
+    initial_version: (u32, u32, u32),
 ) -> anyhow::Result<()> {
-    let current_major = to.0;
-    let current_minor = to.1;
-    let current_patch = to.2;
+    let target_major: u32 = versioning::VERSION_MAJOR;
+    let target_minor: u32 = versioning::VERSION_MINOR;
+    let target_patch: u32 = versioning::VERSION_PATCH;
+    let target_version = (target_major, target_minor, target_patch);
+
+    if initial_version == target_version {
+        return Ok(());
+    }
 
     let upgrade_functions: &[&dyn UpgradeIndexScheduler] = &[
-        // This is the last upgrade function, it will be called when the index is up to date.
-        // any other upgrade function should be added before this one.
-        &ToCurrentNoOp {},
+        // List all upgrade functions to apply in order here.
     ];
 
-    let start = match from {
-        (1, 12, _) => 0,
-        (1, 13, _) => 0,
-        (1, 14, _) => 0,
-        (1, 15, _) => 0,
-        (1, 16, _) => 0,
-        (1, 17, _) => 0,
-        (1, 18, _) => 0,
-        (1, 19, _) => 0,
-        (1, 20, _) => 0,
-        (1, 21, _) => 0,
-        (1, 22, _) => 0,
-        (1, 23, _) => 0,
-        (1, 24, _) => 0,
-        (1, 25, _) => 0,
-        (1, 26, _) => 0,
-        (1, 27, _) => 0,
-        (1, 28, _) => 0,
-        (major, minor, patch) => {
-            if major > current_major
-                || (major == current_major && minor > current_minor)
-                || (major == current_major && minor == current_minor && patch > current_patch)
-            {
+    let (initial_major, initial_minor, initial_patch) = initial_version;
+
+    if initial_version > target_version {
         bail!(
-                "Database version {major}.{minor}.{patch} is higher than the Meilisearch version {current_major}.{current_minor}.{current_patch}. Downgrade is not supported",
+                "Database version {initial_major}.{initial_minor}.{initial_patch} is higher than the Meilisearch version {target_major}.{target_minor}.{target_patch}. Downgrade is not supported",
             );
-            } else if major < 1 || (major == current_major && minor < 12) {
+    }
+
+    if initial_version < (1, 12, 0) {
         bail!(
-                "Database version {major}.{minor}.{patch} is too old for the experimental dumpless upgrade feature. Please generate a dump using the v{major}.{minor}.{patch} and import it in the v{current_major}.{current_minor}.{current_patch}",
+                "Database version {initial_major}.{initial_minor}.{initial_patch} is too old for the experimental dumpless upgrade feature. Please generate a dump using the v{initial_major}.{initial_minor}.{initial_patch} and import it in the v{target_major}.{target_minor}.{target_patch}",
             );
-            } else {
-                bail!("Unknown database version: v{major}.{minor}.{patch}");
     }
-        }
-    };
 
     info!("Upgrading the task queue");
-    let mut local_from = from;
-    for upgrade in upgrade_functions[start..].iter() {
-        let target = upgrade.target_version();
-        info!(
-            "Upgrading from v{}.{}.{} to v{}.{}.{}",
-            local_from.0, local_from.1, local_from.2, target.0, target.1, target.2
-        );
     let mut wtxn = env.write_txn()?;
-        upgrade.upgrade(env, &mut wtxn, local_from)?;
-        versioning.set_version(&mut wtxn, target)?;
-        wtxn.commit()?;
-        local_from = target;
+    let migration_count = upgrade_functions.len();
+    for (migration_index, upgrade) in upgrade_functions.iter().enumerate() {
+        if upgrade.must_upgrade(initial_version) {
+            info!(
+                "[{migration_index}/{migration_count}]Applying migration: {}",
+                upgrade.description()
+            );
+
+            upgrade.upgrade(env, &mut wtxn)?;
+
+            info!(
+                "[{}/{migration_count}]Migration applied: {}",
+                migration_index + 1,
+                upgrade.description()
+            )
+        } else {
+            info!(
+                "[{migration_index}/{migration_count}]Skipping unnecessary migration: {}",
+                upgrade.description()
+            )
+        }
     }
 
-    let mut wtxn = env.write_txn()?;
+    versioning.set_version(&mut wtxn, target_version)?;
+    info!("Task queue upgraded, spawning the upgrade database task");
+
     let queue = TaskQueue::new(env, &mut wtxn)?;
     let uid = queue.next_task_id(&wtxn)?;
     queue.register(
@@ -98,9 +97,9 @@ pub fn upgrade_index_scheduler(
             finished_at: None,
             error: None,
             canceled_by: None,
-            details: Some(Details::UpgradeDatabase { from, to }),
+            details: Some(Details::UpgradeDatabase { from: initial_version, to: target_version }),
             status: Status::Enqueued,
-            kind: KindWithContent::UpgradeDatabase { from },
+            kind: KindWithContent::UpgradeDatabase { from: initial_version },
             network: None,
             custom_metadata: None,
         },
@@ -109,21 +108,3 @@ pub fn upgrade_index_scheduler(
 
     Ok(())
 }
-
-#[allow(non_camel_case_types)]
-struct ToCurrentNoOp {}
-
-impl UpgradeIndexScheduler for ToCurrentNoOp {
-    fn upgrade(
-        &self,
-        _env: &Env<WithoutTls>,
-        _wtxn: &mut RwTxn,
-        _original: (u32, u32, u32),
-    ) -> anyhow::Result<()> {
-        Ok(())
-    }
-
-    fn target_version(&self) -> (u32, u32, u32) {
-        (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)
-    }
-}

crates/index-scheduler/src/versioning.rs

@@ -64,14 +64,7 @@ impl Versioning {
         };
         wtxn.commit()?;
 
-        let bin_major: u32 = versioning::VERSION_MAJOR;
-        let bin_minor: u32 = versioning::VERSION_MINOR;
-        let bin_patch: u32 = versioning::VERSION_PATCH;
-        let to = (bin_major, bin_minor, bin_patch);
-
-        if from != to {
-            upgrade_index_scheduler(env, &this, from, to)?;
-        }
+        upgrade_index_scheduler(env, &this, from)?;
 
         // Once we reach this point it means the upgrade process, if there was one is entirely finished
         // we can safely say we reached the latest version of the index scheduler

crates/meilisearch-types/src/network.rs

@@ -1,6 +1,7 @@
-use serde::{Deserialize, Serialize};
 use std::collections::BTreeMap;
 
+use serde::{Deserialize, Serialize};
+
 #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Default)]
 #[serde(rename_all = "camelCase")]
 pub struct Network {

crates/meilisearch/src/analytics/segment_analytics.rs

@@ -1,7 +1,7 @@
 use std::any::TypeId;
 use std::collections::{HashMap, HashSet};
 use std::fs;
-use std::path::{Path, PathBuf};
+use std::path::Path;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
@@ -344,14 +344,14 @@ impl Infos {
             experimental_no_edition_2024_for_dumps,
             experimental_vector_store_setting: vector_store_setting,
             gpu_enabled: meilisearch_types::milli::vector::is_cuda_enabled(),
-            db_path: db_path != PathBuf::from("./data.ms"),
+            db_path: db_path != Path::new("./data.ms"),
             import_dump: import_dump.is_some(),
-            dump_dir: dump_dir != PathBuf::from("dumps/"),
+            dump_dir: dump_dir != Path::new("dumps/"),
             ignore_missing_dump,
             ignore_dump_if_db_exists,
             import_snapshot: import_snapshot.is_some(),
             schedule_snapshot,
-            snapshot_dir: snapshot_dir != PathBuf::from("snapshots/"),
+            snapshot_dir: snapshot_dir != Path::new("snapshots/"),
             uses_s3_snapshots: s3_snapshot_options.is_some(),
             ignore_missing_snapshot,
             ignore_snapshot_if_db_exists,

crates/meilisearch/src/routes/metrics.rs

@@ -183,7 +183,11 @@ pub async fn get_metrics(
     crate::metrics::MEILISEARCH_LAST_FINISHED_BATCHES_PROGRESS_TRACE_MS.reset();
     let (batches, _total) = index_scheduler.get_batches_from_authorized_indexes(
         // Fetch the finished batches...
-        &Query { statuses: Some(vec![Status::Succeeded, Status::Failed]), ..Query::default() },
+        &Query {
+            statuses: Some(vec![Status::Succeeded, Status::Failed]),
+            limit: 1,
+            ..Query::default()
+        },
         auth_filters,
     )?;
     // ...and get the last batch only.
@@ -210,7 +214,7 @@ pub async fn get_metrics(
     let task_queue_latency_seconds = index_scheduler
         .get_tasks_from_authorized_indexes(
             &Query {
-                limit: Some(1),
+                limit: 1,
                 reverse: Some(true),
                 statuses: Some(vec![Status::Enqueued, Status::Processing]),
                 ..Query::default()

crates/meilisearch/src/routes/tasks.rs

@@ -126,7 +126,7 @@ pub struct TasksFilterQuery {
 impl TasksFilterQuery {
     pub(crate) fn into_query(self) -> Query {
         Query {
-            limit: Some(self.limit.0),
+            limit: self.limit.0 as usize,
             from: self.from.as_deref().copied(),
             reverse: self.reverse.as_deref().copied(),
             batch_uids: self.batch_uids.merge_star_and_none(),
@@ -225,7 +225,8 @@ pub struct TaskDeletionOrCancelationQuery {
 impl TaskDeletionOrCancelationQuery {
     fn into_query(self) -> Query {
         Query {
-            limit: None,
+            // We want to delete all tasks that match the given filters
+            limit: usize::MAX,
             from: None,
             reverse: None,
             batch_uids: self.batch_uids.merge_star_and_none(),

crates/meilisearch/src/search/mod.rs

@@ -789,11 +789,12 @@ impl TryFrom<Value> for ExternalDocumentId {
     }
 }
 
-#[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, ToSchema, Serialize)]
+#[derive(Default, Debug, Copy, Clone, PartialEq, Eq, Deserr, ToSchema, Serialize)]
 #[deserr(rename_all = camelCase)]
 #[serde(rename_all = "camelCase")]
 pub enum MatchingStrategy {
     /// Remove query words from last to first
+    #[default]
     Last,
     /// All query words are mandatory
     All,
@@ -801,12 +802,6 @@ pub enum MatchingStrategy {
     Frequency,
 }
 
-impl Default for MatchingStrategy {
-    fn default() -> Self {
-        Self::Last
-    }
-}
-
 impl From<MatchingStrategy> for TermsMatchingStrategy {
     fn from(other: MatchingStrategy) -> Self {
         match other {

crates/meilisearch/tests/auth/tenant_token.rs

@@ -187,7 +187,7 @@ macro_rules! compute_forbidden_search {
 
 #[actix_rt::test]
 async fn search_authorized_simple_token() {
-    let tenant_tokens = vec![
+    let tenant_tokens = [
         hashmap! {
             "searchRules" => json!({"*": {}}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
@@ -239,7 +239,7 @@ async fn search_authorized_simple_token() {
 
 #[actix_rt::test]
 async fn search_authorized_filter_token() {
-    let tenant_tokens = vec![
+    let tenant_tokens = [
         hashmap! {
             "searchRules" => json!({"*": {"filter": "color = blue"}}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
@@ -292,7 +292,7 @@ async fn search_authorized_filter_token() {
 
 #[actix_rt::test]
 async fn filter_search_authorized_filter_token() {
-    let tenant_tokens = vec![
+    let tenant_tokens = [
         hashmap! {
             "searchRules" => json!({"*": {"filter": "color = blue"}}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
@@ -353,7 +353,7 @@ async fn filter_search_authorized_filter_token() {
 /// Tests that those Tenant Token are incompatible with the REFUSED_KEYS defined above.
 #[actix_rt::test]
 async fn error_search_token_forbidden_parent_key() {
-    let tenant_tokens = vec![
+    let tenant_tokens = [
         hashmap! {
             "searchRules" => json!({"*": {}}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
@@ -389,7 +389,7 @@ async fn error_search_token_forbidden_parent_key() {
 
 #[actix_rt::test]
 async fn error_search_forbidden_token() {
-    let tenant_tokens = vec![
+    let tenant_tokens = [
         // bad index
         hashmap! {
             "searchRules" => json!({"products": {}}),

crates/meilisearch/tests/auth/tenant_token_multi_search.rs

@@ -680,7 +680,7 @@ async fn multi_search_authorized_simple_token() {
 
 #[actix_rt::test]
 async fn single_search_authorized_filter_token() {
-    let tenant_tokens = vec![
+    let tenant_tokens = [
         hashmap! {
             "searchRules" => json!({"*": {"filter": "color = blue"}}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
@@ -733,7 +733,7 @@ async fn single_search_authorized_filter_token() {
 
 #[actix_rt::test]
 async fn multi_search_authorized_filter_token() {
-    let both_tenant_tokens = vec![
+    let both_tenant_tokens = [
         hashmap! {
             "searchRules" => json!({"sales": {"filter": "color = blue"}, "products": {"filter": "doggos.age <= 5"}}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
@@ -842,7 +842,7 @@ async fn filter_single_search_authorized_filter_token() {
 
 #[actix_rt::test]
 async fn filter_multi_search_authorized_filter_token() {
-    let tenant_tokens = vec![
+    let tenant_tokens = [
         hashmap! {
             "searchRules" => json!({"sales": {"filter": "color = blue"}, "products": {"filter": "doggos.age <= 5"}}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
@@ -900,7 +900,7 @@ async fn filter_multi_search_authorized_filter_token() {
 /// Tests that those Tenant Token are incompatible with the REFUSED_KEYS defined above.
 #[actix_rt::test]
 async fn error_single_search_token_forbidden_parent_key() {
-    let tenant_tokens = vec![
+    let tenant_tokens = [
         hashmap! {
             "searchRules" => json!({"*": {}}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
@@ -941,7 +941,7 @@ async fn error_single_search_token_forbidden_parent_key() {
 /// Tests that those Tenant Token are incompatible with the REFUSED_KEYS defined above.
 #[actix_rt::test]
 async fn error_multi_search_token_forbidden_parent_key() {
-    let tenant_tokens = vec![
+    let tenant_tokens = [
         hashmap! {
             "searchRules" => json!({"*": {}}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())

crates/meilisearch/tests/settings/get_settings.rs

@@ -197,7 +197,7 @@ test_setting_routes!(
     {
       setting: vector_store,
       update_verb: patch,
-      default_value: null
+      default_value: "experimental"
     },
 );
 

crates/meilisearch/tests/settings/mod.rs

@@ -2,6 +2,7 @@ mod chat;
 mod distinct;
 mod errors;
 mod get_settings;
+mod parent_seachable_fields;
 mod prefix_search_settings;
 mod proximity_settings;
 mod tokenizer_customization;

crates/meilisearch/tests/settings/parent_seachable_fields.rs

@@ -0,0 +1,114 @@
+use meili_snap::{json_string, snapshot};
+use once_cell::sync::Lazy;
+
+use crate::common::Server;
+use crate::json;
+
+static DOCUMENTS: Lazy<crate::common::Value> = Lazy::new(|| {
+    json!([
+        {
+            "id": 1,
+            "meta": {
+                "title": "Soup of the day",
+                "description": "many the fish",
+            }
+        },
+        {
+            "id": 2,
+            "meta": {
+                "title": "Soup of day",
+                "description": "many the lazy fish",
+            }
+        },
+        {
+            "id": 3,
+            "meta": {
+                "title": "the Soup of day",
+                "description": "many the fish",
+            }
+        },
+    ])
+});
+
+#[actix_rt::test]
+async fn nested_field_becomes_searchable() {
+    let server = Server::new_shared();
+    let index = server.unique_index();
+
+    let (task, _status_code) = index.add_documents(DOCUMENTS.clone(), None).await;
+    server.wait_task(task.uid()).await.succeeded();
+
+    let (response, code) = index
+        .update_settings(json!({
+            "searchableAttributes": ["meta.title"]
+        }))
+        .await;
+    assert_eq!("202", code.as_str(), "{response:?}");
+    server.wait_task(response.uid()).await.succeeded();
+
+    // We expect no documents when searching for
+    // a nested non-searchable field
+    index
+        .search(json!({"q": "many fish"}), |response, code| {
+            snapshot!(code, @"200 OK");
+            snapshot!(json_string!(response["hits"]), @r###"[]"###);
+        })
+        .await;
+
+    let (response, code) = index
+        .update_settings(json!({
+            "searchableAttributes": ["meta.title", "meta.description"]
+        }))
+        .await;
+    assert_eq!("202", code.as_str(), "{response:?}");
+    server.wait_task(response.uid()).await.succeeded();
+
+    // We expect all the documents when the nested field becomes searchable
+    index
+        .search(json!({"q": "many fish"}), |response, code| {
+            snapshot!(code, @"200 OK");
+            snapshot!(json_string!(response["hits"]), @r###"
+            [
+              {
+                "id": 1,
+                "meta": {
+                  "title": "Soup of the day",
+                  "description": "many the fish"
+                }
+              },
+              {
+                "id": 3,
+                "meta": {
+                  "title": "the Soup of day",
+                  "description": "many the fish"
+                }
+              },
+              {
+                "id": 2,
+                "meta": {
+                  "title": "Soup of day",
+                  "description": "many the lazy fish"
+                }
+              }
+            ]
+            "###);
+        })
+        .await;
+
+    let (response, code) = index
+        .update_settings(json!({
+            "searchableAttributes": ["meta.title"]
+        }))
+        .await;
+    assert_eq!("202", code.as_str(), "{response:?}");
+    server.wait_task(response.uid()).await.succeeded();
+
+    // We expect no documents when searching for
+    // a nested non-searchable field
+    index
+        .search(json!({"q": "many fish"}), |response, code| {
+            snapshot!(code, @"200 OK");
+            snapshot!(json_string!(response["hits"]), @r###"[]"###);
+        })
+        .await;
+}

crates/meilisearch/tests/upgrade/mod.rs

@@ -42,8 +42,16 @@ async fn version_too_old() {
     std::fs::create_dir_all(&db_path).unwrap();
     std::fs::write(db_path.join("VERSION"), "1.11.9999").unwrap();
     let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
-    let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
-    snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v1.28.1");
+    let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err().to_string();
+
+    let major = meilisearch_types::versioning::VERSION_MAJOR;
+    let minor = meilisearch_types::versioning::VERSION_MINOR;
+    let patch = meilisearch_types::versioning::VERSION_PATCH;
+
+    let current_version = format!("{major}.{minor}.{patch}");
+    let err = err.replace(&current_version, "[current version]");
+
+    snapshot!(err, @"Database version 1.11.9999 is too old for the experimental dumpless upgrade feature. Please generate a dump using the v1.11.9999 and import it in the v[current version]");
 }
 
 #[actix_rt::test]
@@ -54,11 +62,21 @@ async fn version_requires_downgrade() {
     std::fs::create_dir_all(&db_path).unwrap();
     let major = meilisearch_types::versioning::VERSION_MAJOR;
     let minor = meilisearch_types::versioning::VERSION_MINOR;
-    let patch = meilisearch_types::versioning::VERSION_PATCH + 1;
-    std::fs::write(db_path.join("VERSION"), format!("{major}.{minor}.{patch}")).unwrap();
+    let mut patch = meilisearch_types::versioning::VERSION_PATCH;
+
+    let current_version = format!("{major}.{minor}.{patch}");
+    patch += 1;
+    let future_version = format!("{major}.{minor}.{patch}");
+
+    std::fs::write(db_path.join("VERSION"), &future_version).unwrap();
     let options = Opt { experimental_dumpless_upgrade: true, ..default_settings };
     let err = Server::new_with_options(options).await.map(|_| ()).unwrap_err();
-    snapshot!(err, @"Database version 1.28.2 is higher than the Meilisearch version 1.28.1. Downgrade is not supported");
+
+    let err = err.to_string();
+    let err = err.replace(&current_version, "[current version]");
+    let err = err.replace(&future_version, "[future version]");
+
+    snapshot!(err, @"Database version [future version] is higher than the Meilisearch version [current version]. Downgrade is not supported");
 }
 
 #[actix_rt::test]

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "progress": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.28.1"
+        "upgradeTo": "[current version]"
       },
       "stats": {
         "totalNbTasks": 1,

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "progress": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.28.1"
+        "upgradeTo": "[current version]"
       },
       "stats": {
         "totalNbTasks": 1,

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/batches_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "progress": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.28.1"
+        "upgradeTo": "[current version]"
       },
       "stats": {
         "totalNbTasks": 1,

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41.snap

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "canceledBy": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.28.1"
+        "upgradeTo": "[current version]"
       },
       "error": null,
       "duration": "[duration]",

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41.snap

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "canceledBy": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.28.1"
+        "upgradeTo": "[current version]"
       },
       "error": null,
       "duration": "[duration]",

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41.snap

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "canceledBy": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.28.1"
+        "upgradeTo": "[current version]"
       },
       "error": null,
       "duration": "[duration]",

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_batch_queue_once_everything_has_been_processed.snap

@@ -8,7 +8,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "progress": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.28.1"
+        "upgradeTo": "[current version]"
       },
       "stats": {
         "totalNbTasks": 1,

crates/meilisearch/tests/upgrade/v1_12/snapshots/v1_12_0.rs/check_the_index_scheduler/the_whole_task_queue_once_everything_has_been_processed.snap

@@ -12,7 +12,7 @@ source: crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs
       "canceledBy": null,
       "details": {
         "upgradeFrom": "v1.12.0",
-        "upgradeTo": "v1.28.1"
+        "upgradeTo": "[current version]"
       },
       "error": null,
       "duration": "[duration]",

crates/meilisearch/tests/upgrade/v1_12/v1_12_0.rs

@@ -166,55 +166,55 @@ async fn check_the_index_scheduler(server: &Server) {
     // We rewrite the first task for all calls because it may be the upgrade database with unknown dates and duration.
     // The other tasks should NOT change
     let (tasks, _) = server.tasks_filter("limit=1000").await;
-    snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "the_whole_task_queue_once_everything_has_been_processed");
+    snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "the_whole_task_queue_once_everything_has_been_processed");
     let (batches, _) = server.batches_filter("limit=1000").await;
-    snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "the_whole_batch_queue_once_everything_has_been_processed");
+    snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "the_whole_batch_queue_once_everything_has_been_processed");
 
     // Tests all the tasks query parameters
     let (tasks, _) = server.tasks_filter("uids=10").await;
-    snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_uids_equal_10");
+    snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_uids_equal_10");
     let (tasks, _) = server.tasks_filter("batchUids=10").await;
-    snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_batchUids_equal_10");
+    snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_batchUids_equal_10");
     let (tasks, _) = server.tasks_filter("statuses=canceled").await;
-    snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_statuses_equal_canceled");
+    snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_statuses_equal_canceled");
     // types has already been tested above to retrieve the upgrade database
     let (tasks, _) = server.tasks_filter("canceledBy=19").await;
-    snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_canceledBy_equal_19");
+    snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_canceledBy_equal_19");
     let (tasks, _) = server.tasks_filter("beforeEnqueuedAt=2025-01-16T16:47:41Z").await;
-    snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41");
+    snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41");
     let (tasks, _) = server.tasks_filter("afterEnqueuedAt=2025-01-16T16:47:41Z").await;
-    snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41");
+    snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41");
     let (tasks, _) = server.tasks_filter("beforeStartedAt=2025-01-16T16:47:41Z").await;
-    snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeStartedAt_equal_2025-01-16T16_47_41");
+    snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeStartedAt_equal_2025-01-16T16_47_41");
     let (tasks, _) = server.tasks_filter("afterStartedAt=2025-01-16T16:47:41Z").await;
-    snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41");
+    snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterStartedAt_equal_2025-01-16T16_47_41");
     let (tasks, _) = server.tasks_filter("beforeFinishedAt=2025-01-16T16:47:41Z").await;
-    snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeFinishedAt_equal_2025-01-16T16_47_41");
+    snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_beforeFinishedAt_equal_2025-01-16T16_47_41");
     let (tasks, _) = server.tasks_filter("afterFinishedAt=2025-01-16T16:47:41Z").await;
-    snapshot!(json_string!(tasks, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41");
+    snapshot!(json_string!(tasks, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]" }), name: "tasks_filter_afterFinishedAt_equal_2025-01-16T16_47_41");
 
     // Tests all the batches query parameters
     let (batches, _) = server.batches_filter("uids=10").await;
-    snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_uids_equal_10");
+    snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_uids_equal_10");
     let (batches, _) = server.batches_filter("batchUids=10").await;
-    snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_batchUids_equal_10");
+    snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_batchUids_equal_10");
     let (batches, _) = server.batches_filter("statuses=canceled").await;
-    snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_statuses_equal_canceled");
+    snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_statuses_equal_canceled");
     // types has already been tested above to retrieve the upgrade database
     let (batches, _) = server.batches_filter("canceledBy=19").await;
-    snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_canceledBy_equal_19");
+    snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_canceledBy_equal_19");
     let (batches, _) = server.batches_filter("beforeEnqueuedAt=2025-01-16T16:47:41Z").await;
-    snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41");
+    snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeEnqueuedAt_equal_2025-01-16T16_47_41");
     let (batches, _) = server.batches_filter("afterEnqueuedAt=2025-01-16T16:47:41Z").await;
-    snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41");
+    snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterEnqueuedAt_equal_2025-01-16T16_47_41");
     let (batches, _) = server.batches_filter("beforeStartedAt=2025-01-16T16:47:41Z").await;
-    snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41");
+    snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeStartedAt_equal_2025-01-16T16_47_41");
     let (batches, _) = server.batches_filter("afterStartedAt=2025-01-16T16:47:41Z").await;
-    snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterStartedAt_equal_2025-01-16T16_47_41");
+    snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterStartedAt_equal_2025-01-16T16_47_41");
     let (batches, _) = server.batches_filter("beforeFinishedAt=2025-01-16T16:47:41Z").await;
-    snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41");
+    snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_beforeFinishedAt_equal_2025-01-16T16_47_41");
     let (batches, _) = server.batches_filter("afterFinishedAt=2025-01-16T16:47:41Z").await;
-    snapshot!(json_string!(batches, { ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41");
+    snapshot!(json_string!(batches, { ".results[0].details.upgradeTo" => "[current version]", ".results[0].duration" => "[duration]", ".results[0].enqueuedAt" => "[date]", ".results[0].startedAt" => "[date]", ".results[0].finishedAt" => "[date]", ".results[0].stats.progressTrace" => "[progressTrace]", ".results[0].stats.internalDatabaseSizes" => "[internalDatabaseSizes]", ".results[0].stats.writeChannelCongestion" => "[writeChannelCongestion]" }), name: "batches_filter_afterFinishedAt_equal_2025-01-16T16_47_41");
 
     let (stats, _) = server.stats().await;
     assert_json_snapshot!(stats, {

crates/meilisearch/tests/vector/binary_quantized.rs

@@ -104,8 +104,8 @@ async fn binary_quantize_before_sending_documents() {
             "manual": {
               "embeddings": [
                 [
-                  -1.0,
-                  -1.0,
+                  0.0,
+                  0.0,
                   1.0
                 ]
               ],
@@ -122,7 +122,7 @@ async fn binary_quantize_before_sending_documents() {
                 [
                   1.0,
                   1.0,
-                  -1.0
+                  0.0
                 ]
               ],
               "regenerate": false
@@ -191,8 +191,8 @@ async fn binary_quantize_after_sending_documents() {
             "manual": {
               "embeddings": [
                 [
-                  -1.0,
-                  -1.0,
+                  0.0,
+                  0.0,
                   1.0
                 ]
               ],
@@ -209,7 +209,7 @@ async fn binary_quantize_after_sending_documents() {
                 [
                   1.0,
                   1.0,
-                  -1.0
+                  0.0
                 ]
               ],
               "regenerate": false

crates/meilisearch/tests/vector/huggingface.rs

@@ -0,0 +1,43 @@
+use meili_snap::snapshot;
+
+use crate::common::{GetAllDocumentsOptions, Server};
+use crate::json;
+
+#[actix_rt::test]
+async fn hf_bge_m3_force_cls_settings() {
+    let server = Server::new_shared();
+    let index = server.unique_index();
+
+    let (response, code) = index
+        .update_settings(json!({
+          "embedders": {
+              "default": {
+                  "source": "huggingFace",
+                  "model": "baai/bge-m3",
+                  "revision": "5617a9f61b028005a4858fdac845db406aefb181",
+                  "pooling": "forceCls",
+                  // minimal template to allow potential document embedding if used later
+                  "documentTemplate": "{{doc.title}}"
+              }
+          }
+        }))
+        .await;
+    snapshot!(code, @"202 Accepted");
+    server.wait_task(response.uid()).await.succeeded();
+
+    // Try to embed one simple document
+    let (task, code) =
+        index.add_documents(json!([{ "id": 1, "title": "Hello world" }]), None).await;
+    snapshot!(code, @"202 Accepted");
+    server.wait_task(task.uid()).await.succeeded();
+
+    // Retrieve the document with vectors and assert embeddings were produced
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() })
+        .await;
+    let has_vectors = documents["results"][0]["_vectors"]["default"]["embeddings"]
+        .as_array()
+        .map(|a| !a.is_empty())
+        .unwrap_or(false);
+    snapshot!(has_vectors, @"true");
+}

crates/meilisearch/tests/vector/mod.rs

@@ -1,5 +1,6 @@
 mod binary_quantized;
 mod fragments;
+mod huggingface;
 #[cfg(feature = "test-ollama")]
 mod ollama;
 mod openai;

crates/meilisearch/tests/vector/ollama.rs

@@ -500,13 +500,6 @@ async fn test_both_apis() {
     snapshot!(code, @"200 OK");
     snapshot!(json_string!(response["hits"]), @r###"
     [
-      {
-        "id": 0,
-        "name": "kefir",
-        "gender": "M",
-        "birthyear": 2023,
-        "breed": "Patou"
-      },
       {
         "id": 2,
         "name": "Vénus",
@@ -527,6 +520,13 @@ async fn test_both_apis() {
         "gender": "M",
         "birthyear": 1995,
         "breed": "Labrador Retriever"
+      },
+      {
+        "id": 0,
+        "name": "kefir",
+        "gender": "M",
+        "birthyear": 2023,
+        "breed": "Patou"
       }
     ]
     "###);
@@ -540,13 +540,6 @@ async fn test_both_apis() {
     snapshot!(code, @"200 OK");
     snapshot!(json_string!(response["hits"]), @r###"
     [
-      {
-        "id": 0,
-        "name": "kefir",
-        "gender": "M",
-        "birthyear": 2023,
-        "breed": "Patou"
-      },
       {
         "id": 2,
         "name": "Vénus",
@@ -567,6 +560,13 @@ async fn test_both_apis() {
         "gender": "M",
         "birthyear": 1995,
         "breed": "Labrador Retriever"
+      },
+      {
+        "id": 0,
+        "name": "kefir",
+        "gender": "M",
+        "birthyear": 2023,
+        "breed": "Patou"
       }
     ]
     "###);
@@ -581,18 +581,11 @@ async fn test_both_apis() {
     snapshot!(json_string!(response["hits"]), @r###"
     [
       {
-        "id": 0,
-        "name": "kefir",
+        "id": 1,
+        "name": "Intel",
         "gender": "M",
-        "birthyear": 2023,
-        "breed": "Patou"
-      },
-      {
-        "id": 3,
-        "name": "Max",
-        "gender": "M",
-        "birthyear": 1995,
-        "breed": "Labrador Retriever"
+        "birthyear": 2011,
+        "breed": "Beagle"
       },
       {
         "id": 2,
@@ -602,11 +595,18 @@ async fn test_both_apis() {
         "breed": "Jack Russel Terrier"
       },
       {
-        "id": 1,
-        "name": "Intel",
+        "id": 3,
+        "name": "Max",
         "gender": "M",
-        "birthyear": 2011,
-        "breed": "Beagle"
+        "birthyear": 1995,
+        "breed": "Labrador Retriever"
+      },
+      {
+        "id": 0,
+        "name": "kefir",
+        "gender": "M",
+        "birthyear": 2023,
+        "breed": "Patou"
       }
     ]
     "###);
@@ -621,18 +621,11 @@ async fn test_both_apis() {
     snapshot!(json_string!(response["hits"]), @r###"
     [
       {
-        "id": 0,
-        "name": "kefir",
+        "id": 1,
+        "name": "Intel",
         "gender": "M",
-        "birthyear": 2023,
-        "breed": "Patou"
-      },
-      {
-        "id": 3,
-        "name": "Max",
-        "gender": "M",
-        "birthyear": 1995,
-        "breed": "Labrador Retriever"
+        "birthyear": 2011,
+        "breed": "Beagle"
       },
       {
         "id": 2,
@@ -642,11 +635,18 @@ async fn test_both_apis() {
         "breed": "Jack Russel Terrier"
       },
       {
-        "id": 1,
-        "name": "Intel",
+        "id": 3,
+        "name": "Max",
         "gender": "M",
-        "birthyear": 2011,
-        "breed": "Beagle"
+        "birthyear": 1995,
+        "breed": "Labrador Retriever"
+      },
+      {
+        "id": 0,
+        "name": "kefir",
+        "gender": "M",
+        "birthyear": 2023,
+        "breed": "Patou"
       }
     ]
     "###);
@@ -661,18 +661,11 @@ async fn test_both_apis() {
     snapshot!(json_string!(response["hits"]), @r###"
     [
       {
-        "id": 0,
-        "name": "kefir",
+        "id": 1,
+        "name": "Intel",
         "gender": "M",
-        "birthyear": 2023,
-        "breed": "Patou"
-      },
-      {
-        "id": 3,
-        "name": "Max",
-        "gender": "M",
-        "birthyear": 1995,
-        "breed": "Labrador Retriever"
+        "birthyear": 2011,
+        "breed": "Beagle"
       },
       {
         "id": 2,
@@ -682,11 +675,18 @@ async fn test_both_apis() {
         "breed": "Jack Russel Terrier"
       },
       {
-        "id": 1,
-        "name": "Intel",
+        "id": 3,
+        "name": "Max",
         "gender": "M",
-        "birthyear": 2011,
-        "breed": "Beagle"
+        "birthyear": 1995,
+        "breed": "Labrador Retriever"
+      },
+      {
+        "id": 0,
+        "name": "kefir",
+        "gender": "M",
+        "birthyear": 2023,
+        "breed": "Patou"
       }
     ]
     "###);
@@ -701,18 +701,11 @@ async fn test_both_apis() {
     snapshot!(json_string!(response["hits"]), @r###"
     [
       {
-        "id": 0,
-        "name": "kefir",
+        "id": 1,
+        "name": "Intel",
         "gender": "M",
-        "birthyear": 2023,
-        "breed": "Patou"
-      },
-      {
-        "id": 3,
-        "name": "Max",
-        "gender": "M",
-        "birthyear": 1995,
-        "breed": "Labrador Retriever"
+        "birthyear": 2011,
+        "breed": "Beagle"
       },
       {
         "id": 2,
@@ -722,11 +715,18 @@ async fn test_both_apis() {
         "breed": "Jack Russel Terrier"
       },
       {
-        "id": 1,
-        "name": "Intel",
+        "id": 3,
+        "name": "Max",
         "gender": "M",
-        "birthyear": 2011,
-        "breed": "Beagle"
+        "birthyear": 1995,
+        "breed": "Labrador Retriever"
+      },
+      {
+        "id": 0,
+        "name": "kefir",
+        "gender": "M",
+        "birthyear": 2023,
+        "breed": "Patou"
       }
     ]
     "###);

crates/milli/Cargo.toml

@@ -91,7 +91,7 @@ rhai = { version = "1.23.6", features = [
     "sync",
 ] }
 arroy = "0.6.4-nested-rtxns"
-hannoy = { version = "0.0.9-nested-rtxns-2", features = ["arroy"] }
+hannoy = { version = "0.1.0-nested-rtxns", features = ["arroy"] }
 rand = "0.8.5"
 tracing = "0.1.41"
 ureq = { version = "2.12.1", features = ["json"] }

crates/milli/src/fields_ids_map/metadata.rs

@@ -18,6 +18,8 @@ use crate::{
 pub struct Metadata {
     /// The weight as defined in the FieldidsWeightsMap of the searchable attribute if it is searchable.
     pub searchable: Option<Weight>,
+    /// The field is part of the exact attributes.
+    pub exact: bool,
     /// The field is part of the sortable attributes.
     pub sortable: bool,
     /// The field is defined as the distinct attribute.
@@ -209,6 +211,7 @@ impl Metadata {
 #[derive(Debug, Clone)]
 pub struct MetadataBuilder {
     searchable_attributes: Option<Vec<String>>,
+    exact_searchable_attributes: Vec<String>,
     filterable_attributes: Vec<FilterableAttributesRule>,
     sortable_attributes: HashSet<String>,
     localized_attributes: Option<Vec<LocalizedAttributesRule>>,
@@ -220,15 +223,18 @@ impl MetadataBuilder {
     pub fn from_index(index: &Index, rtxn: &RoTxn) -> Result<Self> {
         let searchable_attributes = index
             .user_defined_searchable_fields(rtxn)?
-            .map(|fields| fields.into_iter().map(|s| s.to_string()).collect());
+            .map(|fields| fields.into_iter().map(String::from).collect());
+        let exact_searchable_attributes =
+            index.exact_attributes(rtxn)?.into_iter().map(String::from).collect();
         let filterable_attributes = index.filterable_attributes_rules(rtxn)?;
         let sortable_attributes = index.sortable_fields(rtxn)?;
         let localized_attributes = index.localized_attributes_rules(rtxn)?;
-        let distinct_attribute = index.distinct_field(rtxn)?.map(|s| s.to_string());
+        let distinct_attribute = index.distinct_field(rtxn)?.map(String::from);
         let asc_desc_attributes = index.asc_desc_fields(rtxn)?;
 
         Ok(Self::new(
             searchable_attributes,
+            exact_searchable_attributes,
             filterable_attributes,
             sortable_attributes,
             localized_attributes,
@@ -242,6 +248,7 @@ impl MetadataBuilder {
     /// This is used for testing, prefer using `MetadataBuilder::from_index` instead.
     pub fn new(
         searchable_attributes: Option<Vec<String>>,
+        exact_searchable_attributes: Vec<String>,
         filterable_attributes: Vec<FilterableAttributesRule>,
         sortable_attributes: HashSet<String>,
         localized_attributes: Option<Vec<LocalizedAttributesRule>>,
@@ -256,6 +263,7 @@ impl MetadataBuilder {
 
         Self {
             searchable_attributes,
+            exact_searchable_attributes,
             filterable_attributes,
             sortable_attributes,
             localized_attributes,
@@ -269,6 +277,7 @@ impl MetadataBuilder {
             // Vectors fields are not searchable, filterable, distinct or asc_desc
             return Metadata {
                 searchable: None,
+                exact: false,
                 sortable: false,
                 distinct: false,
                 asc_desc: false,
@@ -296,6 +305,7 @@ impl MetadataBuilder {
             // Geo fields are not searchable, distinct or asc_desc
             return Metadata {
                 searchable: None,
+                exact: false,
                 sortable,
                 distinct: false,
                 asc_desc: false,
@@ -309,6 +319,7 @@ impl MetadataBuilder {
             debug_assert!(!sortable, "geojson fields should not be sortable");
             return Metadata {
                 searchable: None,
+                exact: false,
                 sortable,
                 distinct: false,
                 asc_desc: false,
@@ -329,6 +340,8 @@ impl MetadataBuilder {
             None => Some(0),
         };
 
+        let exact = self.exact_searchable_attributes.iter().any(|attr| is_faceted_by(field, attr));
+
         let distinct =
             self.distinct_attribute.as_ref().is_some_and(|distinct_field| field == distinct_field);
         let asc_desc = self.asc_desc_attributes.contains(field);
@@ -343,6 +356,7 @@ impl MetadataBuilder {
 
         Metadata {
             searchable,
+            exact,
             sortable,
             distinct,
             asc_desc,

crates/milli/src/index.rs

@@ -281,6 +281,9 @@ impl Index {
                 &mut wtxn,
                 (constants::VERSION_MAJOR, constants::VERSION_MINOR, constants::VERSION_PATCH),
             )?;
+            // The database before v1.29 defaulted to using arroy, so we
+            // need to set it explicitly because the new default is hannoy.
+            this.put_vector_store(&mut wtxn, VectorStoreBackend::Hannoy)?;
         }
         wtxn.commit()?;
 

crates/milli/src/search/mod.rs

@@ -385,9 +385,10 @@ pub struct SearchResult {
     pub query_vector: Option<Embedding>,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+#[derive(Default, Debug, Clone, Copy, PartialEq, Eq)]
 pub enum TermsMatchingStrategy {
     // remove last word first
+    #[default]
     Last,
     // all words are mandatory
     All,
@@ -395,12 +396,6 @@ pub enum TermsMatchingStrategy {
     Frequency,
 }
 
-impl Default for TermsMatchingStrategy {
-    fn default() -> Self {
-        Self::Last
-    }
-}
-
 impl From<MatchingStrategy> for TermsMatchingStrategy {
     fn from(other: MatchingStrategy) -> Self {
         match other {

crates/milli/src/update/index_documents/helpers/grenad_helpers.rs

@@ -124,7 +124,7 @@ impl GrenadParameters {
     /// This should be called inside of a rayon thread pool,
     /// otherwise, it will take the global number of threads.
     pub fn max_memory_by_thread(&self) -> Option<usize> {
-        self.max_memory.map(|max_memory| (max_memory / rayon::current_num_threads()))
+        self.max_memory.map(|max_memory| max_memory / rayon::current_num_threads())
     }
 }
 

crates/milli/src/update/index_documents/mod.rs

@@ -54,11 +54,12 @@ pub struct DocumentAdditionResult {
     pub number_of_documents: u64,
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
+#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
 #[non_exhaustive]
 pub enum IndexDocumentsMethod {
     /// Replace the previous document with the new one,
     /// removing all the already known attributes.
+    #[default]
     ReplaceDocuments,
 
     /// Merge the previous version of the document with the new version,
@@ -66,12 +67,6 @@ pub enum IndexDocumentsMethod {
     UpdateDocuments,
 }
 
-impl Default for IndexDocumentsMethod {
-    fn default() -> Self {
-        Self::ReplaceDocuments
-    }
-}
-
 pub struct IndexDocuments<'t, 'i, 'a, FP, FA> {
     wtxn: &'t mut heed::RwTxn<'i>,
     index: &'i Index,
@@ -806,6 +801,10 @@ mod tests {
     use crate::vector::db::IndexEmbeddingConfig;
     use crate::{all_obkv_to_json, db_snap, Filter, FilterableAttributesRule, Search, UserError};
 
+    fn no_cancel() -> bool {
+        false
+    }
+
     #[test]
     fn simple_document_replacement() {
         let index = TempIndex::new();
@@ -1985,7 +1984,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -2038,7 +2037,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -2057,7 +2056,7 @@ mod tests {
             primary_key,
             &document_changes,
             RuntimeEmbedders::default(),
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -2127,7 +2126,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -2146,7 +2145,7 @@ mod tests {
             primary_key,
             &document_changes,
             RuntimeEmbedders::default(),
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -2317,7 +2316,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -2333,7 +2332,7 @@ mod tests {
             primary_key,
             &document_changes,
             embedders,
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -2381,7 +2380,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -2397,7 +2396,7 @@ mod tests {
             primary_key,
             &document_changes,
             embedders,
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -2436,7 +2435,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -2452,7 +2451,7 @@ mod tests {
             primary_key,
             &document_changes,
             embedders,
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -2490,7 +2489,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -2506,7 +2505,7 @@ mod tests {
             primary_key,
             &document_changes,
             embedders,
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -2546,7 +2545,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -2562,7 +2561,7 @@ mod tests {
             primary_key,
             &document_changes,
             embedders,
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -2607,7 +2606,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -2623,7 +2622,7 @@ mod tests {
             primary_key,
             &document_changes,
             embedders,
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -2661,7 +2660,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -2677,7 +2676,7 @@ mod tests {
             primary_key,
             &document_changes,
             embedders,
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -2715,7 +2714,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -2731,7 +2730,7 @@ mod tests {
             primary_key,
             &document_changes,
             embedders,
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -2927,7 +2926,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -2943,7 +2942,7 @@ mod tests {
             primary_key,
             &document_changes,
             embedders,
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -2988,7 +2987,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -3004,7 +3003,7 @@ mod tests {
             primary_key,
             &document_changes,
             embedders,
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )
@@ -3046,7 +3045,7 @@ mod tests {
                 &rtxn,
                 None,
                 &mut new_fields_ids_map,
-                &|| false,
+                &no_cancel,
                 Progress::default(),
                 None,
             )
@@ -3062,7 +3061,7 @@ mod tests {
             primary_key,
             &document_changes,
             embedders,
-            &|| false,
+            &no_cancel,
             &Progress::default(),
             &Default::default(),
         )

crates/milli/src/update/new/extract/searchable/extract_word_docids.rs

@@ -8,17 +8,26 @@ use bumpalo::Bump;
 
 use super::match_searchable_field;
 use super::tokenize_document::{tokenizer_builder, DocumentTokenizer};
+use crate::fields_ids_map::metadata::Metadata;
 use crate::update::new::document::DocumentContext;
 use crate::update::new::extract::cache::BalancedCaches;
 use crate::update::new::extract::perm_json_p::contained_in;
+use crate::update::new::extract::searchable::has_searchable_children;
 use crate::update::new::indexer::document_changes::{
     extract, DocumentChanges, Extractor, IndexingContext,
 };
+use crate::update::new::indexer::settings_changes::{
+    settings_change_extract, DocumentsIndentifiers, SettingsChangeExtractor,
+};
 use crate::update::new::ref_cell_ext::RefCellExt as _;
 use crate::update::new::steps::IndexingStep;
 use crate::update::new::thread_local::{FullySend, MostlySend, ThreadLocal};
-use crate::update::new::DocumentChange;
-use crate::{bucketed_position, DocumentId, FieldId, Result, MAX_POSITION_PER_ATTRIBUTE};
+use crate::update::new::{DocumentChange, DocumentIdentifiers};
+use crate::update::settings::SettingsDelta;
+use crate::{
+    bucketed_position, DocumentId, FieldId, PatternMatch, Result, UserError,
+    MAX_POSITION_PER_ATTRIBUTE,
+};
 
 const MAX_COUNTED_WORDS: usize = 30;
 
@@ -34,6 +43,15 @@ pub struct WordDocidsBalancedCaches<'extractor> {
 
 unsafe impl MostlySend for WordDocidsBalancedCaches<'_> {}
 
+/// Whether to extract or skip fields during word extraction.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum FieldDbExtraction {
+    /// Extract the word and put it in to the fid-based databases.
+    Extract,
+    /// Do not store the word in the fid-based databases.
+    Skip,
+}
+
 impl<'extractor> WordDocidsBalancedCaches<'extractor> {
     pub fn new_in(buckets: usize, max_memory: Option<usize>, alloc: &'extractor Bump) -> Self {
         Self {
@@ -47,12 +65,14 @@ impl<'extractor> WordDocidsBalancedCaches<'extractor> {
         }
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn insert_add_u32(
         &mut self,
         field_id: FieldId,
         position: u16,
         word: &str,
         exact: bool,
+        field_db_extraction: FieldDbExtraction,
         docid: u32,
         bump: &Bump,
     ) -> Result<()> {
@@ -66,11 +86,13 @@ impl<'extractor> WordDocidsBalancedCaches<'extractor> {
         let buffer_size = word_bytes.len() + 1 + size_of::<FieldId>();
         let mut buffer = BumpVec::with_capacity_in(buffer_size, bump);
 
+        if field_db_extraction == FieldDbExtraction::Extract {
             buffer.clear();
             buffer.extend_from_slice(word_bytes);
             buffer.push(0);
             buffer.extend_from_slice(&field_id.to_be_bytes());
             self.word_fid_docids.insert_add_u32(&buffer, docid)?;
+        }
 
         let position = bucketed_position(position);
         buffer.clear();
@@ -83,21 +105,26 @@ impl<'extractor> WordDocidsBalancedCaches<'extractor> {
             self.flush_fid_word_count(&mut buffer)?;
         }
 
+        if field_db_extraction == FieldDbExtraction::Extract {
             self.fid_word_count
                 .entry(field_id)
                 .and_modify(|(_current_count, new_count)| *new_count.get_or_insert(0) += 1)
                 .or_insert((None, Some(1)));
+        }
+
         self.current_docid = Some(docid);
 
         Ok(())
     }
 
+    #[allow(clippy::too_many_arguments)]
     fn insert_del_u32(
         &mut self,
         field_id: FieldId,
         position: u16,
         word: &str,
         exact: bool,
+        field_db_extraction: FieldDbExtraction,
         docid: u32,
         bump: &Bump,
     ) -> Result<()> {
@@ -111,11 +138,13 @@ impl<'extractor> WordDocidsBalancedCaches<'extractor> {
         let buffer_size = word_bytes.len() + 1 + size_of::<FieldId>();
         let mut buffer = BumpVec::with_capacity_in(buffer_size, bump);
 
+        if field_db_extraction == FieldDbExtraction::Extract {
             buffer.clear();
             buffer.extend_from_slice(word_bytes);
             buffer.push(0);
             buffer.extend_from_slice(&field_id.to_be_bytes());
             self.word_fid_docids.insert_del_u32(&buffer, docid)?;
+        }
 
         let position = bucketed_position(position);
         buffer.clear();
@@ -128,10 +157,12 @@ impl<'extractor> WordDocidsBalancedCaches<'extractor> {
             self.flush_fid_word_count(&mut buffer)?;
         }
 
+        if field_db_extraction == FieldDbExtraction::Extract {
             self.fid_word_count
                 .entry(field_id)
                 .and_modify(|(current_count, _new_count)| *current_count.get_or_insert(0) += 1)
                 .or_insert((Some(1), None));
+        }
 
         self.current_docid = Some(docid);
 
@@ -325,6 +356,24 @@ impl WordDocidsExtractors {
             exact_attributes.iter().any(|attr| contained_in(fname, attr))
                 || disabled_typos_terms.is_exact(word)
         };
+
+        let mut should_tokenize = |field_name: &str| {
+            let Some((field_id, meta)) = new_fields_ids_map.id_with_metadata_or_insert(field_name)
+            else {
+                return Err(UserError::AttributeLimitReached.into());
+            };
+
+            let pattern_match = if meta.is_searchable() {
+                PatternMatch::Match
+            } else {
+                // TODO: should be a match on the field_name using `match_field_legacy` function,
+                //       but for legacy reasons we iterate over all the fields to fill the field_id_map.
+                PatternMatch::Parent
+            };
+
+            Ok((field_id, pattern_match))
+        };
+
         match document_change {
             DocumentChange::Deletion(inner) => {
                 let mut token_fn = |fname: &str, fid, pos, word: &str| {
@@ -333,13 +382,14 @@ impl WordDocidsExtractors {
                         pos,
                         word,
                         is_exact(fname, word),
+                        FieldDbExtraction::Extract,
                         inner.docid(),
                         doc_alloc,
                     )
                 };
                 document_tokenizer.tokenize_document(
                     inner.current(rtxn, index, context.db_fields_ids_map)?,
-                    new_fields_ids_map,
+                    &mut should_tokenize,
                     &mut token_fn,
                 )?;
             }
@@ -361,13 +411,14 @@ impl WordDocidsExtractors {
                         pos,
                         word,
                         is_exact(fname, word),
+                        FieldDbExtraction::Extract,
                         inner.docid(),
                         doc_alloc,
                     )
                 };
                 document_tokenizer.tokenize_document(
                     inner.current(rtxn, index, context.db_fields_ids_map)?,
-                    new_fields_ids_map,
+                    &mut should_tokenize,
                     &mut token_fn,
                 )?;
 
@@ -377,13 +428,14 @@ impl WordDocidsExtractors {
                         pos,
                         word,
                         is_exact(fname, word),
+                        FieldDbExtraction::Extract,
                         inner.docid(),
                         doc_alloc,
                     )
                 };
                 document_tokenizer.tokenize_document(
                     inner.merged(rtxn, index, context.db_fields_ids_map)?,
-                    new_fields_ids_map,
+                    &mut should_tokenize,
                     &mut token_fn,
                 )?;
             }
@@ -394,13 +446,14 @@ impl WordDocidsExtractors {
                         pos,
                         word,
                         is_exact(fname, word),
+                        FieldDbExtraction::Extract,
                         inner.docid(),
                         doc_alloc,
                     )
                 };
                 document_tokenizer.tokenize_document(
                     inner.inserted(),
-                    new_fields_ids_map,
+                    &mut should_tokenize,
                     &mut token_fn,
                 )?;
             }
@@ -411,3 +464,292 @@ impl WordDocidsExtractors {
         cached_sorter.flush_fid_word_count(&mut buffer)
     }
 }
+
+pub struct WordDocidsSettingsExtractorsData<'a, SD> {
+    tokenizer: DocumentTokenizer<'a>,
+    max_memory_by_thread: Option<usize>,
+    buckets: usize,
+    settings_delta: &'a SD,
+}
+
+impl<'extractor, SD: SettingsDelta + Sync> SettingsChangeExtractor<'extractor>
+    for WordDocidsSettingsExtractorsData<'_, SD>
+{
+    type Data = RefCell<Option<WordDocidsBalancedCaches<'extractor>>>;
+
+    fn init_data<'doc>(&'doc self, extractor_alloc: &'extractor Bump) -> crate::Result<Self::Data> {
+        Ok(RefCell::new(Some(WordDocidsBalancedCaches::new_in(
+            self.buckets,
+            self.max_memory_by_thread,
+            extractor_alloc,
+        ))))
+    }
+
+    fn process<'doc>(
+        &'doc self,
+        documents: impl Iterator<Item = crate::Result<DocumentIdentifiers<'doc>>>,
+        context: &'doc DocumentContext<Self::Data>,
+    ) -> crate::Result<()> {
+        for document in documents {
+            let document = document?;
+            SettingsChangeWordDocidsExtractors::extract_document_from_settings_change(
+                document,
+                context,
+                &self.tokenizer,
+                self.settings_delta,
+            )?;
+        }
+        Ok(())
+    }
+}
+
+pub struct SettingsChangeWordDocidsExtractors;
+
+impl SettingsChangeWordDocidsExtractors {
+    pub fn run_extraction<'fid, 'indexer, 'index, 'extractor, SD, MSP>(
+        settings_delta: &SD,
+        documents: &'indexer DocumentsIndentifiers<'indexer>,
+        indexing_context: IndexingContext<'fid, 'indexer, 'index, MSP>,
+        extractor_allocs: &'extractor mut ThreadLocal<FullySend<Bump>>,
+        step: IndexingStep,
+    ) -> Result<WordDocidsCaches<'extractor>>
+    where
+        SD: SettingsDelta + Sync,
+        MSP: Fn() -> bool + Sync,
+    {
+        // Warning: this is duplicated code from extract_word_pair_proximity_docids.rs
+        // TODO we need to read the new AND old settings to support changing global parameters
+        let rtxn = indexing_context.index.read_txn()?;
+        let stop_words = indexing_context.index.stop_words(&rtxn)?;
+        let allowed_separators = indexing_context.index.allowed_separators(&rtxn)?;
+        let allowed_separators: Option<Vec<_>> =
+            allowed_separators.as_ref().map(|s| s.iter().map(String::as_str).collect());
+        let dictionary = indexing_context.index.dictionary(&rtxn)?;
+        let dictionary: Option<Vec<_>> =
+            dictionary.as_ref().map(|s| s.iter().map(String::as_str).collect());
+        let mut builder = tokenizer_builder(
+            stop_words.as_ref(),
+            allowed_separators.as_deref(),
+            dictionary.as_deref(),
+        );
+        let tokenizer = builder.build();
+        let localized_attributes_rules =
+            indexing_context.index.localized_attributes_rules(&rtxn)?.unwrap_or_default();
+        let document_tokenizer = DocumentTokenizer {
+            tokenizer: &tokenizer,
+            localized_attributes_rules: &localized_attributes_rules,
+            max_positions_per_attributes: MAX_POSITION_PER_ATTRIBUTE,
+        };
+        let extractor_data = WordDocidsSettingsExtractorsData {
+            tokenizer: document_tokenizer,
+            max_memory_by_thread: indexing_context.grenad_parameters.max_memory_by_thread(),
+            buckets: rayon::current_num_threads(),
+            settings_delta,
+        };
+        let datastore = ThreadLocal::new();
+        {
+            let span = tracing::debug_span!(target: "indexing::documents::extract", "vectors");
+            let _entered = span.enter();
+
+            settings_change_extract(
+                documents,
+                &extractor_data,
+                indexing_context,
+                extractor_allocs,
+                &datastore,
+                step,
+            )?;
+        }
+
+        let mut merger = WordDocidsCaches::new();
+        for cache in datastore.into_iter().flat_map(RefCell::into_inner) {
+            merger.push(cache)?;
+        }
+
+        Ok(merger)
+    }
+
+    /// Extracts document words from a settings change.
+    fn extract_document_from_settings_change<SD: SettingsDelta>(
+        document: DocumentIdentifiers<'_>,
+        context: &DocumentContext<RefCell<Option<WordDocidsBalancedCaches>>>,
+        document_tokenizer: &DocumentTokenizer,
+        settings_delta: &SD,
+    ) -> Result<()> {
+        let mut cached_sorter_ref = context.data.borrow_mut_or_yield();
+        let cached_sorter = cached_sorter_ref.as_mut().unwrap();
+        let doc_alloc = &context.doc_alloc;
+
+        let new_fields_ids_map = settings_delta.new_fields_ids_map();
+        let old_fields_ids_map = context.index.fields_ids_map_with_metadata(&context.rtxn)?;
+        let old_searchable = settings_delta.old_searchable_attributes().as_ref();
+        let new_searchable = settings_delta.new_searchable_attributes().as_ref();
+
+        let current_document = document.current(
+            &context.rtxn,
+            context.index,
+            old_fields_ids_map.as_fields_ids_map(),
+        )?;
+
+        #[derive(Debug, Clone, Copy, PartialEq)]
+        enum ActionToOperate {
+            ReindexAllFields,
+            // TODO improve by listing field prefixes
+            IndexAddedFields,
+            SkipDocument,
+        }
+
+        let mut action = ActionToOperate::SkipDocument;
+        // Here we do a preliminary check to determine the action to take.
+        // This check doesn't trigger the tokenizer as we never return
+        // PatternMatch::Match.
+        document_tokenizer.tokenize_document(
+            current_document,
+            &mut |field_name| {
+                let fid = new_fields_ids_map.id(field_name).expect("All fields IDs must exist");
+
+                // If the document must be reindexed, early return NoMatch to stop the scanning process.
+                if action == ActionToOperate::ReindexAllFields {
+                    return Ok((fid, PatternMatch::NoMatch));
+                }
+
+                let old_field_metadata = old_fields_ids_map.metadata(fid).unwrap();
+                let new_field_metadata = new_fields_ids_map.metadata(fid).unwrap();
+
+                action = match (old_field_metadata, new_field_metadata) {
+                    // At least one field is added or removed from the exact fields => ReindexAllFields
+                    (Metadata { exact: old_exact, .. }, Metadata { exact: new_exact, .. })
+                        if old_exact != new_exact =>
+                    {
+                        ActionToOperate::ReindexAllFields
+                    }
+                    // At least one field is removed from the searchable fields => ReindexAllFields
+                    (Metadata { searchable: Some(_), .. }, Metadata { searchable: None, .. }) => {
+                        ActionToOperate::ReindexAllFields
+                    }
+                    // At least one field is added in the searchable fields => IndexAddedFields
+                    (Metadata { searchable: None, .. }, Metadata { searchable: Some(_), .. }) => {
+                        // We can safely overwrite the action, because we early return when action is ReindexAllFields.
+                        ActionToOperate::IndexAddedFields
+                    }
+                    _ => action,
+                };
+
+                Ok((fid, PatternMatch::Parent))
+            },
+            &mut |_, _, _, _| Ok(()),
+        )?;
+
+        // Early return when we don't need to index the document
+        if action == ActionToOperate::SkipDocument {
+            return Ok(());
+        }
+
+        let mut should_tokenize = |field_name: &str| {
+            let field_id = new_fields_ids_map.id(field_name).expect("All fields IDs must exist");
+            let old_field_metadata = old_fields_ids_map.metadata(field_id).unwrap();
+            let new_field_metadata = new_fields_ids_map.metadata(field_id).unwrap();
+
+            let pattern_match = match action {
+                ActionToOperate::ReindexAllFields => {
+                    if old_field_metadata.is_searchable() || new_field_metadata.is_searchable() {
+                        PatternMatch::Match
+                    // If any old or new field is searchable then we need to iterate over all fields
+                    // else if any field matches we need to iterate over all fields
+                    } else if has_searchable_children(
+                        field_name,
+                        old_searchable.zip(new_searchable).map(|(old, new)| old.iter().chain(new)),
+                    ) {
+                        PatternMatch::Parent
+                    } else {
+                        PatternMatch::NoMatch
+                    }
+                }
+                ActionToOperate::IndexAddedFields => {
+                    // Was not searchable but now is
+                    if !old_field_metadata.is_searchable() && new_field_metadata.is_searchable() {
+                        PatternMatch::Match
+                    // If the field is now a parent of a searchable field
+                    } else if has_searchable_children(field_name, new_searchable) {
+                        PatternMatch::Parent
+                    } else {
+                        PatternMatch::NoMatch
+                    }
+                }
+                ActionToOperate::SkipDocument => unreachable!(),
+            };
+
+            Ok((field_id, pattern_match))
+        };
+
+        let old_disabled_typos_terms = settings_delta.old_disabled_typos_terms();
+        let new_disabled_typos_terms = settings_delta.new_disabled_typos_terms();
+        let mut token_fn = |_field_name: &str, field_id, pos, word: &str| {
+            let old_field_metadata = old_fields_ids_map.metadata(field_id).unwrap();
+            let new_field_metadata = new_fields_ids_map.metadata(field_id).unwrap();
+
+            match (old_field_metadata, new_field_metadata) {
+                (
+                    Metadata { searchable: Some(_), exact: old_exact, .. },
+                    Metadata { searchable: None, .. },
+                ) => cached_sorter.insert_del_u32(
+                    field_id,
+                    pos,
+                    word,
+                    old_exact || old_disabled_typos_terms.is_exact(word),
+                    // We deleted the field globally
+                    FieldDbExtraction::Skip,
+                    document.docid(),
+                    doc_alloc,
+                ),
+                (
+                    Metadata { searchable: None, .. },
+                    Metadata { searchable: Some(_), exact: new_exact, .. },
+                ) => cached_sorter.insert_add_u32(
+                    field_id,
+                    pos,
+                    word,
+                    new_exact || new_disabled_typos_terms.is_exact(word),
+                    FieldDbExtraction::Extract,
+                    document.docid(),
+                    doc_alloc,
+                ),
+                (Metadata { searchable: None, .. }, Metadata { searchable: None, .. }) => {
+                    unreachable!()
+                }
+                (Metadata { exact: old_exact, .. }, Metadata { exact: new_exact, .. }) => {
+                    cached_sorter.insert_del_u32(
+                        field_id,
+                        pos,
+                        word,
+                        old_exact || old_disabled_typos_terms.is_exact(word),
+                        // The field has already been extracted
+                        FieldDbExtraction::Skip,
+                        document.docid(),
+                        doc_alloc,
+                    )?;
+                    cached_sorter.insert_add_u32(
+                        field_id,
+                        pos,
+                        word,
+                        new_exact || new_disabled_typos_terms.is_exact(word),
+                        // The field has already been extracted
+                        FieldDbExtraction::Skip,
+                        document.docid(),
+                        doc_alloc,
+                    )
+                }
+            }
+        };
+
+        // TODO we must tokenize twice when we change global parameters like stop words,
+        //      the language settings, dictionary, separators, non-separators...
+        document_tokenizer.tokenize_document(
+            current_document,
+            &mut should_tokenize,
+            &mut token_fn,
+        )?;
+
+        Ok(())
+    }
+}

crates/milli/src/update/new/extract/searchable/extract_word_pair_proximity_docids.rs

@@ -6,17 +6,24 @@ use bumpalo::Bump;
 
 use super::match_searchable_field;
 use super::tokenize_document::{tokenizer_builder, DocumentTokenizer};
+use crate::fields_ids_map::metadata::Metadata;
+use crate::proximity::ProximityPrecision::*;
 use crate::proximity::{index_proximity, MAX_DISTANCE};
 use crate::update::new::document::{Document, DocumentContext};
 use crate::update::new::extract::cache::BalancedCaches;
 use crate::update::new::indexer::document_changes::{
     extract, DocumentChanges, Extractor, IndexingContext,
 };
+use crate::update::new::indexer::settings_change_extract;
+use crate::update::new::indexer::settings_changes::{
+    DocumentsIndentifiers, SettingsChangeExtractor,
+};
 use crate::update::new::ref_cell_ext::RefCellExt as _;
 use crate::update::new::steps::IndexingStep;
 use crate::update::new::thread_local::{FullySend, ThreadLocal};
-use crate::update::new::DocumentChange;
-use crate::{FieldId, GlobalFieldsIdsMap, Result, MAX_POSITION_PER_ATTRIBUTE};
+use crate::update::new::{DocumentChange, DocumentIdentifiers};
+use crate::update::settings::SettingsDelta;
+use crate::{FieldId, PatternMatch, Result, UserError, MAX_POSITION_PER_ATTRIBUTE};
 
 pub struct WordPairProximityDocidsExtractorData<'a> {
     tokenizer: DocumentTokenizer<'a>,
@@ -116,7 +123,7 @@ impl WordPairProximityDocidsExtractor {
     // and to store the docids of the documents that have a number of words in a given field
     // equal to or under than MAX_COUNTED_WORDS.
     fn extract_document_change(
-        context: &DocumentContext<RefCell<BalancedCaches>>,
+        context: &DocumentContext<RefCell<BalancedCaches<'_>>>,
         document_tokenizer: &DocumentTokenizer,
         searchable_attributes: Option<&[&str]>,
         document_change: DocumentChange,
@@ -147,8 +154,12 @@ impl WordPairProximityDocidsExtractor {
                 process_document_tokens(
                     document,
                     document_tokenizer,
-                    new_fields_ids_map,
                     &mut word_positions,
+                    &mut |field_name| {
+                        new_fields_ids_map
+                            .id_with_metadata_or_insert(field_name)
+                            .ok_or(UserError::AttributeLimitReached.into())
+                    },
                     &mut |(w1, w2), prox| {
                         del_word_pair_proximity.push(((w1, w2), prox));
                     },
@@ -170,8 +181,12 @@ impl WordPairProximityDocidsExtractor {
                 process_document_tokens(
                     document,
                     document_tokenizer,
-                    new_fields_ids_map,
                     &mut word_positions,
+                    &mut |field_name| {
+                        new_fields_ids_map
+                            .id_with_metadata_or_insert(field_name)
+                            .ok_or(UserError::AttributeLimitReached.into())
+                    },
                     &mut |(w1, w2), prox| {
                         del_word_pair_proximity.push(((w1, w2), prox));
                     },
@@ -180,8 +195,12 @@ impl WordPairProximityDocidsExtractor {
                 process_document_tokens(
                     document,
                     document_tokenizer,
-                    new_fields_ids_map,
                     &mut word_positions,
+                    &mut |field_name| {
+                        new_fields_ids_map
+                            .id_with_metadata_or_insert(field_name)
+                            .ok_or(UserError::AttributeLimitReached.into())
+                    },
                     &mut |(w1, w2), prox| {
                         add_word_pair_proximity.push(((w1, w2), prox));
                     },
@@ -192,8 +211,12 @@ impl WordPairProximityDocidsExtractor {
                 process_document_tokens(
                     document,
                     document_tokenizer,
-                    new_fields_ids_map,
                     &mut word_positions,
+                    &mut |field_name| {
+                        new_fields_ids_map
+                            .id_with_metadata_or_insert(field_name)
+                            .ok_or(UserError::AttributeLimitReached.into())
+                    },
                     &mut |(w1, w2), prox| {
                         add_word_pair_proximity.push(((w1, w2), prox));
                     },
@@ -257,8 +280,8 @@ fn drain_word_positions(
 fn process_document_tokens<'doc>(
     document: impl Document<'doc>,
     document_tokenizer: &DocumentTokenizer,
-    fields_ids_map: &mut GlobalFieldsIdsMap,
     word_positions: &mut VecDeque<(Rc<str>, u16)>,
+    field_id_and_metadata: &mut impl FnMut(&str) -> Result<(FieldId, Metadata)>,
     word_pair_proximity: &mut impl FnMut((Rc<str>, Rc<str>), u8),
 ) -> Result<()> {
     let mut field_id = None;
@@ -279,8 +302,248 @@ fn process_document_tokens<'doc>(
         word_positions.push_back((Rc::from(word), pos));
         Ok(())
     };
-    document_tokenizer.tokenize_document(document, fields_ids_map, &mut token_fn)?;
+
+    let mut should_tokenize = |field_name: &str| {
+        let (field_id, meta) = field_id_and_metadata(field_name)?;
+
+        let pattern_match = if meta.is_searchable() {
+            PatternMatch::Match
+        } else {
+            // TODO: should be a match on the field_name using `match_field_legacy` function,
+            //       but for legacy reasons we iterate over all the fields to fill the field_id_map.
+            PatternMatch::Parent
+        };
+
+        Ok((field_id, pattern_match))
+    };
+
+    document_tokenizer.tokenize_document(document, &mut should_tokenize, &mut token_fn)?;
 
     drain_word_positions(word_positions, word_pair_proximity);
     Ok(())
 }
+
+pub struct WordPairProximityDocidsSettingsExtractorsData<'a, SD> {
+    tokenizer: DocumentTokenizer<'a>,
+    max_memory_by_thread: Option<usize>,
+    buckets: usize,
+    settings_delta: &'a SD,
+}
+
+impl<'extractor, SD: SettingsDelta + Sync> SettingsChangeExtractor<'extractor>
+    for WordPairProximityDocidsSettingsExtractorsData<'_, SD>
+{
+    type Data = RefCell<BalancedCaches<'extractor>>;
+
+    fn init_data<'doc>(&'doc self, extractor_alloc: &'extractor Bump) -> crate::Result<Self::Data> {
+        Ok(RefCell::new(BalancedCaches::new_in(
+            self.buckets,
+            self.max_memory_by_thread,
+            extractor_alloc,
+        )))
+    }
+
+    fn process<'doc>(
+        &'doc self,
+        documents: impl Iterator<Item = crate::Result<DocumentIdentifiers<'doc>>>,
+        context: &'doc DocumentContext<Self::Data>,
+    ) -> crate::Result<()> {
+        for document in documents {
+            let document = document?;
+            SettingsChangeWordPairProximityDocidsExtractors::extract_document_from_settings_change(
+                document,
+                context,
+                &self.tokenizer,
+                self.settings_delta,
+            )?;
+        }
+        Ok(())
+    }
+}
+
+pub struct SettingsChangeWordPairProximityDocidsExtractors;
+
+impl SettingsChangeWordPairProximityDocidsExtractors {
+    pub fn run_extraction<'fid, 'indexer, 'index, 'extractor, SD, MSP>(
+        settings_delta: &SD,
+        documents: &'indexer DocumentsIndentifiers<'indexer>,
+        indexing_context: IndexingContext<'fid, 'indexer, 'index, MSP>,
+        extractor_allocs: &'extractor mut ThreadLocal<FullySend<Bump>>,
+        step: IndexingStep,
+    ) -> Result<Vec<BalancedCaches<'extractor>>>
+    where
+        SD: SettingsDelta + Sync,
+        MSP: Fn() -> bool + Sync,
+    {
+        // Warning: this is duplicated code from extract_word_docids.rs
+        let rtxn = indexing_context.index.read_txn()?;
+        let stop_words = indexing_context.index.stop_words(&rtxn)?;
+        let allowed_separators = indexing_context.index.allowed_separators(&rtxn)?;
+        let allowed_separators: Option<Vec<_>> =
+            allowed_separators.as_ref().map(|s| s.iter().map(String::as_str).collect());
+        let dictionary = indexing_context.index.dictionary(&rtxn)?;
+        let dictionary: Option<Vec<_>> =
+            dictionary.as_ref().map(|s| s.iter().map(String::as_str).collect());
+        let mut builder = tokenizer_builder(
+            stop_words.as_ref(),
+            allowed_separators.as_deref(),
+            dictionary.as_deref(),
+        );
+        let tokenizer = builder.build();
+        let localized_attributes_rules =
+            indexing_context.index.localized_attributes_rules(&rtxn)?.unwrap_or_default();
+        let document_tokenizer = DocumentTokenizer {
+            tokenizer: &tokenizer,
+            localized_attributes_rules: &localized_attributes_rules,
+            max_positions_per_attributes: MAX_POSITION_PER_ATTRIBUTE,
+        };
+        let extractor_data = WordPairProximityDocidsSettingsExtractorsData {
+            tokenizer: document_tokenizer,
+            max_memory_by_thread: indexing_context.grenad_parameters.max_memory_by_thread(),
+            buckets: rayon::current_num_threads(),
+            settings_delta,
+        };
+        let datastore = ThreadLocal::new();
+        {
+            let span = tracing::trace_span!(target: "indexing::documents::extract", "word_pair_proximity_docids_extraction");
+            let _entered = span.enter();
+
+            settings_change_extract(
+                documents,
+                &extractor_data,
+                indexing_context,
+                extractor_allocs,
+                &datastore,
+                step,
+            )?;
+        }
+
+        Ok(datastore.into_iter().map(RefCell::into_inner).collect())
+    }
+
+    /// Extracts document words from a settings change.
+    fn extract_document_from_settings_change<SD: SettingsDelta>(
+        document: DocumentIdentifiers<'_>,
+        context: &DocumentContext<RefCell<BalancedCaches<'_>>>,
+        document_tokenizer: &DocumentTokenizer,
+        settings_delta: &SD,
+    ) -> Result<()> {
+        let mut cached_sorter = context.data.borrow_mut_or_yield();
+        let doc_alloc = &context.doc_alloc;
+
+        let new_fields_ids_map = settings_delta.new_fields_ids_map();
+        let old_fields_ids_map = settings_delta.old_fields_ids_map();
+        let old_proximity_precision = *settings_delta.old_proximity_precision();
+        let new_proximity_precision = *settings_delta.new_proximity_precision();
+
+        let current_document = document.current(
+            &context.rtxn,
+            context.index,
+            old_fields_ids_map.as_fields_ids_map(),
+        )?;
+
+        #[derive(Debug, Clone, Copy, PartialEq)]
+        enum ActionToOperate {
+            ReindexAllFields,
+            SkipDocument,
+        }
+
+        // TODO prefix_fid delete_old_fid_based_databases
+        let mut action = match (old_proximity_precision, new_proximity_precision) {
+            (ByAttribute, ByWord) => ActionToOperate::ReindexAllFields,
+            (_, _) => ActionToOperate::SkipDocument,
+        };
+
+        // Here we do a preliminary check to determine the action to take.
+        // This check doesn't trigger the tokenizer as we never return
+        // PatternMatch::Match.
+        if action != ActionToOperate::ReindexAllFields {
+            document_tokenizer.tokenize_document(
+                current_document,
+                &mut |field_name| {
+                    let fid = new_fields_ids_map.id(field_name).expect("All fields IDs must exist");
+
+                    // If the document must be reindexed, early return NoMatch to stop the scanning process.
+                    if action == ActionToOperate::ReindexAllFields {
+                        return Ok((fid, PatternMatch::NoMatch));
+                    }
+
+                    let old_field_metadata = old_fields_ids_map.metadata(fid).unwrap();
+                    let new_field_metadata = new_fields_ids_map.metadata(fid).unwrap();
+
+                    action = match (old_field_metadata, new_field_metadata) {
+                        // At least one field is removed or added from the searchable fields
+                        (
+                            Metadata { searchable: Some(_), .. },
+                            Metadata { searchable: None, .. },
+                        )
+                        | (
+                            Metadata { searchable: None, .. },
+                            Metadata { searchable: Some(_), .. },
+                        ) => ActionToOperate::ReindexAllFields,
+                        _ => action,
+                    };
+
+                    Ok((fid, PatternMatch::Parent))
+                },
+                &mut |_, _, _, _| Ok(()),
+            )?;
+        }
+
+        // Early return when we don't need to index the document
+        if action == ActionToOperate::SkipDocument {
+            return Ok(());
+        }
+
+        let mut del_word_pair_proximity = bumpalo::collections::Vec::new_in(doc_alloc);
+        let mut add_word_pair_proximity = bumpalo::collections::Vec::new_in(doc_alloc);
+
+        // is a vecdequeue, and will be smol, so can stay on the heap for now
+        let mut word_positions: VecDeque<(Rc<str>, u16)> =
+            VecDeque::with_capacity(MAX_DISTANCE as usize);
+
+        process_document_tokens(
+            current_document,
+            // TODO Tokenize must be based on old settings
+            document_tokenizer,
+            &mut word_positions,
+            &mut |field_name| {
+                Ok(old_fields_ids_map.id_with_metadata(field_name).expect("All fields must exist"))
+            },
+            &mut |(w1, w2), prox| {
+                del_word_pair_proximity.push(((w1, w2), prox));
+            },
+        )?;
+
+        process_document_tokens(
+            current_document,
+            // TODO Tokenize must be based on new settings
+            document_tokenizer,
+            &mut word_positions,
+            &mut |field_name| {
+                Ok(new_fields_ids_map.id_with_metadata(field_name).expect("All fields must exist"))
+            },
+            &mut |(w1, w2), prox| {
+                add_word_pair_proximity.push(((w1, w2), prox));
+            },
+        )?;
+
+        let mut key_buffer = bumpalo::collections::Vec::new_in(doc_alloc);
+
+        del_word_pair_proximity.sort_unstable();
+        del_word_pair_proximity.dedup_by(|(k1, _), (k2, _)| k1 == k2);
+        for ((w1, w2), prox) in del_word_pair_proximity.iter() {
+            let key = build_key(*prox, w1, w2, &mut key_buffer);
+            cached_sorter.insert_del_u32(key, document.docid())?;
+        }
+
+        add_word_pair_proximity.sort_unstable();
+        add_word_pair_proximity.dedup_by(|(k1, _), (k2, _)| k1 == k2);
+        for ((w1, w2), prox) in add_word_pair_proximity.iter() {
+            let key = build_key(*prox, w1, w2, &mut key_buffer);
+            cached_sorter.insert_add_u32(key, document.docid())?;
+        }
+
+        Ok(())
+    }
+}

crates/milli/src/update/new/extract/searchable/mod.rs

@@ -2,8 +2,12 @@ mod extract_word_docids;
 mod extract_word_pair_proximity_docids;
 mod tokenize_document;
 
-pub use extract_word_docids::{WordDocidsCaches, WordDocidsExtractors};
-pub use extract_word_pair_proximity_docids::WordPairProximityDocidsExtractor;
+pub use extract_word_docids::{
+    SettingsChangeWordDocidsExtractors, WordDocidsCaches, WordDocidsExtractors,
+};
+pub use extract_word_pair_proximity_docids::{
+    SettingsChangeWordPairProximityDocidsExtractors, WordPairProximityDocidsExtractor,
+};
 
 use crate::attribute_patterns::{match_field_legacy, PatternMatch};
 
@@ -27,3 +31,17 @@ pub fn match_searchable_field(
 
     selection
 }
+
+/// return `true` if the provided `field_name` is a parent of at least one of the fields contained in `searchable`,
+/// or if `searchable` is `None`.
+fn has_searchable_children<I, A>(field_name: &str, searchable: Option<I>) -> bool
+where
+    I: IntoIterator<Item = A>,
+    A: AsRef<str>,
+{
+    searchable.is_none_or(|fields| {
+        fields
+            .into_iter()
+            .any(|attr| match_field_legacy(attr.as_ref(), field_name) == PatternMatch::Parent)
+    })
+}

crates/milli/src/update/new/extract/searchable/tokenize_document.rs

@@ -8,10 +8,7 @@ use crate::update::new::document::Document;
 use crate::update::new::extract::perm_json_p::{
     seek_leaf_values_in_array, seek_leaf_values_in_object, Depth,
 };
-use crate::{
-    FieldId, GlobalFieldsIdsMap, InternalError, LocalizedAttributesRule, Result, UserError,
-    MAX_WORD_LENGTH,
-};
+use crate::{FieldId, InternalError, LocalizedAttributesRule, Result, MAX_WORD_LENGTH};
 
 // todo: should be crate::proximity::MAX_DISTANCE but it has been forgotten
 const MAX_DISTANCE: u32 = 8;
@@ -26,26 +23,25 @@ impl DocumentTokenizer<'_> {
     pub fn tokenize_document<'doc>(
         &self,
         document: impl Document<'doc>,
-        field_id_map: &mut GlobalFieldsIdsMap,
+        should_tokenize: &mut impl FnMut(&str) -> Result<(FieldId, PatternMatch)>,
         token_fn: &mut impl FnMut(&str, FieldId, u16, &str) -> Result<()>,
     ) -> Result<()> {
         let mut field_position = HashMap::new();
-        let mut tokenize_field = |field_name: &str, _depth, value: &Value| {
-            let Some((field_id, meta)) = field_id_map.id_with_metadata_or_insert(field_name) else {
-                return Err(UserError::AttributeLimitReached.into());
-            };
-
-            if meta.is_searchable() {
-                self.tokenize_field(field_id, field_name, value, token_fn, &mut field_position)?;
-            }
-
-            // todo: should be a match on the field_name using `match_field_legacy` function,
-            // but for legacy reasons we iterate over all the fields to fill the field_id_map.
-            Ok(PatternMatch::Match)
-        };
-
         for entry in document.iter_top_level_fields() {
             let (field_name, value) = entry?;
+
+            if let (_, PatternMatch::NoMatch) = should_tokenize(field_name)? {
+                continue;
+            }
+
+            let mut tokenize_field = |field_name: &str, _depth, value: &Value| {
+                let (fid, pattern_match) = should_tokenize(field_name)?;
+                if pattern_match == PatternMatch::Match {
+                    self.tokenize_field(fid, field_name, value, token_fn, &mut field_position)?;
+                }
+                Ok(pattern_match)
+            };
+
             // parse json.
             match serde_json::to_value(value).map_err(InternalError::SerdeJson)? {
                 Value::Object(object) => seek_leaf_values_in_object(
@@ -192,7 +188,7 @@ mod test {
     use super::*;
     use crate::fields_ids_map::metadata::{FieldIdMapWithMetadata, MetadataBuilder};
     use crate::update::new::document::{DocumentFromVersions, Versions};
-    use crate::FieldsIdsMap;
+    use crate::{FieldsIdsMap, GlobalFieldsIdsMap, UserError};
 
     #[test]
     fn test_tokenize_document() {
@@ -231,6 +227,7 @@ mod test {
                 Default::default(),
                 Default::default(),
                 Default::default(),
+                Default::default(),
                 None,
                 None,
                 Default::default(),
@@ -251,15 +248,19 @@ mod test {
         let document = Versions::single(document);
         let document = DocumentFromVersions::new(&document);
 
+        let mut should_tokenize = |field_name: &str| {
+            let Some(field_id) = global_fields_ids_map.id_or_insert(field_name) else {
+                return Err(UserError::AttributeLimitReached.into());
+            };
+
+            Ok((field_id, PatternMatch::Match))
+        };
+
         document_tokenizer
-            .tokenize_document(
-                document,
-                &mut global_fields_ids_map,
-                &mut |_fname, fid, pos, word| {
+            .tokenize_document(document, &mut should_tokenize, &mut |_fname, fid, pos, word| {
                 words.insert([fid, pos], word.to_string());
                 Ok(())
-                },
-            )
+            })
             .unwrap();
 
         snapshot!(format!("{:#?}", words), @r###"

crates/milli/src/update/new/extract/vectors/mod.rs

@@ -1,5 +1,6 @@
 use std::cell::RefCell;
 use std::fmt::Debug;
+use std::sync::RwLock;
 
 use bumpalo::collections::Vec as BVec;
 use bumpalo::Bump;
@@ -27,7 +28,10 @@ use crate::vector::extractor::{
 use crate::vector::session::{EmbedSession, Input, Metadata, OnEmbed};
 use crate::vector::settings::ReindexAction;
 use crate::vector::{Embedding, RuntimeEmbedder, RuntimeEmbedders, RuntimeFragment};
-use crate::{DocumentId, FieldDistribution, InternalError, Result, ThreadPoolNoAbort, UserError};
+use crate::{
+    DocumentId, FieldDistribution, GlobalFieldsIdsMap, InternalError, Result, ThreadPoolNoAbort,
+    UserError,
+};
 
 pub struct EmbeddingExtractor<'a, 'b> {
     embedders: &'a RuntimeEmbedders,
@@ -321,6 +325,15 @@ impl<'extractor, SD: SettingsDelta + Sync> SettingsChangeExtractor<'extractor>
         let old_embedders = self.settings_delta.old_embedders();
         let unused_vectors_distribution = UnusedVectorsDistributionBump::new_in(&context.doc_alloc);
 
+        // We get a reference to the new and old fields ids maps but
+        // note that those are local versions where updates to them
+        // will not be reflected in the database. It's not an issue
+        // because new settings do not generate new fields.
+        let new_fields_ids_map = RwLock::new(self.settings_delta.new_fields_ids_map().clone());
+        let new_fields_ids_map = RefCell::new(GlobalFieldsIdsMap::new(&new_fields_ids_map));
+        let old_fields_ids_map = RwLock::new(self.settings_delta.old_fields_ids_map().clone());
+        let old_fields_ids_map = RefCell::new(GlobalFieldsIdsMap::new(&old_fields_ids_map));
+
         let mut all_chunks = BVec::with_capacity_in(embedders.len(), &context.doc_alloc);
         let embedder_configs = context.index.embedding_configs();
         for (embedder_name, action) in self.settings_delta.embedder_actions().iter() {
@@ -396,6 +409,7 @@ impl<'extractor, SD: SettingsDelta + Sync> SettingsChangeExtractor<'extractor>
                         if !must_regenerate {
                             continue;
                         }
+
                         // we need to regenerate the prompts for the document
                         chunks.settings_change_autogenerated(
                             document.docid(),
@@ -406,7 +420,8 @@ impl<'extractor, SD: SettingsDelta + Sync> SettingsChangeExtractor<'extractor>
                                 context.db_fields_ids_map,
                             )?,
                             self.settings_delta,
-                            context.new_fields_ids_map,
+                            &old_fields_ids_map,
+                            &new_fields_ids_map,
                             &unused_vectors_distribution,
                             old_is_user_provided,
                             fragments_changed,
@@ -442,7 +457,8 @@ impl<'extractor, SD: SettingsDelta + Sync> SettingsChangeExtractor<'extractor>
                                     context.db_fields_ids_map,
                                 )?,
                                 self.settings_delta,
-                                context.new_fields_ids_map,
+                                &old_fields_ids_map,
+                                &new_fields_ids_map,
                                 &unused_vectors_distribution,
                                 old_is_user_provided,
                                 true,
@@ -638,7 +654,8 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> {
         external_docid: &'a str,
         document: D,
         settings_delta: &SD,
-        fields_ids_map: &'a RefCell<crate::GlobalFieldsIdsMap>,
+        old_fields_ids_map: &'a RefCell<GlobalFieldsIdsMap<'a>>,
+        new_fields_ids_map: &'a RefCell<GlobalFieldsIdsMap<'a>>,
         unused_vectors_distribution: &UnusedVectorsDistributionBump<'a>,
         old_is_user_provided: bool,
         full_reindex: bool,
@@ -733,10 +750,17 @@ impl<'a, 'b, 'extractor> Chunks<'a, 'b, 'extractor> {
                     old_embedder.as_ref().map(|old_embedder| &old_embedder.document_template)
                 };
 
-                let extractor =
-                    DocumentTemplateExtractor::new(document_template, doc_alloc, fields_ids_map);
+                let extractor = DocumentTemplateExtractor::new(
+                    document_template,
+                    doc_alloc,
+                    new_fields_ids_map,
+                );
                 let old_extractor = old_document_template.map(|old_document_template| {
-                    DocumentTemplateExtractor::new(old_document_template, doc_alloc, fields_ids_map)
+                    DocumentTemplateExtractor::new(
+                        old_document_template,
+                        doc_alloc,
+                        old_fields_ids_map,
+                    )
                 });
                 let metadata =
                     Metadata { docid, external_docid, extractor_id: extractor.extractor_id() };

crates/milli/src/update/new/indexer/extract.rs

@@ -372,11 +372,10 @@ where
     SD: SettingsDelta + Sync,
 {
     // Create the list of document ids to extract
-    let rtxn = indexing_context.index.read_txn()?;
-    let all_document_ids =
-        indexing_context.index.documents_ids(&rtxn)?.into_iter().collect::<Vec<_>>();
-    let primary_key =
-        primary_key_from_db(indexing_context.index, &rtxn, &indexing_context.db_fields_ids_map)?;
+    let index = indexing_context.index;
+    let rtxn = index.read_txn()?;
+    let all_document_ids = index.documents_ids(&rtxn)?.into_iter().collect::<Vec<_>>();
+    let primary_key = primary_key_from_db(index, &rtxn, &indexing_context.db_fields_ids_map)?;
     let documents = DocumentsIndentifiers::new(&all_document_ids, primary_key);
 
     let span =
@@ -391,6 +390,133 @@ where
         extractor_allocs,
     )?;
 
+    {
+        let WordDocidsCaches {
+            word_docids,
+            word_fid_docids,
+            exact_word_docids,
+            word_position_docids,
+            fid_word_count_docids,
+        } = {
+            let span = tracing::trace_span!(target: "indexing::documents::extract", "word_docids");
+            let _entered = span.enter();
+            SettingsChangeWordDocidsExtractors::run_extraction(
+                settings_delta,
+                &documents,
+                indexing_context,
+                extractor_allocs,
+                IndexingStep::ExtractingWords,
+            )?
+        };
+
+        indexing_context.progress.update_progress(IndexingStep::MergingWordCaches);
+
+        {
+            let span = tracing::trace_span!(target: "indexing::documents::merge", "word_docids");
+            let _entered = span.enter();
+            indexing_context.progress.update_progress(MergingWordCache::WordDocids);
+
+            merge_and_send_docids(
+                word_docids,
+                index.word_docids.remap_types(),
+                index,
+                extractor_sender.docids::<WordDocids>(),
+                &indexing_context.must_stop_processing,
+            )?;
+        }
+
+        {
+            let span =
+                tracing::trace_span!(target: "indexing::documents::merge", "word_fid_docids");
+            let _entered = span.enter();
+            indexing_context.progress.update_progress(MergingWordCache::WordFieldIdDocids);
+
+            merge_and_send_docids(
+                word_fid_docids,
+                index.word_fid_docids.remap_types(),
+                index,
+                extractor_sender.docids::<WordFidDocids>(),
+                &indexing_context.must_stop_processing,
+            )?;
+        }
+
+        {
+            let span =
+                tracing::trace_span!(target: "indexing::documents::merge", "exact_word_docids");
+            let _entered = span.enter();
+            indexing_context.progress.update_progress(MergingWordCache::ExactWordDocids);
+
+            merge_and_send_docids(
+                exact_word_docids,
+                index.exact_word_docids.remap_types(),
+                index,
+                extractor_sender.docids::<ExactWordDocids>(),
+                &indexing_context.must_stop_processing,
+            )?;
+        }
+
+        {
+            let span =
+                tracing::trace_span!(target: "indexing::documents::merge", "word_position_docids");
+            let _entered = span.enter();
+            indexing_context.progress.update_progress(MergingWordCache::WordPositionDocids);
+
+            merge_and_send_docids(
+                word_position_docids,
+                index.word_position_docids.remap_types(),
+                index,
+                extractor_sender.docids::<WordPositionDocids>(),
+                &indexing_context.must_stop_processing,
+            )?;
+        }
+
+        {
+            let span =
+                tracing::trace_span!(target: "indexing::documents::merge", "fid_word_count_docids");
+            let _entered = span.enter();
+            indexing_context.progress.update_progress(MergingWordCache::FieldIdWordCountDocids);
+
+            merge_and_send_docids(
+                fid_word_count_docids,
+                index.field_id_word_count_docids.remap_types(),
+                index,
+                extractor_sender.docids::<FidWordCountDocids>(),
+                &indexing_context.must_stop_processing,
+            )?;
+        }
+    }
+
+    // Run the proximity extraction only if the precision is ByWord.
+    let new_proximity_precision = settings_delta.new_proximity_precision();
+    if *new_proximity_precision == ProximityPrecision::ByWord {
+        let caches = {
+            let span = tracing::trace_span!(target: "indexing::documents::extract", "word_pair_proximity_docids");
+            let _entered = span.enter();
+
+            SettingsChangeWordPairProximityDocidsExtractors::run_extraction(
+                settings_delta,
+                &documents,
+                indexing_context,
+                extractor_allocs,
+                IndexingStep::ExtractingWordProximity,
+            )?
+        };
+
+        {
+            let span = tracing::trace_span!(target: "indexing::documents::merge", "word_pair_proximity_docids");
+            let _entered = span.enter();
+            indexing_context.progress.update_progress(IndexingStep::MergingWordProximity);
+
+            merge_and_send_docids(
+                caches,
+                index.word_pair_proximity_docids.remap_types(),
+                index,
+                extractor_sender.docids::<WordPairProximityDocids>(),
+                &indexing_context.must_stop_processing,
+            )?;
+        }
+    }
+
     'vectors: {
         if settings_delta.embedder_actions().is_empty() {
             break 'vectors;

crates/milli/src/update/new/indexer/mod.rs

@@ -1,4 +1,4 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, BTreeSet};
 use std::sync::atomic::AtomicBool;
 use std::sync::{Arc, Once, RwLock};
 use std::thread::{self, Builder};
@@ -8,9 +8,11 @@ use document_changes::{DocumentChanges, IndexingContext};
 pub use document_deletion::DocumentDeletion;
 pub use document_operation::{DocumentOperation, PayloadStats};
 use hashbrown::HashMap;
-use heed::{RoTxn, RwTxn};
+use heed::types::DecodeIgnore;
+use heed::{BytesDecode, Database, RoTxn, RwTxn};
 pub use partial_dump::PartialDump;
 pub use post_processing::recompute_word_fst_from_word_docids_database;
+pub use settings_changes::settings_change_extract;
 pub use update_by_function::UpdateByFunction;
 pub use write::ChannelCongestion;
 use write::{build_vectors, update_index, write_to_db};
@@ -20,12 +22,18 @@ use super::steps::IndexingStep;
 use super::thread_local::ThreadLocal;
 use crate::documents::PrimaryKey;
 use crate::fields_ids_map::metadata::{FieldIdMapWithMetadata, MetadataBuilder};
+use crate::heed_codec::StrBEU16Codec;
 use crate::progress::{EmbedderStats, Progress};
+use crate::proximity::ProximityPrecision;
+use crate::update::new::steps::SettingsIndexerStep;
+use crate::update::new::FacetFieldIdsDelta;
 use crate::update::settings::SettingsDelta;
 use crate::update::GrenadParameters;
 use crate::vector::settings::{EmbedderAction, RemoveFragments, WriteBackToDocuments};
 use crate::vector::{Embedder, RuntimeEmbedders, VectorStore};
-use crate::{FieldsIdsMap, GlobalFieldsIdsMap, Index, InternalError, Result, ThreadPoolNoAbort};
+use crate::{
+    Error, FieldsIdsMap, GlobalFieldsIdsMap, Index, InternalError, Result, ThreadPoolNoAbort,
+};
 
 #[cfg(not(feature = "enterprise"))]
 pub mod community_edition;
@@ -242,6 +250,20 @@ where
     SD: SettingsDelta + Sync,
 {
     delete_old_embedders_and_fragments(wtxn, index, settings_delta)?;
+    delete_old_fid_based_databases(wtxn, index, settings_delta, must_stop_processing, progress)?;
+
+    // Clear word_pair_proximity if byWord to byAttribute
+    let old_proximity_precision = settings_delta.old_proximity_precision();
+    let new_proximity_precision = settings_delta.new_proximity_precision();
+    if *old_proximity_precision == ProximityPrecision::ByWord
+        && *new_proximity_precision == ProximityPrecision::ByAttribute
+    {
+        index.word_pair_proximity_docids.clear(wtxn)?;
+    }
+
+    // TODO delete useless searchable databases
+    //      - Clear fid_prefix_* in the post processing
+    //      - clear the prefix + fid_prefix if setting `PrefixSearch` is enabled
 
     let mut bbbuffers = Vec::new();
     let finished_extraction = AtomicBool::new(false);
@@ -300,6 +322,8 @@ where
                 .unwrap()
             })?;
 
+        let global_fields_ids_map = GlobalFieldsIdsMap::new(&new_fields_ids_map);
+
         let new_embedders = settings_delta.new_embedders();
         let embedder_actions = settings_delta.embedder_actions();
         let index_embedder_category_ids = settings_delta.new_embedder_category_id();
@@ -334,6 +358,18 @@ where
         })
         .unwrap()?;
 
+        pool.install(|| {
+            // WARN When implementing the facets don't forget this
+            let facet_field_ids_delta = FacetFieldIdsDelta::new(0, 0);
+            post_processing::post_process(
+                indexing_context,
+                wtxn,
+                global_fields_ids_map,
+                facet_field_ids_delta,
+            )
+        })
+        .unwrap()?;
+
         indexing_context.progress.update_progress(IndexingStep::BuildingGeoJson);
         index.cellulite.build(
             wtxn,
@@ -463,6 +499,106 @@ where
     Ok(())
 }
 
+/// Deletes entries refering the provided
+/// fids from the fid-based databases.
+fn delete_old_fid_based_databases<SD, MSP>(
+    wtxn: &mut RwTxn<'_>,
+    index: &Index,
+    settings_delta: &SD,
+    must_stop_processing: &MSP,
+    progress: &Progress,
+) -> Result<()>
+where
+    SD: SettingsDelta + Sync,
+    MSP: Fn() -> bool + Sync,
+{
+    let fids_to_delete: Option<BTreeSet<_>> = {
+        let rtxn = index.read_txn()?;
+        let fields_ids_map = index.fields_ids_map(&rtxn)?;
+        let old_searchable_attributes = settings_delta.old_searchable_attributes().as_ref();
+        let new_searchable_attributes = settings_delta.new_searchable_attributes().as_ref();
+        old_searchable_attributes.zip(new_searchable_attributes).map(|(old, new)| {
+            old.iter()
+                // Ignore the field if it is not searchable anymore
+                // or if it was never referenced in any document
+                .filter_map(|name| if new.contains(name) { None } else { fields_ids_map.id(name) })
+                .collect()
+        })
+    };
+
+    let Some(fids_to_delete) = fids_to_delete else {
+        return Ok(());
+    };
+
+    progress.update_progress(SettingsIndexerStep::DeletingOldWordFidDocids);
+    delete_old_word_fid_docids(wtxn, index.word_fid_docids, must_stop_processing, &fids_to_delete)?;
+
+    progress.update_progress(SettingsIndexerStep::DeletingOldFidWordCountDocids);
+    delete_old_fid_word_count_docids(wtxn, index, must_stop_processing, &fids_to_delete)?;
+
+    progress.update_progress(SettingsIndexerStep::DeletingOldWordPrefixFidDocids);
+    delete_old_word_fid_docids(
+        wtxn,
+        index.word_prefix_fid_docids,
+        must_stop_processing,
+        &fids_to_delete,
+    )?;
+
+    Ok(())
+}
+
+fn delete_old_word_fid_docids<'txn, MSP, DC>(
+    wtxn: &mut RwTxn<'txn>,
+    database: Database<StrBEU16Codec, DC>,
+    must_stop_processing: &MSP,
+    fids_to_delete: &BTreeSet<u16>,
+) -> Result<(), Error>
+where
+    MSP: Fn() -> bool + Sync,
+    DC: BytesDecode<'txn>,
+{
+    let mut iter = database.iter_mut(wtxn)?.remap_data_type::<DecodeIgnore>();
+    while let Some(((_word, fid), ())) = iter.next().transpose()? {
+        // TODO should I call it that often?
+        if must_stop_processing() {
+            return Err(Error::InternalError(InternalError::AbortedIndexation));
+        }
+
+        if fids_to_delete.contains(&fid) {
+            // safety: We don't keep any references to the data.
+            unsafe { iter.del_current()? };
+        }
+    }
+
+    Ok(())
+}
+
+fn delete_old_fid_word_count_docids<MSP>(
+    wtxn: &mut RwTxn<'_>,
+    index: &Index,
+    must_stop_processing: &MSP,
+    fids_to_delete: &BTreeSet<u16>,
+) -> Result<(), Error>
+where
+    MSP: Fn() -> bool + Sync,
+{
+    let db = index.field_id_word_count_docids.remap_data_type::<DecodeIgnore>();
+    for &fid_to_delete in fids_to_delete {
+        if must_stop_processing() {
+            return Err(Error::InternalError(InternalError::AbortedIndexation));
+        }
+
+        let mut iter = db.prefix_iter_mut(wtxn, &(fid_to_delete, 0))?;
+        while let Some(((fid, _word_count), ())) = iter.next().transpose()? {
+            debug_assert_eq!(fid, fid_to_delete);
+            // safety: We don't keep any references to the data.
+            unsafe { iter.del_current()? };
+        }
+    }
+
+    Ok(())
+}
+
 fn indexer_memory_settings(
     current_num_threads: usize,
     grenad_parameters: GrenadParameters,

crates/milli/src/update/new/steps.rs

@@ -28,6 +28,9 @@ make_enum_progress! {
         ChangingVectorStore,
         UsingStableIndexer,
         UsingExperimentalIndexer,
+        DeletingOldWordFidDocids,
+        DeletingOldFidWordCountDocids,
+        DeletingOldWordPrefixFidDocids,
     }
 }
 

crates/milli/src/update/settings.rs

@@ -48,10 +48,11 @@ use crate::{
     ChannelCongestion, FieldId, FilterableAttributesRule, Index, LocalizedAttributesRule, Result,
 };
 
-#[derive(Debug, Clone, PartialEq, Eq, Copy)]
+#[derive(Default, Debug, Clone, PartialEq, Eq, Copy)]
 pub enum Setting<T> {
     Set(T),
     Reset,
+    #[default]
     NotSet,
 }
 
@@ -71,12 +72,6 @@ where
     }
 }
 
-impl<T> Default for Setting<T> {
-    fn default() -> Self {
-        Self::NotSet
-    }
-}
-
 impl<T> Setting<T> {
     pub fn set(self) -> Option<T> {
         match self {
@@ -1589,33 +1584,33 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
 
         // only use the new indexer when only the embedder possibly changed
         if let Self {
-            searchable_fields: Setting::NotSet,
+            searchable_fields: _,
             displayed_fields: Setting::NotSet,
             filterable_fields: Setting::NotSet,
             sortable_fields: Setting::NotSet,
             criteria: Setting::NotSet,
-            stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
+            stop_words: Setting::NotSet, // TODO (require force reindexing of searchables)
+            non_separator_tokens: Setting::NotSet, // TODO (require force reindexing of searchables)
+            separator_tokens: Setting::NotSet, // TODO (require force reindexing of searchables)
+            dictionary: Setting::NotSet, // TODO (require force reindexing of searchables)
             distinct_field: Setting::NotSet,
             synonyms: Setting::NotSet,
             primary_key: Setting::NotSet,
             authorize_typos: Setting::NotSet,
             min_word_len_two_typos: Setting::NotSet,
             min_word_len_one_typo: Setting::NotSet,
-            exact_words: Setting::NotSet,
-            exact_attributes: Setting::NotSet,
+            exact_words: Setting::NotSet, // TODO (require force reindexing of searchables)
+            exact_attributes: _,
             max_values_per_facet: Setting::NotSet,
             sort_facet_values_by: Setting::NotSet,
             pagination_max_total_hits: Setting::NotSet,
-            proximity_precision: Setting::NotSet,
+            proximity_precision: _,
             embedder_settings: _,
             search_cutoff: Setting::NotSet,
-            localized_attributes_rules: Setting::NotSet,
-            prefix_search: Setting::NotSet,
+            localized_attributes_rules: Setting::NotSet, // TODO to start with
+            prefix_search: Setting::NotSet,              // TODO continue with this
             facet_search: Setting::NotSet,
-            disable_on_numbers: Setting::NotSet,
+            disable_on_numbers: Setting::NotSet, // TODO (require force reindexing of searchables)
             chat: Setting::NotSet,
             vector_store: Setting::NotSet,
             wtxn: _,
@@ -1632,10 +1627,12 @@ impl<'a, 't, 'i> Settings<'a, 't, 'i> {
             // Update index settings
             let embedding_config_updates = self.update_embedding_configs()?;
             self.update_user_defined_searchable_attributes()?;
+            self.update_exact_attributes()?;
+            self.update_proximity_precision()?;
 
-            let mut new_inner_settings =
-                InnerIndexSettings::from_index(self.index, self.wtxn, None)?;
-            new_inner_settings.recompute_searchables(self.wtxn, self.index)?;
+            // Note that we don't need to update the searchables here,
+            // as it will be done after the settings update.
+            let new_inner_settings = InnerIndexSettings::from_index(self.index, self.wtxn, None)?;
 
             let primary_key_id = self
                 .index
@@ -2062,9 +2059,12 @@ impl InnerIndexSettings {
         let sortable_fields = index.sortable_fields(rtxn)?;
         let asc_desc_fields = index.asc_desc_fields(rtxn)?;
         let distinct_field = index.distinct_field(rtxn)?.map(|f| f.to_string());
-        let user_defined_searchable_attributes = index
-            .user_defined_searchable_fields(rtxn)?
-            .map(|fields| fields.into_iter().map(|f| f.to_string()).collect());
+        let user_defined_searchable_attributes = match index.user_defined_searchable_fields(rtxn)? {
+            Some(fields) if fields.contains(&"*") => None,
+            Some(fields) => Some(fields.into_iter().map(|f| f.to_string()).collect()),
+            None => None,
+        };
+
         let builder = MetadataBuilder::from_index(index, rtxn)?;
         let fields_ids_map = FieldIdMapWithMetadata::new(fields_ids_map, builder);
         let disabled_typos_terms = index.disabled_typos_terms(rtxn)?;
@@ -2578,8 +2578,20 @@ fn deserialize_sub_embedder(
 /// Implement this trait for the settings delta type.
 /// This is used in the new settings update flow and will allow to easily replace the old settings delta type: `InnerIndexSettingsDiff`.
 pub trait SettingsDelta {
-    fn new_embedders(&self) -> &RuntimeEmbedders;
+    fn old_fields_ids_map(&self) -> &FieldIdMapWithMetadata;
+    fn new_fields_ids_map(&self) -> &FieldIdMapWithMetadata;
+
+    fn old_searchable_attributes(&self) -> &Option<Vec<String>>;
+    fn new_searchable_attributes(&self) -> &Option<Vec<String>>;
+
+    fn old_disabled_typos_terms(&self) -> &DisabledTyposTerms;
+    fn new_disabled_typos_terms(&self) -> &DisabledTyposTerms;
+
+    fn old_proximity_precision(&self) -> &ProximityPrecision;
+    fn new_proximity_precision(&self) -> &ProximityPrecision;
+
     fn old_embedders(&self) -> &RuntimeEmbedders;
+    fn new_embedders(&self) -> &RuntimeEmbedders;
     fn new_embedder_category_id(&self) -> &HashMap<String, u8>;
     fn embedder_actions(&self) -> &BTreeMap<String, EmbedderAction>;
     fn try_for_each_fragment_diff<F, E>(
@@ -2589,7 +2601,6 @@ pub trait SettingsDelta {
     ) -> std::result::Result<(), E>
     where
         F: FnMut(FragmentDiff) -> std::result::Result<(), E>;
-    fn new_fields_ids_map(&self) -> &FieldIdMapWithMetadata;
 }
 
 pub struct FragmentDiff<'a> {
@@ -2598,26 +2609,47 @@ pub struct FragmentDiff<'a> {
 }
 
 impl SettingsDelta for InnerIndexSettingsDiff {
-    fn new_embedders(&self) -> &RuntimeEmbedders {
-        &self.new.runtime_embedders
+    fn old_fields_ids_map(&self) -> &FieldIdMapWithMetadata {
+        &self.old.fields_ids_map
+    }
+    fn new_fields_ids_map(&self) -> &FieldIdMapWithMetadata {
+        &self.new.fields_ids_map
+    }
+
+    fn old_searchable_attributes(&self) -> &Option<Vec<String>> {
+        &self.old.user_defined_searchable_attributes
+    }
+    fn new_searchable_attributes(&self) -> &Option<Vec<String>> {
+        &self.new.user_defined_searchable_attributes
+    }
+
+    fn old_disabled_typos_terms(&self) -> &DisabledTyposTerms {
+        &self.old.disabled_typos_terms
+    }
+    fn new_disabled_typos_terms(&self) -> &DisabledTyposTerms {
+        &self.new.disabled_typos_terms
+    }
+
+    fn old_proximity_precision(&self) -> &ProximityPrecision {
+        &self.old.proximity_precision
+    }
+    fn new_proximity_precision(&self) -> &ProximityPrecision {
+        &self.new.proximity_precision
     }
 
     fn old_embedders(&self) -> &RuntimeEmbedders {
         &self.old.runtime_embedders
     }
+    fn new_embedders(&self) -> &RuntimeEmbedders {
+        &self.new.runtime_embedders
+    }
 
     fn new_embedder_category_id(&self) -> &HashMap<String, u8> {
         &self.new.embedder_category_id
     }
-
     fn embedder_actions(&self) -> &BTreeMap<String, EmbedderAction> {
         &self.embedding_config_updates
     }
-
-    fn new_fields_ids_map(&self) -> &FieldIdMapWithMetadata {
-        &self.new.fields_ids_map
-    }
-
     fn try_for_each_fragment_diff<F, E>(
         &self,
         embedder_name: &str,

crates/milli/src/update/test_settings.rs

@@ -14,28 +14,21 @@ fn set_and_reset_searchable_fields() {
     let index = TempIndex::new();
 
     // First we send 3 documents with ids from 1 to 3.
-    let mut wtxn = index.write_txn().unwrap();
-
     index
-        .add_documents_using_wtxn(
-            &mut wtxn,
-            documents!([
+        .add_documents(documents!([
             { "id": 1, "name": "kevin", "age": 23 },
             { "id": 2, "name": "kevina", "age": 21},
             { "id": 3, "name": "benoit", "age": 34 }
-            ]),
-        )
+        ]))
         .unwrap();
 
     // We change the searchable fields to be the "name" field only.
     index
-        .update_settings_using_wtxn(&mut wtxn, |settings| {
+        .update_settings(|settings| {
             settings.set_searchable_fields(vec!["name".into()]);
         })
         .unwrap();
 
-    wtxn.commit().unwrap();
-
     db_snap!(index, fields_ids_map, @r###"
     0   id               |
     1   name             |

crates/milli/src/update/upgrade/mod.rs

@@ -5,103 +5,36 @@ mod v1_15;
 mod v1_16;
 
 use heed::RwTxn;
-use v1_12::{V1_12_3_To_V1_13_0, V1_12_To_V1_12_3};
-use v1_13::{V1_13_0_To_V1_13_1, V1_13_1_To_Latest_V1_13};
-use v1_14::Latest_V1_13_To_Latest_V1_14;
-use v1_15::Latest_V1_14_To_Latest_V1_15;
-use v1_16::Latest_V1_15_To_V1_16_0;
+use v1_12::{FixFieldDistribution, RecomputeStats};
+use v1_13::AddNewStats;
+use v1_14::UpgradeArroyVersion;
+use v1_15::RecomputeWordFst;
+use v1_16::SwitchToMultimodal;
 
 use crate::constants::{VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH};
 use crate::progress::{Progress, VariableNameStep};
 use crate::{Index, InternalError, Result};
 
 trait UpgradeIndex {
+    /// Returns `true` if `upgrade` should be called when the index started with version `initial_version`.
+    fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool;
+
     /// Returns `true` if the index scheduler must regenerate its cached stats.
-    fn upgrade(
-        &self,
-        wtxn: &mut RwTxn,
-        index: &Index,
-        original: (u32, u32, u32),
-        progress: Progress,
-    ) -> Result<bool>;
-    fn target_version(&self) -> (u32, u32, u32);
+    fn upgrade(&self, wtxn: &mut RwTxn, index: &Index, progress: Progress) -> Result<bool>;
+
+    /// Description of the upgrade for progress display purposes.
+    fn description(&self) -> &'static str;
 }
 
 const UPGRADE_FUNCTIONS: &[&dyn UpgradeIndex] = &[
-    &V1_12_To_V1_12_3 {},
-    &V1_12_3_To_V1_13_0 {},
-    &V1_13_0_To_V1_13_1 {},
-    &V1_13_1_To_Latest_V1_13 {},
-    &Latest_V1_13_To_Latest_V1_14 {},
-    &Latest_V1_14_To_Latest_V1_15 {},
-    &Latest_V1_15_To_V1_16_0 {},
-    &ToTargetNoOp { target: (1, 18, 0) },
-    &ToTargetNoOp { target: (1, 19, 0) },
-    &ToTargetNoOp { target: (1, 20, 0) },
-    &ToTargetNoOp { target: (1, 21, 0) },
-    &ToTargetNoOp { target: (1, 22, 0) },
-    &ToTargetNoOp { target: (1, 23, 0) },
-    &ToTargetNoOp { target: (1, 24, 0) },
-    &ToTargetNoOp { target: (1, 25, 0) },
-    &ToTargetNoOp { target: (1, 26, 0) },
-    &ToTargetNoOp { target: (1, 27, 0) },
-    &ToTargetNoOp { target: (1, 28, 0) },
-    // This is the last upgrade function, it will be called when the index is up to date.
-    // any other upgrade function should be added before this one.
-    &ToCurrentNoOp {},
+    &FixFieldDistribution {},
+    &RecomputeStats {},
+    &AddNewStats {},
+    &UpgradeArroyVersion {},
+    &RecomputeWordFst {},
+    &SwitchToMultimodal {},
 ];
 
-/// Causes a compile-time error if the argument is not in range of `0..UPGRADE_FUNCTIONS.len()`
-macro_rules! function_index {
-    ($start:expr) => {{
-        const _CHECK_INDEX: () = {
-            if $start >= $crate::update::upgrade::UPGRADE_FUNCTIONS.len() {
-                panic!("upgrade functions out of range")
-            }
-        };
-
-        $start
-    }};
-}
-
-const fn start(from: (u32, u32, u32)) -> Option<usize> {
-    let start = match from {
-        (1, 12, 0..=2) => function_index!(0),
-        (1, 12, 3..) => function_index!(1),
-        (1, 13, 0) => function_index!(2),
-        (1, 13, _) => function_index!(4),
-        (1, 14, _) => function_index!(5),
-        // We must handle the current version in the match because in case of a failure some index may have been upgraded but not other.
-        (1, 15, _) => function_index!(6),
-        (1, 16, _) | (1, 17, _) => function_index!(7),
-        (1, 18, _) => function_index!(8),
-        (1, 19, _) => function_index!(9),
-        (1, 20, _) => function_index!(10),
-        (1, 21, _) => function_index!(11),
-        (1, 22, _) => function_index!(12),
-        (1, 23, _) => function_index!(13),
-        (1, 24, _) => function_index!(14),
-        (1, 25, _) => function_index!(15),
-        (1, 26, _) => function_index!(16),
-        (1, 27, _) => function_index!(17),
-        (1, 28, _) => function_index!(18),
-        // We deliberately don't add a placeholder with (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH) here to force manually
-        // considering dumpless upgrade.
-        (_major, _minor, _patch) => return None,
-    };
-
-    Some(start)
-}
-
-/// Causes a compile-time error if the latest package cannot be upgraded.
-///
-/// This serves as a reminder to consider the proper dumpless upgrade implementation when changing the package version.
-const _CHECK_PACKAGE_CAN_UPGRADE: () = {
-    if start((VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)).is_none() {
-        panic!("cannot upgrade from latest package version")
-    }
-};
-
 /// Return true if the cached stats of the index must be regenerated
 pub fn upgrade<MSP>(
     wtxn: &mut RwTxn,
@@ -113,79 +46,34 @@ pub fn upgrade<MSP>(
 where
     MSP: Fn() -> bool + Sync,
 {
-    let from = index.get_version(wtxn)?.unwrap_or(db_version);
+    let upgrade_functions = UPGRADE_FUNCTIONS;
 
-    let start =
-        start(from).ok_or_else(|| InternalError::CannotUpgradeToVersion(from.0, from.1, from.2))?;
+    let initial_version = index.get_version(wtxn)?.unwrap_or(db_version);
 
     enum UpgradeVersion {}
-    let upgrade_path = &UPGRADE_FUNCTIONS[start..];
 
-    let mut current_version = from;
     let mut regenerate_stats = false;
-    for (i, upgrade) in upgrade_path.iter().enumerate() {
+    for (i, upgrade) in upgrade_functions.iter().enumerate() {
         if (must_stop_processing)() {
             return Err(crate::Error::InternalError(InternalError::AbortedIndexation));
         }
-        let target = upgrade.target_version();
+        if upgrade.must_upgrade(initial_version) {
+            regenerate_stats |= upgrade.upgrade(wtxn, index, progress.clone())?;
             progress.update_progress(VariableNameStep::<UpgradeVersion>::new(
-            format!(
-                "Upgrading from v{}.{}.{} to v{}.{}.{}",
-                current_version.0,
-                current_version.1,
-                current_version.2,
-                target.0,
-                target.1,
-                target.2
-            ),
+                upgrade.description(),
                 i as u32,
-            upgrade_path.len() as u32,
+                upgrade_functions.len() as u32,
+            ));
+        } else {
+            progress.update_progress(VariableNameStep::<UpgradeVersion>::new(
+                "Skipping migration that must not be applied",
+                i as u32,
+                upgrade_functions.len() as u32,
             ));
-        regenerate_stats |= upgrade.upgrade(wtxn, index, from, progress.clone())?;
-        index.put_version(wtxn, target)?;
-        current_version = target;
         }
+    }
+
+    index.put_version(wtxn, (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH))?;
 
     Ok(regenerate_stats)
 }
-
-#[allow(non_camel_case_types)]
-struct ToCurrentNoOp {}
-
-impl UpgradeIndex for ToCurrentNoOp {
-    fn upgrade(
-        &self,
-        _wtxn: &mut RwTxn,
-        _index: &Index,
-        _original: (u32, u32, u32),
-        _progress: Progress,
-    ) -> Result<bool> {
-        Ok(false)
-    }
-
-    fn target_version(&self) -> (u32, u32, u32) {
-        (VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)
-    }
-}
-
-/// Perform no operation during the upgrade except changing to the specified target version.
-#[allow(non_camel_case_types)]
-struct ToTargetNoOp {
-    pub target: (u32, u32, u32),
-}
-
-impl UpgradeIndex for ToTargetNoOp {
-    fn upgrade(
-        &self,
-        _wtxn: &mut RwTxn,
-        _index: &Index,
-        _original: (u32, u32, u32),
-        _progress: Progress,
-    ) -> Result<bool> {
-        Ok(false)
-    }
-
-    fn target_version(&self) -> (u32, u32, u32) {
-        self.target
-    }
-}

crates/milli/src/update/upgrade/v1_12.rs

@@ -4,17 +4,10 @@ use super::UpgradeIndex;
 use crate::progress::Progress;
 use crate::{make_enum_progress, Index, Result};
 
-#[allow(non_camel_case_types)]
-pub(super) struct V1_12_To_V1_12_3 {}
+pub(super) struct FixFieldDistribution {}
 
-impl UpgradeIndex for V1_12_To_V1_12_3 {
-    fn upgrade(
-        &self,
-        wtxn: &mut RwTxn,
-        index: &Index,
-        _original: (u32, u32, u32),
-        progress: Progress,
-    ) -> Result<bool> {
+impl UpgradeIndex for FixFieldDistribution {
+    fn upgrade(&self, wtxn: &mut RwTxn, index: &Index, progress: Progress) -> Result<bool> {
         make_enum_progress! {
             enum FieldDistribution {
                 RebuildingFieldDistribution,
@@ -25,27 +18,28 @@ impl UpgradeIndex for V1_12_To_V1_12_3 {
         Ok(true)
     }
 
-    fn target_version(&self) -> (u32, u32, u32) {
-        (1, 12, 3)
+    fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool {
+        initial_version < (1, 12, 3)
+    }
+
+    fn description(&self) -> &'static str {
+        "Recomputing field distribution which was wrong before v1.12.3"
     }
 }
 
-#[allow(non_camel_case_types)]
-pub(super) struct V1_12_3_To_V1_13_0 {}
+pub(super) struct RecomputeStats {}
 
-impl UpgradeIndex for V1_12_3_To_V1_13_0 {
-    fn upgrade(
-        &self,
-        _wtxn: &mut RwTxn,
-        _index: &Index,
-        _original: (u32, u32, u32),
-        _progress: Progress,
-    ) -> Result<bool> {
+impl UpgradeIndex for RecomputeStats {
+    fn upgrade(&self, _wtxn: &mut RwTxn, _index: &Index, _progress: Progress) -> Result<bool> {
         // recompute the indexes stats
         Ok(true)
     }
 
-    fn target_version(&self) -> (u32, u32, u32) {
-        (1, 13, 0)
+    fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool {
+        initial_version < (1, 13, 0)
+    }
+
+    fn description(&self) -> &'static str {
+        "Recomputing stats"
     }
 }

crates/milli/src/update/upgrade/v1_13.rs

@@ -5,17 +5,10 @@ use crate::database_stats::DatabaseStats;
 use crate::progress::Progress;
 use crate::{make_enum_progress, Index, Result};
 
-#[allow(non_camel_case_types)]
-pub(super) struct V1_13_0_To_V1_13_1();
+pub(super) struct AddNewStats();
 
-impl UpgradeIndex for V1_13_0_To_V1_13_1 {
-    fn upgrade(
-        &self,
-        wtxn: &mut RwTxn,
-        index: &Index,
-        _original: (u32, u32, u32),
-        progress: Progress,
-    ) -> Result<bool> {
+impl UpgradeIndex for AddNewStats {
+    fn upgrade(&self, wtxn: &mut RwTxn, index: &Index, progress: Progress) -> Result<bool> {
         make_enum_progress! {
             enum DocumentsStats {
                 CreatingDocumentsStats,
@@ -30,26 +23,11 @@ impl UpgradeIndex for V1_13_0_To_V1_13_1 {
         Ok(true)
     }
 
-    fn target_version(&self) -> (u32, u32, u32) {
-        (1, 13, 1)
-    }
+    fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool {
+        initial_version < (1, 13, 1)
     }
 
-#[allow(non_camel_case_types)]
-pub(super) struct V1_13_1_To_Latest_V1_13();
-
-impl UpgradeIndex for V1_13_1_To_Latest_V1_13 {
-    fn upgrade(
-        &self,
-        _wtxn: &mut RwTxn,
-        _index: &Index,
-        _original: (u32, u32, u32),
-        _progress: Progress,
-    ) -> Result<bool> {
-        Ok(false)
-    }
-
-    fn target_version(&self) -> (u32, u32, u32) {
-        (1, 13, 3)
+    fn description(&self) -> &'static str {
+        "Computing newly introduced document stats"
     }
 }

crates/milli/src/update/upgrade/v1_14.rs

@@ -5,17 +5,10 @@ use super::UpgradeIndex;
 use crate::progress::Progress;
 use crate::{make_enum_progress, Index, Result};
 
-#[allow(non_camel_case_types)]
-pub(super) struct Latest_V1_13_To_Latest_V1_14();
+pub(super) struct UpgradeArroyVersion();
 
-impl UpgradeIndex for Latest_V1_13_To_Latest_V1_14 {
-    fn upgrade(
-        &self,
-        wtxn: &mut RwTxn,
-        index: &Index,
-        _original: (u32, u32, u32),
-        progress: Progress,
-    ) -> Result<bool> {
+impl UpgradeIndex for UpgradeArroyVersion {
+    fn upgrade(&self, wtxn: &mut RwTxn, index: &Index, progress: Progress) -> Result<bool> {
         make_enum_progress! {
             enum VectorStore {
                 UpdateInternalVersions,
@@ -35,7 +28,11 @@ impl UpgradeIndex for Latest_V1_13_To_Latest_V1_14 {
         Ok(false)
     }
 
-    fn target_version(&self) -> (u32, u32, u32) {
-        (1, 14, 0)
+    fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool {
+        initial_version < (1, 14, 0)
+    }
+
+    fn description(&self) -> &'static str {
+        "Updating vector store with an internal version"
     }
 }

crates/milli/src/update/upgrade/v1_15.rs

@@ -7,25 +7,21 @@ use crate::progress::Progress;
 use crate::update::new::indexer::recompute_word_fst_from_word_docids_database;
 use crate::{Index, Result};
 
-#[allow(non_camel_case_types)]
-pub(super) struct Latest_V1_14_To_Latest_V1_15();
+pub(super) struct RecomputeWordFst();
 
-impl UpgradeIndex for Latest_V1_14_To_Latest_V1_15 {
-    fn upgrade(
-        &self,
-        wtxn: &mut RwTxn,
-        index: &Index,
-        _original: (u32, u32, u32),
-        progress: Progress,
-    ) -> Result<bool> {
+impl UpgradeIndex for RecomputeWordFst {
+    fn upgrade(&self, wtxn: &mut RwTxn, index: &Index, progress: Progress) -> Result<bool> {
         // Recompute the word FST from the word docids database.
         recompute_word_fst_from_word_docids_database(index, wtxn, &progress)?;
 
         Ok(false)
     }
+    fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool {
+        initial_version < (1, 15, 0)
+    }
 
-    fn target_version(&self) -> (u32, u32, u32) {
-        (1, 15, 0)
+    fn description(&self) -> &'static str {
+        "Recomputing word FST from word docids database as it was wrong before v1.15.0"
     }
 }
 

crates/milli/src/update/upgrade/v1_16.rs

@@ -6,17 +6,10 @@ use crate::progress::Progress;
 use crate::vector::db::{EmbedderInfo, EmbeddingStatus};
 use crate::{Index, InternalError, Result};
 
-#[allow(non_camel_case_types)]
-pub(super) struct Latest_V1_15_To_V1_16_0();
+pub(super) struct SwitchToMultimodal();
 
-impl UpgradeIndex for Latest_V1_15_To_V1_16_0 {
-    fn upgrade(
-        &self,
-        wtxn: &mut RwTxn,
-        index: &Index,
-        _original: (u32, u32, u32),
-        _progress: Progress,
-    ) -> Result<bool> {
+impl UpgradeIndex for SwitchToMultimodal {
+    fn upgrade(&self, wtxn: &mut RwTxn, index: &Index, _progress: Progress) -> Result<bool> {
         let v1_15_indexing_configs = index
             .main
             .remap_types::<Str, SerdeJson<Vec<super::v1_15::IndexEmbeddingConfig>>>()
@@ -41,8 +34,11 @@ impl UpgradeIndex for Latest_V1_15_To_V1_16_0 {
 
         Ok(false)
     }
+    fn must_upgrade(&self, initial_version: (u32, u32, u32)) -> bool {
+        initial_version < (1, 16, 0)
+    }
 
-    fn target_version(&self) -> (u32, u32, u32) {
-        (1, 16, 0)
+    fn description(&self) -> &'static str {
+        "Migrating the database for multimodal support"
     }
 }

crates/milli/src/vector/embedder/hf.rs

@@ -2,6 +2,7 @@ use candle_core::Tensor;
 use candle_nn::VarBuilder;
 use candle_transformers::models::bert::{BertModel, Config as BertConfig, DTYPE};
 use candle_transformers::models::modernbert::{Config as ModernConfig, ModernBert};
+use candle_transformers::models::xlm_roberta::{Config as XlmRobertaConfig, XLMRobertaModel};
 // FIXME: currently we'll be using the hub to retrieve model, in the future we might want to embed it into Meilisearch itself
 use hf_hub::api::sync::Api;
 use hf_hub::{Repo, RepoType};
@@ -89,6 +90,7 @@ impl Default for EmbedderOptions {
 enum ModelKind {
     Bert(BertModel),
     Modern(ModernBert),
+    XlmRoberta(XLMRobertaModel),
 }
 
 /// Perform embedding of documents and queries
@@ -304,7 +306,8 @@ impl Embedder {
         };
 
         let is_modern = has_arch("modernbert");
-        tracing::debug!(is_modern, model_type, "detected HF architecture");
+        let is_xlm_roberta = has_arch("xlm-roberta") || has_arch("xlm_roberta");
+        tracing::debug!(is_modern, is_xlm_roberta, model_type, "detected HF architecture");
 
         let mut tokenizer = Tokenizer::from_file(&tokenizer_filename)
             .map_err(|inner| NewEmbedderError::open_tokenizer(tokenizer_filename, inner))?;
@@ -340,6 +343,18 @@ impl Embedder {
                 )
             })?;
             ModelKind::Modern(ModernBert::load(vb, &config).map_err(NewEmbedderError::load_model)?)
+        } else if is_xlm_roberta {
+            let config: XlmRobertaConfig = serde_json::from_str(&config_str).map_err(|inner| {
+                NewEmbedderError::deserialize_config(
+                    options.model.clone(),
+                    config_str.clone(),
+                    config_filename.clone(),
+                    inner,
+                )
+            })?;
+            ModelKind::XlmRoberta(
+                XLMRobertaModel::new(&config, vb).map_err(NewEmbedderError::load_model)?,
+            )
         } else {
             let config: BertConfig = serde_json::from_str(&config_str).map_err(|inner| {
                 NewEmbedderError::deserialize_config(
@@ -451,6 +466,19 @@ impl Embedder {
                 let mask = Tensor::stack(&[mask], 0).map_err(EmbedError::tensor_shape)?;
                 model.forward(&token_ids, &mask).map_err(EmbedError::model_forward)?
             }
+            ModelKind::XlmRoberta(model) => {
+                let mut mask_vec = tokens.get_attention_mask().to_vec();
+                if mask_vec.len() > self.max_len {
+                    mask_vec.truncate(self.max_len);
+                }
+                let mask = Tensor::new(mask_vec.as_slice(), &self.device)
+                    .map_err(EmbedError::tensor_shape)?;
+                let mask = Tensor::stack(&[mask], 0).map_err(EmbedError::tensor_shape)?;
+                let token_type_ids = token_ids.zeros_like().map_err(EmbedError::tensor_shape)?;
+                model
+                    .forward(&token_ids, &mask, &token_type_ids, None, None, None)
+                    .map_err(EmbedError::model_forward)?
+            }
         };
 
         let embedding = Self::pooling(embeddings, self.pooling)?;

crates/milli/src/vector/embeddings.rs

@@ -67,7 +67,7 @@ impl<F> Embeddings<F> {
     ///
     /// If `embeddings.len() % self.dimension != 0`, then the append operation fails.
     pub fn append(&mut self, mut embeddings: Vec<F>) -> Result<(), Vec<F>> {
-        if embeddings.len() % self.dimension != 0 {
+        if !embeddings.len().is_multiple_of(self.dimension) {
             return Err(embeddings);
         }
         self.data.append(&mut embeddings);

crates/milli/src/vector/store.rs

@@ -1,5 +1,5 @@
 use hannoy::distances::{Cosine, Hamming};
-use hannoy::ItemId;
+use hannoy::{ItemId, Searched};
 use heed::{RoTxn, RwTxn, Unspecified};
 use ordered_float::OrderedFloat;
 use rand::SeedableRng as _;
@@ -974,7 +974,7 @@ impl VectorStore {
             }
 
             if let Some(mut ret) = searcher.by_item(rtxn, item)? {
-                results.append(&mut ret);
+                results.append(&mut ret.nns);
             }
         }
         results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance));
@@ -1028,10 +1028,9 @@ impl VectorStore {
                 searcher.candidates(filter);
             }
 
-            let (res, _degraded) =
-                &mut searcher
-                    .by_vector_with_cancellation(rtxn, vector, || time_budget.exceeded())?;
-            results.append(res);
+            let Searched { mut nns, did_cancel: _ } =
+                searcher.by_vector_with_cancellation(rtxn, vector, || time_budget.exceeded())?;
+            results.append(&mut nns);
         }
 
         results.sort_unstable_by_key(|(_, distance)| OrderedFloat(*distance));

crates/xtask/Cargo.toml

@@ -22,6 +22,7 @@ reqwest = { version = "0.12.24", features = [
     "json",
     "rustls-tls",
 ], default-features = false }
+semver = "1.0.27"
 serde = { version = "1.0.228", features = ["derive"] }
 serde_json = "1.0.145"
 sha2 = "0.10.9"
@@ -42,3 +43,4 @@ tracing = "0.1.41"
 tracing-subscriber = "0.3.20"
 tracing-trace = { version = "0.1.0", path = "../tracing-trace" }
 uuid = { version = "1.18.1", features = ["v7", "serde"] }
+similar-asserts = "1.7.0"

crates/xtask/src/bench/command.rs

@@ -1,194 +0,0 @@
-use std::collections::BTreeMap;
-use std::fmt::Display;
-use std::io::Read as _;
-
-use anyhow::{bail, Context as _};
-use serde::Deserialize;
-
-use super::assets::{fetch_asset, Asset};
-use super::client::{Client, Method};
-
-#[derive(Clone, Deserialize)]
-pub struct Command {
-    pub route: String,
-    pub method: Method,
-    #[serde(default)]
-    pub body: Body,
-    #[serde(default)]
-    pub synchronous: SyncMode,
-}
-
-#[derive(Default, Clone, Deserialize)]
-#[serde(untagged)]
-pub enum Body {
-    Inline {
-        inline: serde_json::Value,
-    },
-    Asset {
-        asset: String,
-    },
-    #[default]
-    Empty,
-}
-
-impl Body {
-    pub fn get(
-        self,
-        assets: &BTreeMap<String, Asset>,
-        asset_folder: &str,
-    ) -> anyhow::Result<Option<(Vec<u8>, &'static str)>> {
-        Ok(match self {
-            Body::Inline { inline: body } => Some((
-                serde_json::to_vec(&body)
-                    .context("serializing to bytes")
-                    .context("while getting inline body")?,
-                "application/json",
-            )),
-            Body::Asset { asset: name } => Some({
-                let context = || format!("while getting body from asset '{name}'");
-                let (mut file, format) =
-                    fetch_asset(&name, assets, asset_folder).with_context(context)?;
-                let mut buf = Vec::new();
-                file.read_to_end(&mut buf).with_context(context)?;
-                (buf, format.to_content_type(&name))
-            }),
-            Body::Empty => None,
-        })
-    }
-}
-
-impl Display for Command {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{:?} {} ({:?})", self.method, self.route, self.synchronous)
-    }
-}
-
-#[derive(Default, Debug, Clone, Copy, Deserialize)]
-pub enum SyncMode {
-    DontWait,
-    #[default]
-    WaitForResponse,
-    WaitForTask,
-}
-
-pub async fn run_batch(
-    client: &Client,
-    batch: &[Command],
-    assets: &BTreeMap<String, Asset>,
-    asset_folder: &str,
-) -> anyhow::Result<()> {
-    let [.., last] = batch else { return Ok(()) };
-    let sync = last.synchronous;
-
-    let mut tasks = tokio::task::JoinSet::new();
-
-    for command in batch {
-        // FIXME: you probably don't want to copy assets everytime here
-        tasks.spawn({
-            let client = client.clone();
-            let command = command.clone();
-            let assets = assets.clone();
-            let asset_folder = asset_folder.to_owned();
-
-            async move { run(client, command, &assets, &asset_folder).await }
-        });
-    }
-
-    while let Some(result) = tasks.join_next().await {
-        result
-            .context("panicked while executing command")?
-            .context("error while executing command")?;
-    }
-
-    match sync {
-        SyncMode::DontWait => {}
-        SyncMode::WaitForResponse => {}
-        SyncMode::WaitForTask => wait_for_tasks(client).await?,
-    }
-
-    Ok(())
-}
-
-async fn wait_for_tasks(client: &Client) -> anyhow::Result<()> {
-    loop {
-        let response = client
-            .get("tasks?statuses=enqueued,processing")
-            .send()
-            .await
-            .context("could not wait for tasks")?;
-        let response: serde_json::Value = response
-            .json()
-            .await
-            .context("could not deserialize response to JSON")
-            .context("could not wait for tasks")?;
-        match response.get("total") {
-            Some(serde_json::Value::Number(number)) => {
-                let number = number.as_u64().with_context(|| {
-                    format!("waiting for tasks: could not parse 'total' as integer, got {}", number)
-                })?;
-                if number == 0 {
-                    break;
-                } else {
-                    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
-                    continue;
-                }
-            }
-            Some(thing_else) => {
-                bail!(format!(
-                    "waiting for tasks: could not parse 'total' as a number, got '{thing_else}'"
-                ))
-            }
-            None => {
-                bail!(format!(
-                    "waiting for tasks: expected response to contain 'total', got '{response}'"
-                ))
-            }
-        }
-    }
-    Ok(())
-}
-
-#[tracing::instrument(skip(client, command, assets, asset_folder), fields(command = %command))]
-pub async fn run(
-    client: Client,
-    mut command: Command,
-    assets: &BTreeMap<String, Asset>,
-    asset_folder: &str,
-) -> anyhow::Result<()> {
-    // memtake the body here to leave an empty body in its place, so that command is not partially moved-out
-    let body = std::mem::take(&mut command.body)
-        .get(assets, asset_folder)
-        .with_context(|| format!("while getting body for command {command}"))?;
-
-    let request = client.request(command.method.into(), &command.route);
-
-    let request = if let Some((body, content_type)) = body {
-        request.body(body).header(reqwest::header::CONTENT_TYPE, content_type)
-    } else {
-        request
-    };
-
-    let response =
-        request.send().await.with_context(|| format!("error sending command: {}", command))?;
-
-    let code = response.status();
-    if code.is_client_error() {
-        tracing::error!(%command, %code, "error in workload file");
-        let response: serde_json::Value = response
-            .json()
-            .await
-            .context("could not deserialize response as JSON")
-            .context("parsing error in workload file when sending command")?;
-        bail!("error in workload file: server responded with error code {code} and '{response}'")
-    } else if code.is_server_error() {
-        tracing::error!(%command, %code, "server error");
-        let response: serde_json::Value = response
-            .json()
-            .await
-            .context("could not deserialize response as JSON")
-            .context("parsing server error when sending command")?;
-        bail!("server error: server responded with error code {code} and '{response}'")
-    }
-
-    Ok(())
-}

crates/xtask/src/bench/dashboard.rs

@@ -7,9 +7,9 @@ use tokio::task::AbortHandle;
 use tracing_trace::processor::span_stats::CallStats;
 use uuid::Uuid;
 
-use super::client::Client;
 use super::env_info;
-use super::workload::Workload;
+use super::workload::BenchWorkload;
+use crate::common::client::Client;
 
 #[derive(Debug, Clone)]
 pub enum DashboardClient {
@@ -89,7 +89,7 @@ impl DashboardClient {
     pub async fn create_workload(
         &self,
         invocation_uuid: Uuid,
-        workload: &Workload,
+        workload: &BenchWorkload,
     ) -> anyhow::Result<Uuid> {
         let Self::Client(dashboard_client) = self else { return Ok(Uuid::now_v7()) };
 

crates/xtask/src/bench/mod.rs

@@ -1,51 +1,36 @@
-mod assets;
-mod client;
-mod command;
 mod dashboard;
 mod env_info;
-mod meili_process;
 mod workload;
 
-use std::io::LineWriter;
-use std::path::PathBuf;
+use crate::common::args::CommonArgs;
+use crate::common::logs::setup_logs;
+use crate::common::workload::Workload;
+use std::{path::PathBuf, sync::Arc};
 
-use anyhow::Context;
+use anyhow::{bail, Context};
 use clap::Parser;
-use tracing_subscriber::fmt::format::FmtSpan;
-use tracing_subscriber::layer::SubscriberExt;
-use tracing_subscriber::Layer;
 
-use self::client::Client;
-use self::workload::Workload;
+use crate::common::client::Client;
+pub use workload::BenchWorkload;
 
-pub fn default_http_addr() -> String {
-    "127.0.0.1:7700".to_string()
-}
 pub fn default_report_folder() -> String {
     "./bench/reports/".into()
 }
 
-pub fn default_asset_folder() -> String {
-    "./bench/assets/".into()
-}
-
-pub fn default_log_filter() -> String {
-    "info".into()
-}
-
 pub fn default_dashboard_url() -> String {
     "http://localhost:9001".into()
 }
 
 /// Run benchmarks from a workload
 #[derive(Parser, Debug)]
-pub struct BenchDeriveArgs {
-    /// Filename of the workload file, pass multiple filenames
-    /// to run multiple workloads in the specified order.
-    ///
-    /// Each workload run will get its own report file.
-    #[arg(value_name = "WORKLOAD_FILE", last = false)]
-    workload_file: Vec<PathBuf>,
+pub struct BenchArgs {
+    /// Common arguments shared with other commands
+    #[command(flatten)]
+    common: CommonArgs,
+
+    /// Meilisearch master keys
+    #[arg(long)]
+    pub master_key: Option<String>,
 
     /// URL of the dashboard.
     #[arg(long, default_value_t = default_dashboard_url())]
@@ -59,34 +44,14 @@ pub struct BenchDeriveArgs {
     #[arg(long, default_value_t = default_report_folder())]
     report_folder: String,
 
-    /// Directory to store the remote assets.
-    #[arg(long, default_value_t = default_asset_folder())]
-    asset_folder: String,
-
-    /// Log directives
-    #[arg(short, long, default_value_t = default_log_filter())]
-    log_filter: String,
-
     /// Benchmark dashboard API key
     #[arg(long)]
     api_key: Option<String>,
 
-    /// Meilisearch master keys
-    #[arg(long)]
-    master_key: Option<String>,
-
-    /// Authentication bearer for fetching assets
-    #[arg(long)]
-    assets_key: Option<String>,
-
     /// Reason for the benchmark invocation
     #[arg(short, long)]
     reason: Option<String>,
 
-    /// The maximum time in seconds we allow for fetching the task queue before timing out.
-    #[arg(long, default_value_t = 60)]
-    tasks_queue_timeout_secs: u64,
-
     /// The path to the binary to run.
     ///
     /// If unspecified, runs `cargo run` after building Meilisearch with `cargo build`.
@@ -94,18 +59,8 @@ pub struct BenchDeriveArgs {
     binary_path: Option<PathBuf>,
 }
 
-pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
-    // setup logs
-    let filter: tracing_subscriber::filter::Targets =
-        args.log_filter.parse().context("invalid --log-filter")?;
-
-    let subscriber = tracing_subscriber::registry().with(
-        tracing_subscriber::fmt::layer()
-            .with_writer(|| LineWriter::new(std::io::stderr()))
-            .with_span_events(FmtSpan::NEW | FmtSpan::CLOSE)
-            .with_filter(filter),
-    );
-    tracing::subscriber::set_global_default(subscriber).context("could not setup logging")?;
+pub fn run(args: BenchArgs) -> anyhow::Result<()> {
+    setup_logs(&args.common.log_filter)?;
 
     // fetch environment and build info
     let env = env_info::Environment::generate_from_current_config();
@@ -116,8 +71,11 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
     let _scope = rt.enter();
 
     // setup clients
-    let assets_client =
-        Client::new(None, args.assets_key.as_deref(), Some(std::time::Duration::from_secs(3600)))?; // 1h
+    let assets_client = Client::new(
+        None,
+        args.common.assets_key.as_deref(),
+        Some(std::time::Duration::from_secs(3600)), // 1h
+    )?;
 
     let dashboard_client = if args.no_dashboard {
         dashboard::DashboardClient::new_dry()
@@ -134,11 +92,11 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
         None,
     )?;
 
-    let meili_client = Client::new(
+    let meili_client = Arc::new(Client::new(
         Some("http://127.0.0.1:7700".into()),
         args.master_key.as_deref(),
-        Some(std::time::Duration::from_secs(args.tasks_queue_timeout_secs)),
-    )?;
+        Some(std::time::Duration::from_secs(args.common.tasks_queue_timeout_secs)),
+    )?);
 
     // enter runtime
 
@@ -146,11 +104,11 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
         dashboard_client.send_machine_info(&env).await?;
 
         let commit_message = build_info.commit_msg.unwrap_or_default().split('\n').next().unwrap();
-        let max_workloads = args.workload_file.len();
+        let max_workloads = args.common.workload_file.len();
         let reason: Option<&str> = args.reason.as_deref();
         let invocation_uuid = dashboard_client.create_invocation(build_info.clone(), commit_message, env, max_workloads, reason).await?;
 
-        tracing::info!(workload_count = args.workload_file.len(), "handling workload files");
+        tracing::info!(workload_count = args.common.workload_file.len(), "handling workload files");
 
         // main task
         let workload_runs = tokio::spawn(
@@ -158,13 +116,17 @@ pub fn run(args: BenchDeriveArgs) -> anyhow::Result<()> {
                 let dashboard_client = dashboard_client.clone();
                 let mut dashboard_urls = Vec::new();
                 async move {
-            for workload_file in args.workload_file.iter() {
+            for workload_file in args.common.workload_file.iter() {
                 let workload: Workload = serde_json::from_reader(
                     std::fs::File::open(workload_file)
                         .with_context(|| format!("error opening {}", workload_file.display()))?,
                 )
                 .with_context(|| format!("error parsing {} as JSON", workload_file.display()))?;
 
+                let Workload::Bench(workload) = workload else {
+                    bail!("workload file {} is not a bench workload", workload_file.display());
+                };
+
                 let workload_name = workload.name.clone();
 
                 workload::execute(

crates/xtask/src/bench/workload.rs

@@ -1,24 +1,28 @@
-use std::collections::BTreeMap;
+use std::collections::{BTreeMap, HashMap};
 use std::fs::File;
 use std::io::{Seek as _, Write as _};
 use std::path::Path;
+use std::sync::Arc;
 
 use anyhow::{bail, Context as _};
 use futures_util::TryStreamExt as _;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use serde_json::json;
 use tokio::task::JoinHandle;
 use uuid::Uuid;
 
-use super::assets::Asset;
-use super::client::Client;
-use super::command::SyncMode;
 use super::dashboard::DashboardClient;
-use super::BenchDeriveArgs;
-use crate::bench::{assets, meili_process};
+use super::BenchArgs;
+use crate::common::assets::{self, Asset};
+use crate::common::client::Client;
+use crate::common::command::{run_commands, Command};
+use crate::common::instance::Binary;
+use crate::common::process::{self, delete_db, start_meili};
 
-#[derive(Deserialize)]
-pub struct Workload {
+/// A bench workload.
+/// Not to be confused with [a test workload](crate::test::workload::Workload).
+#[derive(Serialize, Deserialize, Debug)]
+pub struct BenchWorkload {
     pub name: String,
     pub run_count: u16,
     pub extra_cli_args: Vec<String>,
@@ -26,30 +30,34 @@ pub struct Workload {
     #[serde(default)]
     pub target: String,
     #[serde(default)]
-    pub precommands: Vec<super::command::Command>,
-    pub commands: Vec<super::command::Command>,
+    pub precommands: Vec<Command>,
+    pub commands: Vec<Command>,
 }
 
-async fn run_commands(
+async fn run_workload_commands(
     dashboard_client: &DashboardClient,
     logs_client: &Client,
-    meili_client: &Client,
+    meili_client: &Arc<Client>,
     workload_uuid: Uuid,
-    workload: &Workload,
-    args: &BenchDeriveArgs,
+    workload: &BenchWorkload,
+    args: &BenchArgs,
     run_number: u16,
 ) -> anyhow::Result<JoinHandle<anyhow::Result<File>>> {
     let report_folder = &args.report_folder;
     let workload_name = &workload.name;
+    let assets = Arc::new(workload.assets.clone());
+    let asset_folder = args.common.asset_folder.clone().leak();
 
-    for batch in workload
-        .precommands
-        .as_slice()
-        .split_inclusive(|command| !matches!(command.synchronous, SyncMode::DontWait))
-    {
-        super::command::run_batch(meili_client, batch, &workload.assets, &args.asset_folder)
+    run_commands(
+        meili_client,
+        &workload.precommands,
+        0,
+        &assets,
+        asset_folder,
+        &mut HashMap::new(),
+        false,
+    )
     .await?;
-    }
 
     std::fs::create_dir_all(report_folder)
         .with_context(|| format!("could not create report directory at {report_folder}"))?;
@@ -59,14 +67,16 @@ async fn run_commands(
 
     let report_handle = start_report(logs_client, trace_filename, &workload.target).await?;
 
-    for batch in workload
-        .commands
-        .as_slice()
-        .split_inclusive(|command| !matches!(command.synchronous, SyncMode::DontWait))
-    {
-        super::command::run_batch(meili_client, batch, &workload.assets, &args.asset_folder)
+    run_commands(
+        meili_client,
+        &workload.commands,
+        0,
+        &assets,
+        asset_folder,
+        &mut HashMap::new(),
+        false,
+    )
     .await?;
-    }
 
     let processor =
         stop_report(dashboard_client, logs_client, workload_uuid, report_filename, report_handle)
@@ -81,14 +91,14 @@ pub async fn execute(
     assets_client: &Client,
     dashboard_client: &DashboardClient,
     logs_client: &Client,
-    meili_client: &Client,
+    meili_client: &Arc<Client>,
     invocation_uuid: Uuid,
     master_key: Option<&str>,
-    workload: Workload,
-    args: &BenchDeriveArgs,
+    workload: BenchWorkload,
+    args: &BenchArgs,
     binary_path: Option<&Path>,
 ) -> anyhow::Result<()> {
-    assets::fetch_assets(assets_client, &workload.assets, &args.asset_folder).await?;
+    assets::fetch_assets(assets_client, &workload.assets, &args.common.asset_folder).await?;
 
     let workload_uuid = dashboard_client.create_workload(invocation_uuid, &workload).await?;
 
@@ -129,38 +139,33 @@ pub async fn execute(
 async fn execute_run(
     dashboard_client: &DashboardClient,
     logs_client: &Client,
-    meili_client: &Client,
+    meili_client: &Arc<Client>,
     workload_uuid: Uuid,
     master_key: Option<&str>,
-    workload: &Workload,
-    args: &BenchDeriveArgs,
+    workload: &BenchWorkload,
+    args: &BenchArgs,
     binary_path: Option<&Path>,
     run_number: u16,
 ) -> anyhow::Result<tokio::task::JoinHandle<anyhow::Result<std::fs::File>>> {
-    meili_process::delete_db();
+    delete_db().await;
 
-    let run_command = match binary_path {
-        Some(binary_path) => tokio::process::Command::new(binary_path),
-        None => {
-            meili_process::build().await?;
-            let mut command = tokio::process::Command::new("cargo");
-            command
-                .arg("run")
-                .arg("--release")
-                .arg("-p")
-                .arg("meilisearch")
-                .arg("--bin")
-                .arg("meilisearch")
-                .arg("--");
-            command
-        }
+    let binary = match binary_path {
+        Some(binary_path) => Binary {
+            source: crate::common::instance::BinarySource::Path(binary_path.to_owned()),
+            extra_cli_args: workload.extra_cli_args.clone(),
+        },
+        None => Binary {
+            source: crate::common::instance::BinarySource::Build {
+                edition: crate::common::instance::Edition::Community,
+            },
+            extra_cli_args: workload.extra_cli_args.clone(),
+        },
     };
 
     let meilisearch =
-        meili_process::start(meili_client, master_key, workload, &args.asset_folder, run_command)
-            .await?;
+        start_meili(meili_client, master_key, &binary, &args.common.asset_folder).await?;
 
-    let processor = run_commands(
+    let processor = run_workload_commands(
         dashboard_client,
         logs_client,
         meili_client,
@@ -171,7 +176,7 @@ async fn execute_run(
     )
     .await?;
 
-    meili_process::kill(meilisearch).await;
+    process::kill_meili(meilisearch).await;
 
     tracing::info!(run_number, "Successful run");