search::federated::ranking_rules -> search::ranking_rules

4626: Edit Documents with Rhai r=ManyTheFish a=Kerollmops

This PR introduces a first version of [the _Update Documents with Function_ (internal)](https://www.notion.so/meilisearch/Update-Documents-by-Function-45f87b13e61c4435b73943768a490808). It uses [the Rhai programming language](https://rhai.rs/) to let users express the modifications they want apply.

You can read more about the way to use this functions on [the Usage PRD Page](https://meilisearch.notion.site/Edit-Documents-with-Rhai-0cff8fea7655436592e7c8a6de932062?pvs=25). The [prototype is available](https://github.com/meilisearch/meilisearch/actions/runs/9038384483) through Docker by using the following command:

```
docker run -p 7700:7700 -v $(pwd)/meili_data:/meili_data getmeili/meilisearch:prototype-edit-documents-with-rhai-3
```

## TODO
 - [x] Support the `DocumentEdition` task in dumps.
 - [x] Remove the unwraps and panics.
 - [x] Improve error codes for the `function` parameter.
 - [x] [Update Rhai to v1.19.0](https://github.com/rhaiscript/rhai/releases/tag/v1.19.0) 🚀
 - [x] Make it an experimental feature (only restrict the HTTP calls).
 - [x] It must be possible not to send a context.
 - [x] Rebase on main.
 - [x] Check that the script cannot do any io.
 - [x] ~Introduce a `Documents.edit` action or~ require the `Documents.all` action.
 - [x] Change the `editionCode` to the clearer `function` field name in the tasks.
 - [x] Support a user provided context and maybe more (but keep function execution isolated for reproducibility).
 - [x] Support deleting documents when the `doc` is `()` (nil, null).
 - [x] Support canceling document edition.
 - [x] Multithread document edition by using rayon (and [rayon-par-bridge](https://docs.rs/rayon-par-bridge/latest/rayon_par_bridge/)).
 - [x] Limit the number of instruction by function execution.
 - [ ] ~Expose the limit of instructions in the settings.~ Not sure, in fact.
 - [x] Ignore unmodified documents in the tasks count.
 - [x] Make the `filter` field optional (not forced to be `null`).

Co-authored-by: Clément Renault <clement@meilisearch.com>

.github/workflows/bench-manual.yml

@@ -18,11 +18,9 @@ jobs:
         timeout-minutes: 180 # 3h
         steps:
             - uses: actions/checkout@v3
-            - uses: actions-rs/toolchain@v1
+            - uses: helix-editor/rust-toolchain@v1
               with:
                 profile: minimal
-                toolchain: stable
-                override: true
 
             - name: Run benchmarks - workload ${WORKLOAD_NAME} - branch ${{ github.ref }} - commit ${{ github.sha }}
               run: |

.github/workflows/bench-pr.yml

@@ -35,11 +35,9 @@ jobs:
             fetch-depth: 0 # fetch full history to be able to get main commit sha
             ref: ${{ steps.comment-branch.outputs.head_ref }}
 
-        - uses: actions-rs/toolchain@v1
+        - uses: helix-editor/rust-toolchain@v1
           with:
             profile: minimal
-            toolchain: stable
-            override: true
 
         - name: Run benchmarks on PR ${{ github.event.issue.id }}
           run: |

.github/workflows/bench-push-indexing.yml

@@ -12,11 +12,9 @@ jobs:
         timeout-minutes: 180 # 3h
         steps:
           - uses: actions/checkout@v3
-          - uses: actions-rs/toolchain@v1
+          - uses: helix-editor/rust-toolchain@v1
             with:
               profile: minimal
-              toolchain: stable
-              override: true
 
           # Run benchmarks
           - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch main - Commit ${{ github.sha }}

.github/workflows/benchmarks-manual.yml

@@ -18,11 +18,9 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/benchmarks-pr.yml

@@ -13,11 +13,9 @@ jobs:
     runs-on: benchmarks
     timeout-minutes: 4320 # 72h
     steps:
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       - name: Check for Command
         id: command

.github/workflows/benchmarks-push-indexing.yml

@@ -16,11 +16,9 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/benchmarks-push-search-geo.yml

@@ -15,11 +15,9 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/benchmarks-push-search-songs.yml

@@ -15,11 +15,9 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/benchmarks-push-search-wiki.yml

@@ -15,11 +15,9 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/flaky-tests.yml

@@ -1,4 +1,6 @@
 name: Look for flaky tests
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 on:
   workflow_dispatch:
   schedule:
@@ -16,10 +18,7 @@ jobs:
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
+    - uses: helix-editor/rust-toolchain@v1
     - name: Install cargo-flaky
       run: cargo install cargo-flaky
     - name: Run cargo flaky in the dumps

.github/workflows/fuzzer-indexing.yml

@@ -1,5 +1,6 @@
 name: Run the indexing fuzzer
-
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 on:
   push:
     branches:
@@ -12,11 +13,9 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Run benchmarks
       - name: Run the fuzzer

.github/workflows/publish-apt-brew-pkg.yml

@@ -15,6 +15,8 @@ jobs:
 
   debian:
     name: Publish debian packagge
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     needs: check-version
     container:
@@ -25,10 +27,7 @@ jobs:
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
+    - uses: helix-editor/rust-toolchain@v1
     - name: Install cargo-deb
       run: cargo install cargo-deb
     - uses: actions/checkout@v3

.github/workflows/publish-binaries.yml

@@ -35,6 +35,8 @@ jobs:
   publish-linux:
     name: Publish binary for Linux
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     needs: check-version
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27
@@ -45,10 +47,7 @@ jobs:
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
+    - uses: helix-editor/rust-toolchain@v1
     - name: Build
       run: cargo build --release --locked
     # No need to upload binaries for dry run (cron)
@@ -78,10 +77,7 @@ jobs:
             asset_name: meilisearch-windows-amd64.exe
     steps:
     - uses: actions/checkout@v3
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
+    - uses: helix-editor/rust-toolchain@v1
     - name: Build
       run: cargo build --release --locked
     # No need to upload binaries for dry run (cron)
@@ -107,12 +103,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
       - name: Installing Rust toolchain
-        uses: actions-rs/toolchain@v1
+        uses: helix-editor/rust-toolchain@v1
         with:
-          toolchain: stable
           profile: minimal
           target: ${{ matrix.target }}
-          override: true
       - name: Cargo build
         uses: actions-rs/cargo@v1
         with:
@@ -132,6 +126,8 @@ jobs:
     name: Publish binary for aarch64
     runs-on: ubuntu-latest
     needs: check-version
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27
       image: ubuntu:18.04
@@ -154,12 +150,10 @@ jobs:
           add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
           apt-get update -y && apt-get install -y docker-ce
       - name: Installing Rust toolchain
-        uses: actions-rs/toolchain@v1
+        uses: helix-editor/rust-toolchain@v1
         with:
-          toolchain: stable
           profile: minimal
           target: ${{ matrix.target }}
-          override: true
       - name: Configure target aarch64 GNU
         ## Environment variable is not passed using env:
         ## LD gold won't work with MUSL

.github/workflows/publish-docker-images.yml

@@ -80,10 +80,11 @@ jobs:
             type=ref,event=tag
             type=raw,value=nightly,enable=${{ github.event_name != 'push' }}
             type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
+            type=semver,pattern=v{{major}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
             type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           push: true
           platforms: linux/amd64,linux/arm64

.github/workflows/test-suite.yml

@@ -21,6 +21,8 @@ jobs:
   test-linux:
     name: Tests on ubuntu-18.04
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
       image: ubuntu:18.04
@@ -31,10 +33,7 @@ jobs:
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
       - name: Setup test with Rust stable
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+        uses: helix-editor/rust-toolchain@v1
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
       - name: Run cargo check without any default features
@@ -59,10 +58,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: helix-editor/rust-toolchain@v1
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -77,6 +73,8 @@ jobs:
   test-all-features:
     name: Tests almost all features
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
       image: ubuntu:18.04
@@ -87,10 +85,7 @@ jobs:
         run: |
           apt-get update
           apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: helix-editor/rust-toolchain@v1
       - name: Run cargo build with almost all features
         run: |
           cargo build --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda)"
@@ -100,6 +95,8 @@ jobs:
 
   test-disabled-tokenization:
     name: Test disabled tokenization
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       image: ubuntu:18.04
@@ -110,13 +107,10 @@ jobs:
         run: |
           apt-get update
           apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: helix-editor/rust-toolchain@v1
       - name: Run cargo tree without default features and check lindera is not present
         run: |
-          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -vqz lindera; then
+          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -qz lindera; then
             echo "lindera has been found in the sources and it shouldn't"
             exit 1
           fi
@@ -127,6 +121,8 @@ jobs:
   # We run tests in debug also, to make sure that the debug_assertions are hit
   test-debug:
     name: Run tests in debug
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
@@ -137,10 +133,7 @@ jobs:
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: helix-editor/rust-toolchain@v1
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
       - name: Run tests in debug
@@ -154,11 +147,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.75.0
-          override: true
           components: clippy
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
@@ -173,10 +164,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
+          toolchain: nightly-2024-07-09
           override: true
           components: rustfmt
       - name: Cache dependencies

.github/workflows/update-cargo-toml-version.yml

@@ -18,11 +18,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
       - name: Install sd
         run: cargo install sd
       - name: Update Cargo.toml file

BENCHMARKS.md

@@ -109,6 +109,12 @@ They are JSON files with the following structure (comments are not actually supp
   "run_count": 3,
   // List of arguments to add to the Meilisearch command line.
   "extra_cli_args": ["--max-indexing-threads=1"],
+  // An expression that can be parsed as a comma-separated list of targets and levels
+  // as described in [tracing_subscriber's documentation](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/targets/struct.Targets.html#examples).
+  // The expression is used to filter the spans that are measured for profiling purposes.
+  // Optional, defaults to "indexing::=trace" (for indexing workloads), common other values is
+  // "search::=trace"
+  "target": "indexing::=trace",
   // List of named assets that can be used in the commands.
   "assets": {
     // name of the asset.

CONTRIBUTING.md

@@ -52,6 +52,16 @@ cargo test
 
 This command will be triggered to each PR as a requirement for merging it.
 
+#### Faster build
+
+You can set the `LINDERA_CACHE` environment variable to speed up your successive builds by up to 2 minutes.
+It'll store some built artifacts in the directory of your choice.
+
+We recommend using the standard `$HOME/.cache/lindera` directory:
+```sh
+export LINDERA_CACHE=$HOME/.cache/lindera
+```
+
 #### Snapshot-based tests
 
 We are using [insta](https://insta.rs) to perform snapshot-based testing.
@@ -63,7 +73,7 @@ Furthermore, we provide some macros on top of insta, notably a way to use snapsh
 
 To effectively debug snapshot-based hashes, we recommend you export the `MEILI_TEST_FULL_SNAPS` environment variable so that snapshot are fully created locally:
 
-```
+```sh
 export MEILI_TEST_FULL_SNAPS=true # add this to your .bashrc, .zshrc, ...
 ```
 

Dockerfile

@@ -1,7 +1,7 @@
 # Compile
-FROM    rust:1.75.0-alpine3.18 AS compiler
+FROM    rust:1.79.0-alpine3.20 AS compiler
 
-RUN     apk add -q --update-cache --no-cache build-base openssl-dev
+RUN     apk add -q --no-cache build-base openssl-dev
 
 WORKDIR /
 
@@ -20,13 +20,12 @@ RUN     set -eux; \
         cargo build --release -p meilisearch -p meilitool
 
 # Run
-FROM    alpine:3.16
+FROM    alpine:3.20
 
 ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
 ENV     MEILI_SERVER_PROVIDER docker
 
-RUN     apk update --quiet \
-        && apk add -q --no-cache libgcc tini curl
+RUN     apk add -q --no-cache libgcc tini curl
 
 # add meilisearch and meilitool to the `/bin` so you can run it from anywhere
 # and it's easy to find.

README.md

@@ -1,9 +1,6 @@
 <p align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-light-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
-  </a>
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-dark-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
+  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo" target="_blank">
+    <img src="assets/meilisearch-logo-kawaii.png">
   </a>
 </p>
 
@@ -25,7 +22,7 @@
 
 <p align="center">⚡ A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow 🔍</p>
 
-[Meilisearch](https://www.meilisearch.com) helps you shape a delightful search experience in a snap, offering features that work out of the box to speed up your workflow.
+[Meilisearch](https://www.meilisearch.com?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=intro) helps you shape a delightful search experience in a snap, offering features that work out of the box to speed up your workflow.
 
 <p align="center" name="demo">
   <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-light-mode-only" target="_blank">
@@ -36,11 +33,18 @@
   </a>
 </p>
 
-🔥 [**Try it!**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-link) 🔥
+## 🖥 Examples
+
+- [**Movies**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — An application to help you find streaming platforms to watch movies using [hybrid search](https://www.meilisearch.com/solutions/hybrid-search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos).
+- [**Ecommerce**](https://ecommerce.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Ecommerce website using disjunctive [facets](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos), range and rating filtering, and pagination.
+- [**Songs**](https://music.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search through 47 million of songs.
+- [**SaaS**](https://saas.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search for contacts, deals, and companies in this [multi-tenant](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) CRM application.
+
+See the list of all our example apps in our [demos repository](https://github.com/meilisearch/demos).
 
 ## ✨ Features
-- **Hybrid search:** Combine the best of both [semantic](https://www.meilisearch.com/docs/learn/experimental/vector_search) & full-text search to get the most relevant results
-- **Search-as-you-type:** find & display results in less than 50 milliseconds to provide an intuitive experience
+- **Hybrid search:** Combine the best of both [semantic](https://www.meilisearch.com/docs/learn/experimental/vector_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) & full-text search to get the most relevant results
+- **Search-as-you-type:** Find & display results in less than 50 milliseconds to provide an intuitive experience
 - **[Typo tolerance](https://www.meilisearch.com/docs/learn/configuration/typo_tolerance?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** get relevant matches even when queries contain typos and misspellings
 - **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your users' search experience with custom filters and build a faceted search interface in a few lines of code
 - **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** sort results based on price, date, or pretty much anything else your users need
@@ -59,7 +63,7 @@ You can consult Meilisearch's documentation at [meilisearch.com/docs](https://ww
 
 ## 🚀 Getting started
 
-For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
+For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [documentation](https://www.meilisearch.com/docs?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
 
 ## 🌍 Supercharge your Meilisearch experience
 
@@ -83,7 +87,7 @@ Finally, for more in-depth information, refer to our articles explaining fundame
 
 ## 📊 Telemetry
 
-Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
+Meilisearch collects **anonymized** user data to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
 
 To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Remember to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
 
@@ -105,11 +109,11 @@ Thank you for your support!
 
 ## 👩‍💻 Contributing
 
-Meilisearch is, and will always be, open-source! If you want to contribute to the project, please take a look at [our contribution guidelines](CONTRIBUTING.md).
+Meilisearch is, and will always be, open-source! If you want to contribute to the project, please look at [our contribution guidelines](CONTRIBUTING.md).
 
 ## 📦 Versioning
 
-Meilisearch releases and their associated binaries are available [in this GitHub page](https://github.com/meilisearch/meilisearch/releases).
+Meilisearch releases and their associated binaries are available on the project's [releases page](https://github.com/meilisearch/meilisearch/releases).
 
 The binaries are versioned following [SemVer conventions](https://semver.org/). To know more, read our [versioning policy](https://github.com/meilisearch/engine-team/blob/main/resources/versioning-policy.md).
 

benchmarks/Cargo.toml

@@ -11,24 +11,24 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.79"
+anyhow = "1.0.86"
 csv = "1.3.0"
 milli = { path = "../milli" }
-mimalloc = { version = "0.1.39", default-features = false }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
+mimalloc = { version = "0.1.43", default-features = false }
+serde_json = { version = "1.0.120", features = ["preserve_order"] }
 
 [dev-dependencies]
 criterion = { version = "0.5.1", features = ["html_reports"] }
 rand = "0.8.5"
 rand_chacha = "0.3.1"
-roaring = "0.10.2"
+roaring = "0.10.6"
 
 [build-dependencies]
-anyhow = "1.0.79"
-bytes = "1.5.0"
+anyhow = "1.0.86"
+bytes = "1.6.0"
 convert_case = "0.6.0"
-flate2 = "1.0.28"
-reqwest = { version = "0.11.23", features = ["blocking", "rustls-tls"], default-features = false }
+flate2 = "1.0.30"
+reqwest = { version = "0.12.5", features = ["blocking", "rustls-tls"], default-features = false }
 
 [features]
 default = ["milli/all-tokenizations"]

build-info/Cargo.toml

@@ -11,8 +11,8 @@ license.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-time = { version = "0.3.34", features = ["parsing"] }
+time = { version = "0.3.36", features = ["parsing"] }
 
 [build-dependencies]
-anyhow = "1.0.80"
-vergen-git2 = "1.0.0-beta.2"
+anyhow = "1.0.86"
+vergen-git2 = "1.0.0"

dump/Cargo.toml

@@ -11,22 +11,21 @@ readme.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.79"
-flate2 = "1.0.28"
-http = "0.2.11"
-meilisearch-auth = { path = "../meilisearch-auth" }
+anyhow = "1.0.86"
+flate2 = "1.0.30"
+http = "1.1.0"
 meilisearch-types = { path = "../meilisearch-types" }
 once_cell = "1.19.0"
-regex = "1.10.2"
-roaring = { version = "0.10.2", features = ["serde"] }
-serde = { version = "1.0.195", features = ["derive"] }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
-tar = "0.4.40"
-tempfile = "3.9.0"
-thiserror = "1.0.56"
-time = { version = "0.3.31", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+regex = "1.10.5"
+roaring = { version = "0.10.6", features = ["serde"] }
+serde = { version = "1.0.204", features = ["derive"] }
+serde_json = { version = "1.0.120", features = ["preserve_order"] }
+tar = "0.4.41"
+tempfile = "3.10.1"
+thiserror = "1.0.61"
+time = { version = "0.3.36", features = ["serde-well-known", "formatting", "parsing", "macros"] }
 tracing = "0.1.40"
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
+uuid = { version = "1.10.0", features = ["serde", "v4"] }
 
 [dev-dependencies]
 big_s = "1.0.2"

dump/src/lib.rs

@@ -104,6 +104,11 @@ pub enum KindDump {
     DocumentDeletionByFilter {
         filter: serde_json::Value,
     },
+    DocumentEdition {
+        filter: Option<serde_json::Value>,
+        context: Option<serde_json::Map<String, serde_json::Value>>,
+        function: String,
+    },
     Settings {
         settings: Box<meilisearch_types::settings::Settings<Unchecked>>,
         is_deletion: bool,
@@ -172,6 +177,9 @@ impl From<KindWithContent> for KindDump {
             KindWithContent::DocumentDeletionByFilter { filter_expr, .. } => {
                 KindDump::DocumentDeletionByFilter { filter: filter_expr }
             }
+            KindWithContent::DocumentEdition { filter_expr, context, function, .. } => {
+                KindDump::DocumentEdition { filter: filter_expr, context, function }
+            }
             KindWithContent::DocumentClear { .. } => KindDump::DocumentClear,
             KindWithContent::SettingsUpdate {
                 new_settings,

dump/src/reader/compat/v2_to_v3.rs

@@ -425,7 +425,7 @@ pub(crate) mod test {
         let mut dump = v2::V2Reader::open(dir).unwrap().to_v3();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/compat/v3_to_v4.rs

@@ -358,7 +358,7 @@ pub(crate) mod test {
         let mut dump = v3::V3Reader::open(dir).unwrap().to_v4();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/compat/v4_to_v5.rs

@@ -394,8 +394,8 @@ pub(crate) mod test {
         let mut dump = v4::V4Reader::open(dir).unwrap().to_v5();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/compat/v5_to_v6.rs

@@ -442,8 +442,8 @@ pub(crate) mod test {
         let mut dump = v5::V5Reader::open(dir).unwrap().to_v6();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/mod.rs

@@ -216,7 +216,7 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2024-05-16 15:51:34.151044 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2024-05-16 15:51:34.151044 +00:00:00");
         insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
 
         // tasks
@@ -337,7 +337,7 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-07-06 7:10:27.21958 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2023-07-06 7:10:27.21958 +00:00:00");
         insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
 
         // tasks
@@ -383,8 +383,8 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
@@ -463,8 +463,8 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
@@ -540,7 +540,7 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
         assert_eq!(dump.instance_uid().unwrap(), None);
 
         // tasks
@@ -633,7 +633,7 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
         assert_eq!(dump.instance_uid().unwrap(), None);
 
         // tasks
@@ -726,7 +726,7 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
         assert_eq!(dump.instance_uid().unwrap(), None);
 
         // tasks

dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-6.snap

@@ -780,7 +780,7 @@ expression: document
           1.3484878540039063
         ]
       ],
-      "userProvided": false
+      "regenerate": true
     }
   }
 }

dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-7.snap

@@ -779,7 +779,7 @@ expression: document
           1.04031240940094
         ]
       ],
-      "userProvided": false
+      "regenerate": true
     }
   }
 }

dump/src/reader/v2/mod.rs

@@ -252,7 +252,7 @@ pub(crate) mod test {
         let mut dump = V2Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
@@ -349,7 +349,7 @@ pub(crate) mod test {
         let mut dump = V2Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/v3/mod.rs

@@ -267,7 +267,7 @@ pub(crate) mod test {
         let mut dump = V3Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/v3/settings.rs

@@ -152,6 +152,7 @@ impl Settings<Unchecked> {
 }
 
 #[derive(Debug, Clone, Deserialize)]
+#[allow(dead_code)] // otherwise rustc complains that the fields go unused
 #[cfg_attr(test, derive(serde::Serialize))]
 #[serde(deny_unknown_fields)]
 #[serde(rename_all = "camelCase")]

dump/src/reader/v4/mod.rs

@@ -262,8 +262,8 @@ pub(crate) mod test {
         let mut dump = V4Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/v4/settings.rs

@@ -182,6 +182,7 @@ impl Settings<Unchecked> {
     }
 }
 
+#[allow(dead_code)] // otherwise rustc complains that the fields go unused
 #[derive(Debug, Clone, Deserialize)]
 #[cfg_attr(test, derive(serde::Serialize))]
 #[serde(deny_unknown_fields)]

dump/src/reader/v5/mod.rs

@@ -299,8 +299,8 @@ pub(crate) mod test {
         let mut dump = V5Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/v5/tasks.rs

@@ -200,6 +200,7 @@ impl std::ops::Deref for IndexUid {
     }
 }
 
+#[allow(dead_code)] // otherwise rustc complains that the fields go unused
 #[derive(Debug)]
 #[cfg_attr(test, derive(serde::Serialize))]
 #[cfg_attr(test, serde(rename_all = "camelCase"))]

dump/src/writer.rs

@@ -281,7 +281,7 @@ pub(crate) mod test {
         let dump_path = dump.path();
 
         // ==== checking global file hierarchy (we want to be sure there isn't too many files or too few)
-        insta::assert_display_snapshot!(create_directory_hierarchy(dump_path), @r###"
+        insta::assert_snapshot!(create_directory_hierarchy(dump_path), @r###"
         .
         ├---- indexes/
         │    └---- doggos/

file-store/Cargo.toml

@@ -11,10 +11,7 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-tempfile = "3.9.0"
-thiserror = "1.0.56"
+tempfile = "3.10.1"
+thiserror = "1.0.61"
 tracing = "0.1.40"
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
-
-[dev-dependencies]
-faux = "0.1.10"
+uuid = { version = "1.10.0", features = ["serde", "v4"] }

filter-parser/Cargo.toml

@@ -14,7 +14,7 @@ license.workspace = true
 [dependencies]
 nom = "7.1.3"
 nom_locate = "4.2.0"
-unescaper = "0.1.3"
+unescaper = "0.1.5"
 
 [dev-dependencies]
-insta = "1.34.0"
+insta = "1.39.0"

filter-parser/src/lib.rs

@@ -564,121 +564,121 @@ pub mod tests {
 
     #[test]
     fn parse_escaped() {
-        insta::assert_display_snapshot!(p(r"title = 'foo\\'"), @r#"{title} = {foo\}"#);
-        insta::assert_display_snapshot!(p(r"title = 'foo\\\\'"), @r#"{title} = {foo\\}"#);
-        insta::assert_display_snapshot!(p(r"title = 'foo\\\\\\'"), @r#"{title} = {foo\\\}"#);
-        insta::assert_display_snapshot!(p(r"title = 'foo\\\\\\\\'"), @r#"{title} = {foo\\\\}"#);
+        insta::assert_snapshot!(p(r"title = 'foo\\'"), @r#"{title} = {foo\}"#);
+        insta::assert_snapshot!(p(r"title = 'foo\\\\'"), @r#"{title} = {foo\\}"#);
+        insta::assert_snapshot!(p(r"title = 'foo\\\\\\'"), @r#"{title} = {foo\\\}"#);
+        insta::assert_snapshot!(p(r"title = 'foo\\\\\\\\'"), @r#"{title} = {foo\\\\}"#);
         // but it also works with other sequences
-        insta::assert_display_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}");
+        insta::assert_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}");
     }
 
     #[test]
     fn parse() {
         // Test equal
-        insta::assert_display_snapshot!(p("channel = Ponce"), @"{channel} = {Ponce}");
-        insta::assert_display_snapshot!(p("subscribers = 12"), @"{subscribers} = {12}");
-        insta::assert_display_snapshot!(p("channel = 'Mister Mv'"), @"{channel} = {Mister Mv}");
-        insta::assert_display_snapshot!(p("channel = \"Mister Mv\""), @"{channel} = {Mister Mv}");
-        insta::assert_display_snapshot!(p("'dog race' = Borzoi"), @"{dog race} = {Borzoi}");
-        insta::assert_display_snapshot!(p("\"dog race\" = Chusky"), @"{dog race} = {Chusky}");
-        insta::assert_display_snapshot!(p("\"dog race\" = \"Bernese Mountain\""), @"{dog race} = {Bernese Mountain}");
-        insta::assert_display_snapshot!(p("'dog race' = 'Bernese Mountain'"), @"{dog race} = {Bernese Mountain}");
-        insta::assert_display_snapshot!(p("\"dog race\" = 'Bernese Mountain'"), @"{dog race} = {Bernese Mountain}");
+        insta::assert_snapshot!(p("channel = Ponce"), @"{channel} = {Ponce}");
+        insta::assert_snapshot!(p("subscribers = 12"), @"{subscribers} = {12}");
+        insta::assert_snapshot!(p("channel = 'Mister Mv'"), @"{channel} = {Mister Mv}");
+        insta::assert_snapshot!(p("channel = \"Mister Mv\""), @"{channel} = {Mister Mv}");
+        insta::assert_snapshot!(p("'dog race' = Borzoi"), @"{dog race} = {Borzoi}");
+        insta::assert_snapshot!(p("\"dog race\" = Chusky"), @"{dog race} = {Chusky}");
+        insta::assert_snapshot!(p("\"dog race\" = \"Bernese Mountain\""), @"{dog race} = {Bernese Mountain}");
+        insta::assert_snapshot!(p("'dog race' = 'Bernese Mountain'"), @"{dog race} = {Bernese Mountain}");
+        insta::assert_snapshot!(p("\"dog race\" = 'Bernese Mountain'"), @"{dog race} = {Bernese Mountain}");
 
         // Test IN
-        insta::assert_display_snapshot!(p("colour IN[]"), @"{colour} IN[]");
-        insta::assert_display_snapshot!(p("colour IN[green]"), @"{colour} IN[{green}, ]");
-        insta::assert_display_snapshot!(p("colour IN[green,]"), @"{colour} IN[{green}, ]");
-        insta::assert_display_snapshot!(p("colour NOT IN[green,blue]"), @"NOT ({colour} IN[{green}, {blue}, ])");
-        insta::assert_display_snapshot!(p(" colour IN [  green , blue , ]"), @"{colour} IN[{green}, {blue}, ]");
+        insta::assert_snapshot!(p("colour IN[]"), @"{colour} IN[]");
+        insta::assert_snapshot!(p("colour IN[green]"), @"{colour} IN[{green}, ]");
+        insta::assert_snapshot!(p("colour IN[green,]"), @"{colour} IN[{green}, ]");
+        insta::assert_snapshot!(p("colour NOT IN[green,blue]"), @"NOT ({colour} IN[{green}, {blue}, ])");
+        insta::assert_snapshot!(p(" colour IN [  green , blue , ]"), @"{colour} IN[{green}, {blue}, ]");
 
         // Test IN + OR/AND/()
-        insta::assert_display_snapshot!(p(" colour IN [green, blue]  AND color = green "), @"AND[{colour} IN[{green}, {blue}, ], {color} = {green}, ]");
-        insta::assert_display_snapshot!(p("NOT (colour IN [green, blue])  AND color = green "), @"AND[NOT ({colour} IN[{green}, {blue}, ]), {color} = {green}, ]");
-        insta::assert_display_snapshot!(p("x = 1 OR NOT (colour IN [green, blue]  OR color = green) "), @"OR[{x} = {1}, NOT (OR[{colour} IN[{green}, {blue}, ], {color} = {green}, ]), ]");
+        insta::assert_snapshot!(p(" colour IN [green, blue]  AND color = green "), @"AND[{colour} IN[{green}, {blue}, ], {color} = {green}, ]");
+        insta::assert_snapshot!(p("NOT (colour IN [green, blue])  AND color = green "), @"AND[NOT ({colour} IN[{green}, {blue}, ]), {color} = {green}, ]");
+        insta::assert_snapshot!(p("x = 1 OR NOT (colour IN [green, blue]  OR color = green) "), @"OR[{x} = {1}, NOT (OR[{colour} IN[{green}, {blue}, ], {color} = {green}, ]), ]");
 
         // Test whitespace start/end
-        insta::assert_display_snapshot!(p(" colour = green "), @"{colour} = {green}");
-        insta::assert_display_snapshot!(p(" (colour = green OR colour = red) "), @"OR[{colour} = {green}, {colour} = {red}, ]");
-        insta::assert_display_snapshot!(p(" colour IN [green, blue]  AND color = green "), @"AND[{colour} IN[{green}, {blue}, ], {color} = {green}, ]");
-        insta::assert_display_snapshot!(p(" colour NOT  IN [green, blue] "), @"NOT ({colour} IN[{green}, {blue}, ])");
-        insta::assert_display_snapshot!(p(" colour IN [green, blue] "), @"{colour} IN[{green}, {blue}, ]");
+        insta::assert_snapshot!(p(" colour = green "), @"{colour} = {green}");
+        insta::assert_snapshot!(p(" (colour = green OR colour = red) "), @"OR[{colour} = {green}, {colour} = {red}, ]");
+        insta::assert_snapshot!(p(" colour IN [green, blue]  AND color = green "), @"AND[{colour} IN[{green}, {blue}, ], {color} = {green}, ]");
+        insta::assert_snapshot!(p(" colour NOT  IN [green, blue] "), @"NOT ({colour} IN[{green}, {blue}, ])");
+        insta::assert_snapshot!(p(" colour IN [green, blue] "), @"{colour} IN[{green}, {blue}, ]");
 
         // Test conditions
-        insta::assert_display_snapshot!(p("channel != ponce"), @"{channel} != {ponce}");
-        insta::assert_display_snapshot!(p("NOT channel = ponce"), @"NOT ({channel} = {ponce})");
-        insta::assert_display_snapshot!(p("subscribers < 1000"), @"{subscribers} < {1000}");
-        insta::assert_display_snapshot!(p("subscribers > 1000"), @"{subscribers} > {1000}");
-        insta::assert_display_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
-        insta::assert_display_snapshot!(p("subscribers >= 1000"), @"{subscribers} >= {1000}");
-        insta::assert_display_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
-        insta::assert_display_snapshot!(p("subscribers 100 TO 1000"), @"{subscribers} {100} TO {1000}");
+        insta::assert_snapshot!(p("channel != ponce"), @"{channel} != {ponce}");
+        insta::assert_snapshot!(p("NOT channel = ponce"), @"NOT ({channel} = {ponce})");
+        insta::assert_snapshot!(p("subscribers < 1000"), @"{subscribers} < {1000}");
+        insta::assert_snapshot!(p("subscribers > 1000"), @"{subscribers} > {1000}");
+        insta::assert_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
+        insta::assert_snapshot!(p("subscribers >= 1000"), @"{subscribers} >= {1000}");
+        insta::assert_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
+        insta::assert_snapshot!(p("subscribers 100 TO 1000"), @"{subscribers} {100} TO {1000}");
 
         // Test NOT
-        insta::assert_display_snapshot!(p("NOT subscribers < 1000"), @"NOT ({subscribers} < {1000})");
-        insta::assert_display_snapshot!(p("NOT subscribers 100 TO 1000"), @"NOT ({subscribers} {100} TO {1000})");
+        insta::assert_snapshot!(p("NOT subscribers < 1000"), @"NOT ({subscribers} < {1000})");
+        insta::assert_snapshot!(p("NOT subscribers 100 TO 1000"), @"NOT ({subscribers} {100} TO {1000})");
 
         // Test NULL + NOT NULL
-        insta::assert_display_snapshot!(p("subscribers IS NULL"), @"{subscribers} IS NULL");
-        insta::assert_display_snapshot!(p("NOT subscribers IS NULL"), @"NOT ({subscribers} IS NULL)");
-        insta::assert_display_snapshot!(p("subscribers IS NOT NULL"), @"NOT ({subscribers} IS NULL)");
-        insta::assert_display_snapshot!(p("NOT subscribers IS NOT NULL"), @"{subscribers} IS NULL");
-        insta::assert_display_snapshot!(p("subscribers  IS   NOT   NULL"), @"NOT ({subscribers} IS NULL)");
+        insta::assert_snapshot!(p("subscribers IS NULL"), @"{subscribers} IS NULL");
+        insta::assert_snapshot!(p("NOT subscribers IS NULL"), @"NOT ({subscribers} IS NULL)");
+        insta::assert_snapshot!(p("subscribers IS NOT NULL"), @"NOT ({subscribers} IS NULL)");
+        insta::assert_snapshot!(p("NOT subscribers IS NOT NULL"), @"{subscribers} IS NULL");
+        insta::assert_snapshot!(p("subscribers  IS   NOT   NULL"), @"NOT ({subscribers} IS NULL)");
 
         // Test EMPTY + NOT EMPTY
-        insta::assert_display_snapshot!(p("subscribers IS EMPTY"), @"{subscribers} IS EMPTY");
-        insta::assert_display_snapshot!(p("NOT subscribers IS EMPTY"), @"NOT ({subscribers} IS EMPTY)");
-        insta::assert_display_snapshot!(p("subscribers IS NOT EMPTY"), @"NOT ({subscribers} IS EMPTY)");
-        insta::assert_display_snapshot!(p("NOT subscribers IS NOT EMPTY"), @"{subscribers} IS EMPTY");
-        insta::assert_display_snapshot!(p("subscribers  IS   NOT   EMPTY"), @"NOT ({subscribers} IS EMPTY)");
+        insta::assert_snapshot!(p("subscribers IS EMPTY"), @"{subscribers} IS EMPTY");
+        insta::assert_snapshot!(p("NOT subscribers IS EMPTY"), @"NOT ({subscribers} IS EMPTY)");
+        insta::assert_snapshot!(p("subscribers IS NOT EMPTY"), @"NOT ({subscribers} IS EMPTY)");
+        insta::assert_snapshot!(p("NOT subscribers IS NOT EMPTY"), @"{subscribers} IS EMPTY");
+        insta::assert_snapshot!(p("subscribers  IS   NOT   EMPTY"), @"NOT ({subscribers} IS EMPTY)");
 
         // Test EXISTS + NOT EXITS
-        insta::assert_display_snapshot!(p("subscribers EXISTS"), @"{subscribers} EXISTS");
-        insta::assert_display_snapshot!(p("NOT subscribers EXISTS"), @"NOT ({subscribers} EXISTS)");
-        insta::assert_display_snapshot!(p("subscribers NOT EXISTS"), @"NOT ({subscribers} EXISTS)");
-        insta::assert_display_snapshot!(p("NOT subscribers NOT EXISTS"), @"{subscribers} EXISTS");
-        insta::assert_display_snapshot!(p("subscribers NOT   EXISTS"), @"NOT ({subscribers} EXISTS)");
+        insta::assert_snapshot!(p("subscribers EXISTS"), @"{subscribers} EXISTS");
+        insta::assert_snapshot!(p("NOT subscribers EXISTS"), @"NOT ({subscribers} EXISTS)");
+        insta::assert_snapshot!(p("subscribers NOT EXISTS"), @"NOT ({subscribers} EXISTS)");
+        insta::assert_snapshot!(p("NOT subscribers NOT EXISTS"), @"{subscribers} EXISTS");
+        insta::assert_snapshot!(p("subscribers NOT   EXISTS"), @"NOT ({subscribers} EXISTS)");
 
         // Test nested NOT
-        insta::assert_display_snapshot!(p("NOT NOT NOT NOT x = 5"), @"{x} = {5}");
-        insta::assert_display_snapshot!(p("NOT NOT (NOT NOT x = 5)"), @"{x} = {5}");
+        insta::assert_snapshot!(p("NOT NOT NOT NOT x = 5"), @"{x} = {5}");
+        insta::assert_snapshot!(p("NOT NOT (NOT NOT x = 5)"), @"{x} = {5}");
 
         // Test geo radius
-        insta::assert_display_snapshot!(p("_geoRadius(12, 13, 14)"), @"_geoRadius({12}, {13}, {14})");
-        insta::assert_display_snapshot!(p("NOT _geoRadius(12, 13, 14)"), @"NOT (_geoRadius({12}, {13}, {14}))");
-        insta::assert_display_snapshot!(p("_geoRadius(12,13,14)"), @"_geoRadius({12}, {13}, {14})");
+        insta::assert_snapshot!(p("_geoRadius(12, 13, 14)"), @"_geoRadius({12}, {13}, {14})");
+        insta::assert_snapshot!(p("NOT _geoRadius(12, 13, 14)"), @"NOT (_geoRadius({12}, {13}, {14}))");
+        insta::assert_snapshot!(p("_geoRadius(12,13,14)"), @"_geoRadius({12}, {13}, {14})");
 
         // Test geo bounding box
-        insta::assert_display_snapshot!(p("_geoBoundingBox([12, 13], [14, 15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
-        insta::assert_display_snapshot!(p("NOT _geoBoundingBox([12, 13], [14, 15])"), @"NOT (_geoBoundingBox([{12}, {13}], [{14}, {15}]))");
-        insta::assert_display_snapshot!(p("_geoBoundingBox([12,13],[14,15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
+        insta::assert_snapshot!(p("_geoBoundingBox([12, 13], [14, 15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
+        insta::assert_snapshot!(p("NOT _geoBoundingBox([12, 13], [14, 15])"), @"NOT (_geoBoundingBox([{12}, {13}], [{14}, {15}]))");
+        insta::assert_snapshot!(p("_geoBoundingBox([12,13],[14,15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
 
         // Test OR + AND
-        insta::assert_display_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain'"), @"AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
-        insta::assert_display_snapshot!(p("channel = ponce OR 'dog race' != 'bernese mountain'"), @"OR[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
-        insta::assert_display_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000"), @"OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, ]");
-        insta::assert_display_snapshot!(
+        insta::assert_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain'"), @"AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
+        insta::assert_snapshot!(p("channel = ponce OR 'dog race' != 'bernese mountain'"), @"OR[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
+        insta::assert_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000"), @"OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, ]");
+        insta::assert_snapshot!(
         p("channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000 OR colour = red OR colour = blue AND size = 7"),
         @"OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, {colour} = {red}, AND[{colour} = {blue}, {size} = {7}, ], ]"
         );
 
         // Test parentheses
-        insta::assert_display_snapshot!(p("channel = ponce AND ( 'dog race' != 'bernese mountain' OR subscribers > 1000 )"), @"AND[{channel} = {ponce}, OR[{dog race} != {bernese mountain}, {subscribers} > {1000}, ], ]");
-        insta::assert_display_snapshot!(p("(channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000) AND _geoRadius(12, 13, 14)"), @"AND[OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, ], _geoRadius({12}, {13}, {14}), ]");
+        insta::assert_snapshot!(p("channel = ponce AND ( 'dog race' != 'bernese mountain' OR subscribers > 1000 )"), @"AND[{channel} = {ponce}, OR[{dog race} != {bernese mountain}, {subscribers} > {1000}, ], ]");
+        insta::assert_snapshot!(p("(channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000) AND _geoRadius(12, 13, 14)"), @"AND[OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, ], _geoRadius({12}, {13}, {14}), ]");
 
         // Test recursion
         // This is the most that is allowed
-        insta::assert_display_snapshot!(
+        insta::assert_snapshot!(
             p("(((((((((((((((((((((((((((((((((((((((((((((((((x = 1)))))))))))))))))))))))))))))))))))))))))))))))))"),
             @"{x} = {1}"
         );
-        insta::assert_display_snapshot!(
+        insta::assert_snapshot!(
             p("NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT x = 1"),
             @"NOT ({x} = {1})"
         );
 
         // Confusing keywords
-        insta::assert_display_snapshot!(p(r#"NOT "OR" EXISTS AND "EXISTS" NOT EXISTS"#), @"AND[NOT ({OR} EXISTS), NOT ({EXISTS} EXISTS), ]");
+        insta::assert_snapshot!(p(r#"NOT "OR" EXISTS AND "EXISTS" NOT EXISTS"#), @"AND[NOT ({OR} EXISTS), NOT ({EXISTS} EXISTS), ]");
     }
 
     #[test]
@@ -689,182 +689,182 @@ pub mod tests {
             Fc::parse(s).unwrap_err().to_string()
         }
 
-        insta::assert_display_snapshot!(p("channel = Ponce = 12"), @r###"
+        insta::assert_snapshot!(p("channel = Ponce = 12"), @r###"
         Found unexpected characters at the end of the filter: `= 12`. You probably forgot an `OR` or an `AND` rule.
         17:21 channel = Ponce = 12
         "###);
 
-        insta::assert_display_snapshot!(p("channel =    "), @r###"
+        insta::assert_snapshot!(p("channel =    "), @r###"
         Was expecting a value but instead got nothing.
         14:14 channel =
         "###);
 
-        insta::assert_display_snapshot!(p("channel = 🐻"), @r###"
+        insta::assert_snapshot!(p("channel = 🐻"), @r###"
         Was expecting a value but instead got `🐻`.
         11:12 channel = 🐻
         "###);
 
-        insta::assert_display_snapshot!(p("channel = 🐻 AND followers < 100"), @r###"
+        insta::assert_snapshot!(p("channel = 🐻 AND followers < 100"), @r###"
         Was expecting a value but instead got `🐻`.
         11:12 channel = 🐻 AND followers < 100
         "###);
 
-        insta::assert_display_snapshot!(p("'OR'"), @r###"
+        insta::assert_snapshot!(p("'OR'"), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `\'OR\'`.
         1:5 'OR'
         "###);
 
-        insta::assert_display_snapshot!(p("OR"), @r###"
+        insta::assert_snapshot!(p("OR"), @r###"
         Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes.
         1:3 OR
         "###);
 
-        insta::assert_display_snapshot!(p("channel Ponce"), @r###"
+        insta::assert_snapshot!(p("channel Ponce"), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `channel Ponce`.
         1:14 channel Ponce
         "###);
 
-        insta::assert_display_snapshot!(p("channel = Ponce OR"), @r###"
+        insta::assert_snapshot!(p("channel = Ponce OR"), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.
         19:19 channel = Ponce OR
         "###);
 
-        insta::assert_display_snapshot!(p("_geoRadius"), @r###"
+        insta::assert_snapshot!(p("_geoRadius"), @r###"
         The `_geoRadius` filter expects three arguments: `_geoRadius(latitude, longitude, radius)`.
         1:11 _geoRadius
         "###);
 
-        insta::assert_display_snapshot!(p("_geoRadius = 12"), @r###"
+        insta::assert_snapshot!(p("_geoRadius = 12"), @r###"
         The `_geoRadius` filter expects three arguments: `_geoRadius(latitude, longitude, radius)`.
         1:16 _geoRadius = 12
         "###);
 
-        insta::assert_display_snapshot!(p("_geoBoundingBox"), @r###"
+        insta::assert_snapshot!(p("_geoBoundingBox"), @r###"
         The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
         1:16 _geoBoundingBox
         "###);
 
-        insta::assert_display_snapshot!(p("_geoBoundingBox = 12"), @r###"
+        insta::assert_snapshot!(p("_geoBoundingBox = 12"), @r###"
         The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
         1:21 _geoBoundingBox = 12
         "###);
 
-        insta::assert_display_snapshot!(p("_geoBoundingBox(1.0, 1.0)"), @r###"
+        insta::assert_snapshot!(p("_geoBoundingBox(1.0, 1.0)"), @r###"
         The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
         1:26 _geoBoundingBox(1.0, 1.0)
         "###);
 
-        insta::assert_display_snapshot!(p("_geoPoint(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("_geoPoint(12, 13, 14)"), @r###"
         `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
         1:22 _geoPoint(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("position <= _geoPoint(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("position <= _geoPoint(12, 13, 14)"), @r###"
         `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
         13:34 position <= _geoPoint(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("_geoDistance(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("_geoDistance(12, 13, 14)"), @r###"
         `_geoDistance` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
         1:25 _geoDistance(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("position <= _geoDistance(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("position <= _geoDistance(12, 13, 14)"), @r###"
         `_geoDistance` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
         13:37 position <= _geoDistance(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("_geo(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("_geo(12, 13, 14)"), @r###"
         `_geo` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
         1:17 _geo(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("position <= _geo(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("position <= _geo(12, 13, 14)"), @r###"
         `_geo` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
         13:29 position <= _geo(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("position <= _geoRadius(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("position <= _geoRadius(12, 13, 14)"), @r###"
         The `_geoRadius` filter is an operation and can't be used as a value.
         13:35 position <= _geoRadius(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("channel = 'ponce"), @r###"
+        insta::assert_snapshot!(p("channel = 'ponce"), @r###"
         Expression `\'ponce` is missing the following closing delimiter: `'`.
         11:17 channel = 'ponce
         "###);
 
-        insta::assert_display_snapshot!(p("channel = \"ponce"), @r###"
+        insta::assert_snapshot!(p("channel = \"ponce"), @r###"
         Expression `\"ponce` is missing the following closing delimiter: `"`.
         11:17 channel = "ponce
         "###);
 
-        insta::assert_display_snapshot!(p("channel = mv OR (followers >= 1000"), @r###"
+        insta::assert_snapshot!(p("channel = mv OR (followers >= 1000"), @r###"
         Expression `(followers >= 1000` is missing the following closing delimiter: `)`.
         17:35 channel = mv OR (followers >= 1000
         "###);
 
-        insta::assert_display_snapshot!(p("channel = mv OR followers >= 1000)"), @r###"
+        insta::assert_snapshot!(p("channel = mv OR followers >= 1000)"), @r###"
         Found unexpected characters at the end of the filter: `)`. You probably forgot an `OR` or an `AND` rule.
         34:35 channel = mv OR followers >= 1000)
         "###);
 
-        insta::assert_display_snapshot!(p("colour NOT EXIST"), @r###"
+        insta::assert_snapshot!(p("colour NOT EXIST"), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `colour NOT EXIST`.
         1:17 colour NOT EXIST
         "###);
 
-        insta::assert_display_snapshot!(p("subscribers 100 TO1000"), @r###"
+        insta::assert_snapshot!(p("subscribers 100 TO1000"), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `subscribers 100 TO1000`.
         1:23 subscribers 100 TO1000
         "###);
 
-        insta::assert_display_snapshot!(p("channel = ponce ORdog != 'bernese mountain'"), @r###"
+        insta::assert_snapshot!(p("channel = ponce ORdog != 'bernese mountain'"), @r###"
         Found unexpected characters at the end of the filter: `ORdog != \'bernese mountain\'`. You probably forgot an `OR` or an `AND` rule.
         17:44 channel = ponce ORdog != 'bernese mountain'
         "###);
 
-        insta::assert_display_snapshot!(p("colour IN blue, green]"), @r###"
+        insta::assert_snapshot!(p("colour IN blue, green]"), @r###"
         Expected `[` after `IN` keyword.
         11:23 colour IN blue, green]
         "###);
 
-        insta::assert_display_snapshot!(p("colour IN [blue, green, 'blue' > 2]"), @r###"
+        insta::assert_snapshot!(p("colour IN [blue, green, 'blue' > 2]"), @r###"
         Expected only comma-separated field names inside `IN[..]` but instead found `> 2]`.
         32:36 colour IN [blue, green, 'blue' > 2]
         "###);
 
-        insta::assert_display_snapshot!(p("colour IN [blue, green, AND]"), @r###"
+        insta::assert_snapshot!(p("colour IN [blue, green, AND]"), @r###"
         Expected only comma-separated field names inside `IN[..]` but instead found `AND]`.
         25:29 colour IN [blue, green, AND]
         "###);
 
-        insta::assert_display_snapshot!(p("colour IN [blue, green"), @r###"
+        insta::assert_snapshot!(p("colour IN [blue, green"), @r###"
         Expected matching `]` after the list of field names given to `IN[`
         23:23 colour IN [blue, green
         "###);
 
-        insta::assert_display_snapshot!(p("colour IN ['blue, green"), @r###"
+        insta::assert_snapshot!(p("colour IN ['blue, green"), @r###"
         Expression `\'blue, green` is missing the following closing delimiter: `'`.
         12:24 colour IN ['blue, green
         "###);
 
-        insta::assert_display_snapshot!(p("x = EXISTS"), @r###"
+        insta::assert_snapshot!(p("x = EXISTS"), @r###"
         Was expecting a value but instead got `EXISTS`, which is a reserved keyword. To use `EXISTS` as a field name or a value, surround it by quotes.
         5:11 x = EXISTS
         "###);
 
-        insta::assert_display_snapshot!(p("AND = 8"), @r###"
+        insta::assert_snapshot!(p("AND = 8"), @r###"
         Was expecting a value but instead got `AND`, which is a reserved keyword. To use `AND` as a field name or a value, surround it by quotes.
         1:4 AND = 8
         "###);
 
-        insta::assert_display_snapshot!(p("((((((((((((((((((((((((((((((((((((((((((((((((((x = 1))))))))))))))))))))))))))))))))))))))))))))))))))"), @r###"
+        insta::assert_snapshot!(p("((((((((((((((((((((((((((((((((((((((((((((((((((x = 1))))))))))))))))))))))))))))))))))))))))))))))))))"), @r###"
         The filter exceeded the maximum depth limit. Try rewriting the filter so that it contains fewer nested conditions.
         51:106 ((((((((((((((((((((((((((((((((((((((((((((((((((x = 1))))))))))))))))))))))))))))))))))))))))))))))))))
         "###);
 
-        insta::assert_display_snapshot!(
+        insta::assert_snapshot!(
             p("NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT x = 1"),
             @r###"
         The filter exceeded the maximum depth limit. Try rewriting the filter so that it contains fewer nested conditions.
@@ -872,40 +872,40 @@ pub mod tests {
         "###
         );
 
-        insta::assert_display_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###"
+        insta::assert_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###"
         Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes.
         5:7 NOT OR EXISTS AND EXISTS NOT EXISTS
         "###);
 
-        insta::assert_display_snapshot!(p(r#"value NULL"#), @r###"
+        insta::assert_snapshot!(p(r#"value NULL"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value NULL`.
         1:11 value NULL
         "###);
-        insta::assert_display_snapshot!(p(r#"value NOT NULL"#), @r###"
+        insta::assert_snapshot!(p(r#"value NOT NULL"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value NOT NULL`.
         1:15 value NOT NULL
         "###);
-        insta::assert_display_snapshot!(p(r#"value EMPTY"#), @r###"
+        insta::assert_snapshot!(p(r#"value EMPTY"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value EMPTY`.
         1:12 value EMPTY
         "###);
-        insta::assert_display_snapshot!(p(r#"value NOT EMPTY"#), @r###"
+        insta::assert_snapshot!(p(r#"value NOT EMPTY"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value NOT EMPTY`.
         1:16 value NOT EMPTY
         "###);
-        insta::assert_display_snapshot!(p(r#"value IS"#), @r###"
+        insta::assert_snapshot!(p(r#"value IS"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS`.
         1:9 value IS
         "###);
-        insta::assert_display_snapshot!(p(r#"value IS NOT"#), @r###"
+        insta::assert_snapshot!(p(r#"value IS NOT"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT`.
         1:13 value IS NOT
         "###);
-        insta::assert_display_snapshot!(p(r#"value IS EXISTS"#), @r###"
+        insta::assert_snapshot!(p(r#"value IS EXISTS"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS EXISTS`.
         1:16 value IS EXISTS
         "###);
-        insta::assert_display_snapshot!(p(r#"value IS NOT EXISTS"#), @r###"
+        insta::assert_snapshot!(p(r#"value IS NOT EXISTS"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT EXISTS`.
         1:20 value IS NOT EXISTS
         "###);

fuzzers/Cargo.toml

@@ -12,9 +12,9 @@ license.workspace = true
 
 [dependencies]
 arbitrary = { version = "1.3.2", features = ["derive"] }
-clap = { version = "4.4.17", features = ["derive"] }
-fastrand = "2.0.1"
+clap = { version = "4.5.9", features = ["derive"] }
+fastrand = "2.1.0"
 milli = { path = "../milli" }
-serde = { version = "1.0.195", features = ["derive"] }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
-tempfile = "3.9.0"
+serde = { version = "1.0.204", features = ["derive"] }
+serde_json = { version = "1.0.120", features = ["preserve_order"] }
+tempfile = "3.10.1"

index-scheduler/Cargo.toml

@@ -11,36 +11,38 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.79"
+anyhow = "1.0.86"
 bincode = "1.3.3"
 csv = "1.3.0"
-derive_builder = "0.12.0"
+derive_builder = "0.20.0"
 dump = { path = "../dump" }
-enum-iterator = "1.5.0"
+enum-iterator = "2.1.0"
 file-store = { path = "../file-store" }
-flate2 = "1.0.28"
+flate2 = "1.0.30"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-page_size = "0.5.0"
-rayon = "1.8.1"
-roaring = { version = "0.10.2", features = ["serde"] }
-serde = { version = "1.0.195", features = ["derive"] }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
+page_size = "0.6.0"
+rayon = "1.10.0"
+roaring = { version = "0.10.6", features = ["serde"] }
+serde = { version = "1.0.204", features = ["derive"] }
+serde_json = { version = "1.0.120", features = ["preserve_order"] }
 synchronoise = "1.0.1"
-tempfile = "3.9.0"
-thiserror = "1.0.56"
-time = { version = "0.3.31", features = [
+tempfile = "3.10.1"
+thiserror = "1.0.61"
+time = { version = "0.3.36", features = [
     "serde-well-known",
     "formatting",
     "parsing",
     "macros",
 ] }
 tracing = "0.1.40"
-ureq = "2.9.7"
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
+ureq = "2.10.0"
+uuid = { version = "1.10.0", features = ["serde", "v4"] }
 
 [dev-dependencies]
+arroy = "0.4.0"
 big_s = "1.0.2"
 crossbeam = "0.8.4"
-insta = { version = "1.34.0", features = ["json", "redactions"] }
+insta = { version = "1.39.0", features = ["json", "redactions"] }
+maplit = "1.0.2"
 meili-snap = { path = "../meili-snap" }

index-scheduler/src/autobatcher.rs

@@ -24,6 +24,7 @@ enum AutobatchKind {
         allow_index_creation: bool,
         primary_key: Option<String>,
     },
+    DocumentEdition,
     DocumentDeletion,
     DocumentDeletionByFilter,
     DocumentClear,
@@ -63,6 +64,7 @@ impl From<KindWithContent> for AutobatchKind {
                 primary_key,
                 ..
             } => AutobatchKind::DocumentImport { method, allow_index_creation, primary_key },
+            KindWithContent::DocumentEdition { .. } => AutobatchKind::DocumentEdition,
             KindWithContent::DocumentDeletion { .. } => AutobatchKind::DocumentDeletion,
             KindWithContent::DocumentClear { .. } => AutobatchKind::DocumentClear,
             KindWithContent::DocumentDeletionByFilter { .. } => {
@@ -98,6 +100,9 @@ pub enum BatchKind {
         primary_key: Option<String>,
         operation_ids: Vec<TaskId>,
     },
+    DocumentEdition {
+        id: TaskId,
+    },
     DocumentDeletion {
         deletion_ids: Vec<TaskId>,
     },
@@ -199,6 +204,7 @@ impl BatchKind {
                 }),
                 allow_index_creation,
             ),
+            K::DocumentEdition => (Break(BatchKind::DocumentEdition { id: task_id }), false),
             K::DocumentDeletion => {
                 (Continue(BatchKind::DocumentDeletion { deletion_ids: vec![task_id] }), false)
             }
@@ -222,7 +228,7 @@ impl BatchKind {
 
         match (self, kind) {
             // We don't batch any of these operations
-            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentDeletionByFilter) => Break(this),
+            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition | K::DocumentDeletionByFilter) => Break(this),
             // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists.
             (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => {
                 Break(this)
@@ -519,6 +525,7 @@ impl BatchKind {
                 | BatchKind::IndexDeletion { .. }
                 | BatchKind::IndexUpdate { .. }
                 | BatchKind::IndexSwap { .. }
+                | BatchKind::DocumentEdition { .. }
                 | BatchKind::DocumentDeletionByFilter { .. },
                 _,
             ) => {

index-scheduler/src/batch.rs

@@ -34,7 +34,7 @@ use meilisearch_types::milli::update::{
 use meilisearch_types::milli::vector::parsed_vectors::{
     ExplicitVectors, VectorOrArrayOfVectors, RESERVED_VECTORS_FIELD_NAME,
 };
-use meilisearch_types::milli::{self, Filter};
+use meilisearch_types::milli::{self, Filter, Object};
 use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked};
 use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status, Task};
 use meilisearch_types::{compression, Index, VERSION_FILE_NAME};
@@ -106,6 +106,10 @@ pub(crate) enum IndexOperation {
         operations: Vec<DocumentOperation>,
         tasks: Vec<Task>,
     },
+    DocumentEdition {
+        index_uid: String,
+        task: Task,
+    },
     IndexDocumentDeletionByFilter {
         index_uid: String,
         task: Task,
@@ -164,7 +168,8 @@ impl Batch {
                 | IndexOperation::DocumentClear { tasks, .. } => {
                     RoaringBitmap::from_iter(tasks.iter().map(|task| task.uid))
                 }
-                IndexOperation::IndexDocumentDeletionByFilter { task, .. } => {
+                IndexOperation::DocumentEdition { task, .. }
+                | IndexOperation::IndexDocumentDeletionByFilter { task, .. } => {
                     RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
                 }
                 IndexOperation::SettingsAndDocumentOperation {
@@ -228,6 +233,7 @@ impl IndexOperation {
     pub fn index_uid(&self) -> &str {
         match self {
             IndexOperation::DocumentOperation { index_uid, .. }
+            | IndexOperation::DocumentEdition { index_uid, .. }
             | IndexOperation::IndexDocumentDeletionByFilter { index_uid, .. }
             | IndexOperation::DocumentClear { index_uid, .. }
             | IndexOperation::Settings { index_uid, .. }
@@ -243,6 +249,9 @@ impl fmt::Display for IndexOperation {
             IndexOperation::DocumentOperation { .. } => {
                 f.write_str("IndexOperation::DocumentOperation")
             }
+            IndexOperation::DocumentEdition { .. } => {
+                f.write_str("IndexOperation::DocumentEdition")
+            }
             IndexOperation::IndexDocumentDeletionByFilter { .. } => {
                 f.write_str("IndexOperation::IndexDocumentDeletionByFilter")
             }
@@ -295,6 +304,21 @@ impl IndexScheduler {
                     _ => unreachable!(),
                 }
             }
+            BatchKind::DocumentEdition { id } => {
+                let task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
+                match &task.kind {
+                    KindWithContent::DocumentEdition { index_uid, .. } => {
+                        Ok(Some(Batch::IndexOperation {
+                            op: IndexOperation::DocumentEdition {
+                                index_uid: index_uid.clone(),
+                                task,
+                            },
+                            must_create_index: false,
+                        }))
+                    }
+                    _ => unreachable!(),
+                }
+            }
             BatchKind::DocumentOperation { method, operation_ids, .. } => {
                 let tasks = self.get_existing_tasks(rtxn, operation_ids)?;
                 let primary_key = tasks
@@ -909,6 +933,7 @@ impl IndexScheduler {
 
                     let fields_ids_map = index.fields_ids_map(&rtxn)?;
                     let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
+                    let embedding_configs = index.embedding_configs(&rtxn)?;
 
                     // 3.1. Dump the documents
                     for ret in index.all_documents(&rtxn)? {
@@ -951,16 +976,21 @@ impl IndexScheduler {
                             };
 
                             for (embedder_name, embeddings) in embeddings {
-                                // don't change the entry if it already exists, because it was user-provided
-                                vectors.entry(embedder_name).or_insert_with(|| {
-                                    let embeddings = ExplicitVectors {
-                                        embeddings: VectorOrArrayOfVectors::from_array_of_vectors(
-                                            embeddings,
-                                        ),
-                                        user_provided: false,
-                                    };
-                                    serde_json::to_value(embeddings).unwrap()
-                                });
+                                let user_provided = embedding_configs
+                                    .iter()
+                                    .find(|conf| conf.name == embedder_name)
+                                    .is_some_and(|conf| conf.user_provided.contains(id));
+
+                                let embeddings = ExplicitVectors {
+                                    embeddings: Some(
+                                        VectorOrArrayOfVectors::from_array_of_vectors(embeddings),
+                                    ),
+                                    regenerate: !user_provided,
+                                };
+                                vectors.insert(
+                                    embedder_name,
+                                    serde_json::to_value(embeddings).unwrap(),
+                                );
                             }
                         }
 
@@ -1380,6 +1410,64 @@ impl IndexScheduler {
 
                 Ok(tasks)
             }
+            IndexOperation::DocumentEdition { mut task, .. } => {
+                let (filter, context, function) =
+                    if let KindWithContent::DocumentEdition {
+                        filter_expr, context, function, ..
+                    } = &task.kind
+                    {
+                        (filter_expr, context, function)
+                    } else {
+                        unreachable!()
+                    };
+                let result_count = edit_documents_by_function(
+                    index_wtxn,
+                    filter,
+                    context.clone(),
+                    function,
+                    self.index_mapper.indexer_config(),
+                    self.must_stop_processing.clone(),
+                    index,
+                );
+                let (original_filter, context, function) = if let Some(Details::DocumentEdition {
+                    original_filter,
+                    context,
+                    function,
+                    ..
+                }) = task.details
+                {
+                    (original_filter, context, function)
+                } else {
+                    // In the case of a `documentDeleteByFilter` the details MUST be set
+                    unreachable!();
+                };
+
+                match result_count {
+                    Ok((deleted_documents, edited_documents)) => {
+                        task.status = Status::Succeeded;
+                        task.details = Some(Details::DocumentEdition {
+                            original_filter,
+                            context,
+                            function,
+                            deleted_documents: Some(deleted_documents),
+                            edited_documents: Some(edited_documents),
+                        });
+                    }
+                    Err(e) => {
+                        task.status = Status::Failed;
+                        task.details = Some(Details::DocumentEdition {
+                            original_filter,
+                            context,
+                            function,
+                            deleted_documents: Some(0),
+                            edited_documents: Some(0),
+                        });
+                        task.error = Some(e.into());
+                    }
+                }
+
+                Ok(vec![task])
+            }
             IndexOperation::IndexDocumentDeletionByFilter { mut task, index_uid: _ } => {
                 let filter =
                     if let KindWithContent::DocumentDeletionByFilter { filter_expr, .. } =
@@ -1668,3 +1756,44 @@ fn delete_document_by_filter<'a>(
         0
     })
 }
+
+fn edit_documents_by_function<'a>(
+    wtxn: &mut RwTxn<'a>,
+    filter: &Option<serde_json::Value>,
+    context: Option<Object>,
+    code: &str,
+    indexer_config: &IndexerConfig,
+    must_stop_processing: MustStopProcessing,
+    index: &'a Index,
+) -> Result<(u64, u64)> {
+    let candidates = match filter.as_ref().map(Filter::from_json) {
+        Some(Ok(Some(filter))) => filter.evaluate(wtxn, index).map_err(|err| match err {
+            milli::Error::UserError(milli::UserError::InvalidFilter(_)) => {
+                Error::from(err).with_custom_error_code(Code::InvalidDocumentFilter)
+            }
+            e => e.into(),
+        })?,
+        None | Some(Ok(None)) => index.documents_ids(wtxn)?,
+        Some(Err(e)) => return Err(e.into()),
+    };
+
+    let config = IndexDocumentsConfig {
+        update_method: IndexDocumentsMethod::ReplaceDocuments,
+        ..Default::default()
+    };
+
+    let mut builder = milli::update::IndexDocuments::new(
+        wtxn,
+        index,
+        indexer_config,
+        config,
+        |indexing_step| tracing::debug!(update = ?indexing_step),
+        || must_stop_processing.get(),
+    )?;
+
+    let (new_builder, count) = builder.edit_documents(&candidates, context, code)?;
+    builder = new_builder;
+
+    let _ = builder.execute()?;
+    Ok(count.unwrap())
+}

index-scheduler/src/features.rs

@@ -68,6 +68,19 @@ impl RoFeatures {
             .into())
         }
     }
+
+    pub fn check_edit_documents_by_function(&self, disabled_action: &'static str) -> Result<()> {
+        if self.runtime.edit_documents_by_function {
+            Ok(())
+        } else {
+            Err(FeatureNotEnabledError {
+                disabled_action,
+                feature: "edit documents by function",
+                issue_link: "https://github.com/orgs/meilisearch/discussions/762",
+            }
+            .into())
+        }
+    }
 }
 
 impl FeatureData {

index-scheduler/src/insta_snapshot.rs

@@ -177,6 +177,17 @@ fn snapshot_details(d: &Details) -> String {
         } => {
             format!("{{ received_documents: {received_documents}, indexed_documents: {indexed_documents:?} }}")
         }
+        Details::DocumentEdition {
+            deleted_documents,
+            edited_documents,
+            original_filter,
+            context,
+            function,
+        } => {
+            format!(
+                "{{ deleted_documents: {deleted_documents:?}, edited_documents: {edited_documents:?}, context: {context:?}, function: {function:?}, original_filter: {original_filter:?} }}"
+            )
+        }
         Details::SettingsUpdate { settings } => {
             format!("{{ settings: {settings:?} }}")
         }

index-scheduler/src/lib.rs

@@ -53,6 +53,7 @@ use meilisearch_types::heed::byteorder::BE;
 use meilisearch_types::heed::types::{SerdeBincode, SerdeJson, Str, I128};
 use meilisearch_types::heed::{self, Database, Env, PutFlags, RoTxn, RwTxn};
 use meilisearch_types::milli::documents::DocumentsBatchBuilder;
+use meilisearch_types::milli::index::IndexEmbeddingConfig;
 use meilisearch_types::milli::update::IndexerConfig;
 use meilisearch_types::milli::vector::{Embedder, EmbedderOptions, EmbeddingConfigs};
 use meilisearch_types::milli::{self, CboRoaringBitmapCodec, Index, RoaringBitmapCodec, BEU32};
@@ -1459,33 +1460,39 @@ impl IndexScheduler {
     // TODO: consider using a type alias or a struct embedder/template
     pub fn embedders(
         &self,
-        embedding_configs: Vec<(String, milli::vector::EmbeddingConfig)>,
+        embedding_configs: Vec<IndexEmbeddingConfig>,
     ) -> Result<EmbeddingConfigs> {
         let res: Result<_> = embedding_configs
             .into_iter()
-            .map(|(name, milli::vector::EmbeddingConfig { embedder_options, prompt })| {
-                let prompt =
-                    Arc::new(prompt.try_into().map_err(meilisearch_types::milli::Error::from)?);
-                // optimistically return existing embedder
-                {
-                    let embedders = self.embedders.read().unwrap();
-                    if let Some(embedder) = embedders.get(&embedder_options) {
-                        return Ok((name, (embedder.clone(), prompt)));
+            .map(
+                |IndexEmbeddingConfig {
+                     name,
+                     config: milli::vector::EmbeddingConfig { embedder_options, prompt },
+                     ..
+                 }| {
+                    let prompt =
+                        Arc::new(prompt.try_into().map_err(meilisearch_types::milli::Error::from)?);
+                    // optimistically return existing embedder
+                    {
+                        let embedders = self.embedders.read().unwrap();
+                        if let Some(embedder) = embedders.get(&embedder_options) {
+                            return Ok((name, (embedder.clone(), prompt)));
+                        }
                     }
-                }
 
-                // add missing embedder
-                let embedder = Arc::new(
-                    Embedder::new(embedder_options.clone())
-                        .map_err(meilisearch_types::milli::vector::Error::from)
-                        .map_err(meilisearch_types::milli::Error::from)?,
-                );
-                {
-                    let mut embedders = self.embedders.write().unwrap();
-                    embedders.insert(embedder_options, embedder.clone());
-                }
-                Ok((name, (embedder, prompt)))
-            })
+                    // add missing embedder
+                    let embedder = Arc::new(
+                        Embedder::new(embedder_options.clone())
+                            .map_err(meilisearch_types::milli::vector::Error::from)
+                            .map_err(meilisearch_types::milli::Error::from)?,
+                    );
+                    {
+                        let mut embedders = self.embedders.write().unwrap();
+                        embedders.insert(embedder_options, embedder.clone());
+                    }
+                    Ok((name, (embedder, prompt)))
+                },
+            )
             .collect();
         res.map(EmbeddingConfigs::new)
     }
@@ -1596,6 +1603,14 @@ impl<'a> Dump<'a> {
                         index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
                     }
                 }
+                KindDump::DocumentEdition { filter, context, function } => {
+                    KindWithContent::DocumentEdition {
+                        index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
+                        filter_expr: filter,
+                        context,
+                        function,
+                    }
+                }
                 KindDump::DocumentClear => KindWithContent::DocumentClear {
                     index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
                 },
@@ -1748,6 +1763,9 @@ mod tests {
     use meilisearch_types::milli::update::IndexDocumentsMethod::{
         ReplaceDocuments, UpdateDocuments,
     };
+    use meilisearch_types::milli::update::Setting;
+    use meilisearch_types::milli::vector::settings::EmbeddingSettings;
+    use meilisearch_types::settings::Unchecked;
     use meilisearch_types::tasks::IndexSwap;
     use meilisearch_types::VERSION_FILE_NAME;
     use tempfile::{NamedTempFile, TempDir};
@@ -1801,7 +1819,7 @@ mod tests {
                 task_db_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose.
                 index_base_map_size: 1000 * 1000, // 1 MB, we don't use MiB on purpose.
                 enable_mdb_writemap: false,
-                index_growth_amount: 1000 * 1000, // 1 MB
+                index_growth_amount: 1000 * 1000 * 1000 * 1000, // 1 TB
                 index_count: 5,
                 indexer_config,
                 autobatching_enabled: true,
@@ -1826,6 +1844,7 @@ mod tests {
             assert_eq!(breakpoint, (Init, false));
             let index_scheduler_handle = IndexSchedulerHandle {
                 _tempdir: tempdir,
+                index_scheduler: index_scheduler.private_clone(),
                 test_breakpoint_rcv: receiver,
                 last_breakpoint: breakpoint.0,
             };
@@ -1914,6 +1933,7 @@ mod tests {
 
     pub struct IndexSchedulerHandle {
         _tempdir: TempDir,
+        index_scheduler: IndexScheduler,
         test_breakpoint_rcv: crossbeam::channel::Receiver<(Breakpoint, bool)>,
         last_breakpoint: Breakpoint,
     }
@@ -1931,9 +1951,13 @@ mod tests {
             {
                 Ok(b) => b,
                 Err(RecvTimeoutError::Timeout) => {
-                    panic!("The scheduler seems to be waiting for a new task while your test is waiting for a breakpoint.")
+                    let state = snapshot_index_scheduler(&self.index_scheduler);
+                    panic!("The scheduler seems to be waiting for a new task while your test is waiting for a breakpoint.\n{state}")
+                }
+                Err(RecvTimeoutError::Disconnected) => {
+                    let state = snapshot_index_scheduler(&self.index_scheduler);
+                    panic!("The scheduler crashed.\n{state}")
                 }
-                Err(RecvTimeoutError::Disconnected) => panic!("The scheduler crashed."),
             };
             // if we've already encountered a breakpoint we're supposed to be stuck on the false
             // and we expect the same variant with the true to come now.
@@ -1952,9 +1976,13 @@ mod tests {
             {
                 Ok(b) => b,
                 Err(RecvTimeoutError::Timeout) => {
-                    panic!("The scheduler seems to be waiting for a new task while your test is waiting for a breakpoint.")
+                    let state = snapshot_index_scheduler(&self.index_scheduler);
+                    panic!("The scheduler seems to be waiting for a new task while your test is waiting for a breakpoint.\n{state}")
+                }
+                Err(RecvTimeoutError::Disconnected) => {
+                    let state = snapshot_index_scheduler(&self.index_scheduler);
+                    panic!("The scheduler crashed.\n{state}")
                 }
-                Err(RecvTimeoutError::Disconnected) => panic!("The scheduler crashed."),
             };
             assert!(!b, "Found the breakpoint handle in a bad state. Check your test suite");
 
@@ -1968,9 +1996,10 @@ mod tests {
         fn advance_till(&mut self, breakpoints: impl IntoIterator<Item = Breakpoint>) {
             for breakpoint in breakpoints {
                 let b = self.advance();
+                let state = snapshot_index_scheduler(&self.index_scheduler);
                 assert_eq!(
                     b, breakpoint,
-                    "Was expecting the breakpoint `{:?}` but instead got `{:?}`.",
+                    "Was expecting the breakpoint `{:?}` but instead got `{:?}`.\n{state}",
                     breakpoint, b
                 );
             }
@@ -1995,6 +2024,7 @@ mod tests {
         // Wait for one successful batch.
         #[track_caller]
         fn advance_one_successful_batch(&mut self) {
+            self.index_scheduler.assert_internally_consistent();
             self.advance_till([Start, BatchCreated]);
             loop {
                 match self.advance() {
@@ -2003,13 +2033,17 @@ mod tests {
                     InsideProcessBatch => (),
                     // the batch went successfully, we can stop the loop and go on with the next states.
                     ProcessBatchSucceeded => break,
-                    AbortedIndexation => panic!("The batch was aborted."),
-                    ProcessBatchFailed => panic!("The batch failed."),
+                    AbortedIndexation => panic!("The batch was aborted.\n{}", snapshot_index_scheduler(&self.index_scheduler)),
+                    ProcessBatchFailed => {
+                        while self.advance() != Start {}
+                        panic!("The batch failed.\n{}", snapshot_index_scheduler(&self.index_scheduler))
+                    },
                     breakpoint => panic!("Encountered an impossible breakpoint `{:?}`, this is probably an issue with the test suite.", breakpoint),
                 }
             }
 
             self.advance_till([AfterProcessing]);
+            self.index_scheduler.assert_internally_consistent();
         }
 
         // Wait for one failed batch.
@@ -2023,8 +2057,8 @@ mod tests {
                     InsideProcessBatch => (),
                     // the batch went failed, we can stop the loop and go on with the next states.
                     ProcessBatchFailed => break,
-                    ProcessBatchSucceeded => panic!("The batch succeeded. (and it wasn't supposed to sorry)"),
-                    AbortedIndexation => panic!("The batch was aborted."),
+                    ProcessBatchSucceeded => panic!("The batch succeeded. (and it wasn't supposed to sorry)\n{}", snapshot_index_scheduler(&self.index_scheduler)),
+                    AbortedIndexation => panic!("The batch was aborted.\n{}", snapshot_index_scheduler(&self.index_scheduler)),
                     breakpoint => panic!("Encountered an impossible breakpoint `{:?}`, this is probably an issue with the test suite.", breakpoint),
                 }
             }
@@ -3052,8 +3086,10 @@ mod tests {
         let rtxn = index.read_txn().unwrap();
 
         let configs = index.embedding_configs(&rtxn).unwrap();
-        let (_, embedding_config) = configs.first().unwrap();
-        insta::assert_json_snapshot!(embedding_config.embedder_options);
+        let IndexEmbeddingConfig { name, config, user_provided } = configs.first().unwrap();
+        insta::assert_snapshot!(name, @"default");
+        insta::assert_debug_snapshot!(user_provided, @"RoaringBitmap<[]>");
+        insta::assert_json_snapshot!(config.embedder_options);
     }
 
     #[test]
@@ -4716,6 +4752,7 @@ mod tests {
           "types": {
             "documentAdditionOrUpdate": 0,
             "documentDeletion": 0,
+            "documentEdition": 0,
             "dumpCreation": 0,
             "indexCreation": 3,
             "indexDeletion": 0,
@@ -4747,6 +4784,7 @@ mod tests {
           "types": {
             "documentAdditionOrUpdate": 0,
             "documentDeletion": 0,
+            "documentEdition": 0,
             "dumpCreation": 0,
             "indexCreation": 3,
             "indexDeletion": 0,
@@ -4785,6 +4823,7 @@ mod tests {
           "types": {
             "documentAdditionOrUpdate": 0,
             "documentDeletion": 0,
+            "documentEdition": 0,
             "dumpCreation": 0,
             "indexCreation": 3,
             "indexDeletion": 0,
@@ -4824,6 +4863,7 @@ mod tests {
           "types": {
             "documentAdditionOrUpdate": 0,
             "documentDeletion": 0,
+            "documentEdition": 0,
             "dumpCreation": 0,
             "indexCreation": 3,
             "indexDeletion": 0,
@@ -4989,7 +5029,6 @@ mod tests {
                 false,
             )
             .unwrap();
-        index_scheduler.assert_internally_consistent();
 
         snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_registering_settings_task_vectors");
 
@@ -5000,7 +5039,7 @@ mod tests {
             insta::assert_json_snapshot!(task.details);
         }
 
-        handle.advance_n_successful_batches(1);
+        handle.advance_one_successful_batch();
         snapshot!(snapshot_index_scheduler(&index_scheduler), name: "settings_update_processed_vectors");
 
         {
@@ -5017,13 +5056,17 @@ mod tests {
             let configs = index.embedding_configs(&rtxn).unwrap();
             // for consistency with the below
             #[allow(clippy::get_first)]
-            let (name, fakerest_config) = configs.get(0).unwrap();
-            insta::assert_json_snapshot!(name, @r###""A_fakerest""###);
+            let IndexEmbeddingConfig { name, config: fakerest_config, user_provided } =
+                configs.get(0).unwrap();
+            insta::assert_snapshot!(name, @"A_fakerest");
+            insta::assert_debug_snapshot!(user_provided, @"RoaringBitmap<[]>");
             insta::assert_json_snapshot!(fakerest_config.embedder_options);
             let fakerest_name = name.clone();
 
-            let (name, simple_hf_config) = configs.get(1).unwrap();
-            insta::assert_json_snapshot!(name, @r###""B_small_hf""###);
+            let IndexEmbeddingConfig { name, config: simple_hf_config, user_provided } =
+                configs.get(1).unwrap();
+            insta::assert_snapshot!(name, @"B_small_hf");
+            insta::assert_debug_snapshot!(user_provided, @"RoaringBitmap<[]>");
             insta::assert_json_snapshot!(simple_hf_config.embedder_options);
             let simple_hf_name = name.clone();
 
@@ -5038,25 +5081,25 @@ mod tests {
         // add one doc, specifying vectors
 
         let doc = serde_json::json!(
-                    {
-                        "id": 0,
-                        "doggo": "Intel",
-                        "breed": "beagle",
-                        "_vectors": {
-                            &fakerest_name: {
-                                // this will never trigger regeneration, which is good because we can't actually generate with
-                                // this embedder
-                                "userProvided": true,
-                                "embeddings": beagle_embed,
-                            },
-                            &simple_hf_name: {
-                                // this will be regenerated on updates
-                                "userProvided": false,
-                                "embeddings": lab_embed,
-                            },
-                            "noise": [0.1, 0.2, 0.3]
-                        }
-                    }
+            {
+                "id": 0,
+                "doggo": "Intel",
+                "breed": "beagle",
+                "_vectors": {
+                    &fakerest_name: {
+                        // this will never trigger regeneration, which is good because we can't actually generate with
+                        // this embedder
+                        "regenerate": false,
+                        "embeddings": beagle_embed,
+                    },
+                    &simple_hf_name: {
+                        // this will be regenerated on updates
+                        "regenerate": true,
+                        "embeddings": lab_embed,
+                    },
+                    "noise": [0.1, 0.2, 0.3]
+                }
+            }
         );
 
         let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0u128).unwrap();
@@ -5078,7 +5121,6 @@ mod tests {
                 false,
             )
             .unwrap();
-        index_scheduler.assert_internally_consistent();
 
         snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after adding Intel");
 
@@ -5091,6 +5133,19 @@ mod tests {
             let index = index_scheduler.index("doggos").unwrap();
             let rtxn = index.read_txn().unwrap();
 
+            // Ensure the document have been inserted into the relevant bitamp
+            let configs = index.embedding_configs(&rtxn).unwrap();
+            // for consistency with the below
+            #[allow(clippy::get_first)]
+            let IndexEmbeddingConfig { name, config: _, user_provided: user_defined } =
+                configs.get(0).unwrap();
+            insta::assert_snapshot!(name, @"A_fakerest");
+            insta::assert_debug_snapshot!(user_defined, @"RoaringBitmap<[0]>");
+
+            let IndexEmbeddingConfig { name, config: _, user_provided } = configs.get(1).unwrap();
+            insta::assert_snapshot!(name, @"B_small_hf");
+            insta::assert_debug_snapshot!(user_provided, @"RoaringBitmap<[]>");
+
             let embeddings = index.embeddings(&rtxn, 0).unwrap();
 
             assert_json_snapshot!(embeddings[&simple_hf_name][0] == lab_embed, @"true");
@@ -5140,7 +5195,6 @@ mod tests {
                 false,
             )
             .unwrap();
-        index_scheduler.assert_internally_consistent();
 
         snapshot!(snapshot_index_scheduler(&index_scheduler), name: "Intel to kefir");
 
@@ -5153,11 +5207,25 @@ mod tests {
                 let index = index_scheduler.index("doggos").unwrap();
                 let rtxn = index.read_txn().unwrap();
 
+                // Ensure the document have been inserted into the relevant bitamp
+                let configs = index.embedding_configs(&rtxn).unwrap();
+                // for consistency with the below
+                #[allow(clippy::get_first)]
+                let IndexEmbeddingConfig { name, config: _, user_provided: user_defined } =
+                    configs.get(0).unwrap();
+                insta::assert_snapshot!(name, @"A_fakerest");
+                insta::assert_debug_snapshot!(user_defined, @"RoaringBitmap<[0]>");
+
+                let IndexEmbeddingConfig { name, config: _, user_provided } =
+                    configs.get(1).unwrap();
+                insta::assert_snapshot!(name, @"B_small_hf");
+                insta::assert_debug_snapshot!(user_provided, @"RoaringBitmap<[]>");
+
                 let embeddings = index.embeddings(&rtxn, 0).unwrap();
 
-                // automatically changed to patou
+                // automatically changed to patou because set to regenerate
                 assert_json_snapshot!(embeddings[&simple_hf_name][0] == patou_embed, @"true");
-                // remained beagle because set to userProvided
+                // remained beagle
                 assert_json_snapshot!(embeddings[&fakerest_name][0] == beagle_embed, @"true");
 
                 let doc = index.documents(&rtxn, std::iter::once(0)).unwrap()[0].1;
@@ -5176,4 +5244,578 @@ mod tests {
             }
         }
     }
+
+    #[test]
+    fn import_vectors_first_and_embedder_later() {
+        let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
+
+        let content = serde_json::json!(
+            [
+                {
+                    "id": 0,
+                    "doggo": "kefir",
+                },
+                {
+                    "id": 1,
+                    "doggo": "intel",
+                    "_vectors": {
+                        "my_doggo_embedder": vec![1; 384],
+                        "unknown embedder": vec![1, 2, 3],
+                    }
+                },
+                {
+                    "id": 2,
+                    "doggo": "max",
+                    "_vectors": {
+                        "my_doggo_embedder": {
+                            "regenerate": false,
+                            "embeddings": vec![2; 384],
+                        },
+                        "unknown embedder": vec![4, 5],
+                    },
+                },
+                {
+                    "id": 3,
+                    "doggo": "marcel",
+                    "_vectors": {
+                        "my_doggo_embedder": {
+                            "regenerate": true,
+                            "embeddings": vec![3; 384],
+                        },
+                    },
+                },
+                {
+                    "id": 4,
+                    "doggo": "sora",
+                    "_vectors": {
+                        "my_doggo_embedder": {
+                            "regenerate": true,
+                        },
+                    },
+                },
+            ]
+        );
+
+        let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0_u128).unwrap();
+        let documents_count =
+            read_json(serde_json::to_string_pretty(&content).unwrap().as_bytes(), &mut file)
+                .unwrap();
+        snapshot!(documents_count, @"5");
+        file.persist().unwrap();
+
+        index_scheduler
+            .register(
+                KindWithContent::DocumentAdditionOrUpdate {
+                    index_uid: S("doggos"),
+                    primary_key: None,
+                    method: ReplaceDocuments,
+                    content_file: uuid,
+                    documents_count,
+                    allow_index_creation: true,
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        handle.advance_one_successful_batch();
+
+        let index = index_scheduler.index("doggos").unwrap();
+        let rtxn = index.read_txn().unwrap();
+        let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
+        let field_ids = field_ids_map.ids().collect::<Vec<_>>();
+        let documents = index
+            .all_documents(&rtxn)
+            .unwrap()
+            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .collect::<Vec<_>>();
+        snapshot!(serde_json::to_string(&documents).unwrap(), name: "documents after initial push");
+
+        let setting = meilisearch_types::settings::Settings::<Unchecked> {
+            embedders: Setting::Set(maplit::btreemap! {
+                S("my_doggo_embedder") => Setting::Set(EmbeddingSettings {
+                    source: Setting::Set(milli::vector::settings::EmbedderSource::HuggingFace),
+                    model: Setting::Set(S("sentence-transformers/all-MiniLM-L6-v2")),
+                    revision: Setting::Set(S("e4ce9877abf3edfe10b0d82785e83bdcb973e22e")),
+                    document_template: Setting::Set(S("{{doc.doggo}}")),
+                    ..Default::default()
+                })
+            }),
+            ..Default::default()
+        };
+        index_scheduler
+            .register(
+                KindWithContent::SettingsUpdate {
+                    index_uid: S("doggos"),
+                    new_settings: Box::new(setting),
+                    is_deletion: false,
+                    allow_index_creation: false,
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        index_scheduler.assert_internally_consistent();
+        handle.advance_one_successful_batch();
+        index_scheduler.assert_internally_consistent();
+
+        let index = index_scheduler.index("doggos").unwrap();
+        let rtxn = index.read_txn().unwrap();
+        let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
+        let field_ids = field_ids_map.ids().collect::<Vec<_>>();
+        let documents = index
+            .all_documents(&rtxn)
+            .unwrap()
+            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .collect::<Vec<_>>();
+        // the all the vectors linked to the new specified embedder have been removed
+        // Only the unknown embedders stays in the document DB
+        snapshot!(serde_json::to_string(&documents).unwrap(), @r###"[{"id":0,"doggo":"kefir"},{"id":1,"doggo":"intel","_vectors":{"unknown embedder":[1.0,2.0,3.0]}},{"id":2,"doggo":"max","_vectors":{"unknown embedder":[4.0,5.0]}},{"id":3,"doggo":"marcel"},{"id":4,"doggo":"sora"}]"###);
+        let conf = index.embedding_configs(&rtxn).unwrap();
+        // even though we specified the vector for the ID 3, it shouldn't be marked
+        // as user provided since we explicitely marked it as NOT user provided.
+        snapshot!(format!("{conf:#?}"), @r###"
+        [
+            IndexEmbeddingConfig {
+                name: "my_doggo_embedder",
+                config: EmbeddingConfig {
+                    embedder_options: HuggingFace(
+                        EmbedderOptions {
+                            model: "sentence-transformers/all-MiniLM-L6-v2",
+                            revision: Some(
+                                "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+                            ),
+                            distribution: None,
+                        },
+                    ),
+                    prompt: PromptData {
+                        template: "{{doc.doggo}}",
+                    },
+                },
+                user_provided: RoaringBitmap<[1, 2]>,
+            },
+        ]
+        "###);
+        let docid = index.external_documents_ids.get(&rtxn, "0").unwrap().unwrap();
+        let embeddings = index.embeddings(&rtxn, docid).unwrap();
+        let embedding = &embeddings["my_doggo_embedder"];
+        assert!(!embedding.is_empty(), "{embedding:?}");
+
+        // the document with the id 3 should keep its original embedding
+        let docid = index.external_documents_ids.get(&rtxn, "3").unwrap().unwrap();
+        let mut embeddings = Vec::new();
+
+        'vectors: for i in 0..=u8::MAX {
+            let reader = arroy::Reader::open(&rtxn, i as u16, index.vector_arroy)
+                .map(Some)
+                .or_else(|e| match e {
+                    arroy::Error::MissingMetadata(_) => Ok(None),
+                    e => Err(e),
+                })
+                .transpose();
+
+            let Some(reader) = reader else {
+                break 'vectors;
+            };
+
+            let embedding = reader.unwrap().item_vector(&rtxn, docid).unwrap();
+            if let Some(embedding) = embedding {
+                embeddings.push(embedding)
+            } else {
+                break 'vectors;
+            }
+        }
+
+        snapshot!(embeddings.len(), @"1");
+        assert!(embeddings[0].iter().all(|i| *i == 3.0), "{:?}", embeddings[0]);
+
+        // If we update marcel it should regenerate its embedding automatically
+
+        let content = serde_json::json!(
+            [
+                {
+                    "id": 3,
+                    "doggo": "marvel",
+                },
+                {
+                    "id": 4,
+                    "doggo": "sorry",
+                },
+            ]
+        );
+
+        let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(1_u128).unwrap();
+        let documents_count =
+            read_json(serde_json::to_string_pretty(&content).unwrap().as_bytes(), &mut file)
+                .unwrap();
+        snapshot!(documents_count, @"2");
+        file.persist().unwrap();
+
+        index_scheduler
+            .register(
+                KindWithContent::DocumentAdditionOrUpdate {
+                    index_uid: S("doggos"),
+                    primary_key: None,
+                    method: UpdateDocuments,
+                    content_file: uuid,
+                    documents_count,
+                    allow_index_creation: true,
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        handle.advance_one_successful_batch();
+
+        // the document with the id 3 should have its original embedding updated
+        let rtxn = index.read_txn().unwrap();
+        let docid = index.external_documents_ids.get(&rtxn, "3").unwrap().unwrap();
+        let doc = index.documents(&rtxn, Some(docid)).unwrap()[0];
+        let doc = obkv_to_json(&field_ids, &field_ids_map, doc.1).unwrap();
+        snapshot!(json_string!(doc), @r###"
+        {
+          "id": 3,
+          "doggo": "marvel"
+        }
+        "###);
+
+        let embeddings = index.embeddings(&rtxn, docid).unwrap();
+        let embedding = &embeddings["my_doggo_embedder"];
+
+        assert!(!embedding.is_empty());
+        assert!(!embedding[0].iter().all(|i| *i == 3.0), "{:?}", embedding[0]);
+
+        // the document with the id 4 should generate an embedding
+        let docid = index.external_documents_ids.get(&rtxn, "4").unwrap().unwrap();
+        let embeddings = index.embeddings(&rtxn, docid).unwrap();
+        let embedding = &embeddings["my_doggo_embedder"];
+
+        assert!(!embedding.is_empty());
+    }
+
+    #[test]
+    fn delete_document_containing_vector() {
+        // 1. Add an embedder
+        // 2. Push two documents containing a simple vector
+        // 3. Delete the first document
+        // 4. The user defined roaring bitmap shouldn't contains the id of the first document anymore
+        // 5. Clear the index
+        // 6. The user defined roaring bitmap shouldn't contains the id of the second document
+        let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
+
+        let setting = meilisearch_types::settings::Settings::<Unchecked> {
+            embedders: Setting::Set(maplit::btreemap! {
+                S("manual") => Setting::Set(EmbeddingSettings {
+                    source: Setting::Set(milli::vector::settings::EmbedderSource::UserProvided),
+                    dimensions: Setting::Set(3),
+                    ..Default::default()
+                })
+            }),
+            ..Default::default()
+        };
+        index_scheduler
+            .register(
+                KindWithContent::SettingsUpdate {
+                    index_uid: S("doggos"),
+                    new_settings: Box::new(setting),
+                    is_deletion: false,
+                    allow_index_creation: true,
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        handle.advance_one_successful_batch();
+
+        let content = serde_json::json!(
+            [
+                {
+                    "id": 0,
+                    "doggo": "kefir",
+                    "_vectors": {
+                        "manual": vec![0, 0, 0],
+                    }
+                },
+                {
+                    "id": 1,
+                    "doggo": "intel",
+                    "_vectors": {
+                        "manual": vec![1, 1, 1],
+                    }
+                },
+            ]
+        );
+
+        let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0_u128).unwrap();
+        let documents_count =
+            read_json(serde_json::to_string_pretty(&content).unwrap().as_bytes(), &mut file)
+                .unwrap();
+        snapshot!(documents_count, @"2");
+        file.persist().unwrap();
+
+        index_scheduler
+            .register(
+                KindWithContent::DocumentAdditionOrUpdate {
+                    index_uid: S("doggos"),
+                    primary_key: None,
+                    method: ReplaceDocuments,
+                    content_file: uuid,
+                    documents_count,
+                    allow_index_creation: false,
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        handle.advance_one_successful_batch();
+
+        index_scheduler
+            .register(
+                KindWithContent::DocumentDeletion {
+                    index_uid: S("doggos"),
+                    documents_ids: vec![S("1")],
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        handle.advance_one_successful_batch();
+
+        let index = index_scheduler.index("doggos").unwrap();
+        let rtxn = index.read_txn().unwrap();
+        let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
+        let field_ids = field_ids_map.ids().collect::<Vec<_>>();
+        let documents = index
+            .all_documents(&rtxn)
+            .unwrap()
+            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .collect::<Vec<_>>();
+        snapshot!(serde_json::to_string(&documents).unwrap(), @r###"[{"id":0,"doggo":"kefir"}]"###);
+        let conf = index.embedding_configs(&rtxn).unwrap();
+        snapshot!(format!("{conf:#?}"), @r###"
+        [
+            IndexEmbeddingConfig {
+                name: "manual",
+                config: EmbeddingConfig {
+                    embedder_options: UserProvided(
+                        EmbedderOptions {
+                            dimensions: 3,
+                            distribution: None,
+                        },
+                    ),
+                    prompt: PromptData {
+                        template: "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}",
+                    },
+                },
+                user_provided: RoaringBitmap<[0]>,
+            },
+        ]
+        "###);
+        let docid = index.external_documents_ids.get(&rtxn, "0").unwrap().unwrap();
+        let embeddings = index.embeddings(&rtxn, docid).unwrap();
+        let embedding = &embeddings["manual"];
+        assert!(!embedding.is_empty(), "{embedding:?}");
+
+        index_scheduler
+            .register(KindWithContent::DocumentClear { index_uid: S("doggos") }, None, false)
+            .unwrap();
+        handle.advance_one_successful_batch();
+
+        let index = index_scheduler.index("doggos").unwrap();
+        let rtxn = index.read_txn().unwrap();
+        let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
+        let field_ids = field_ids_map.ids().collect::<Vec<_>>();
+        let documents = index
+            .all_documents(&rtxn)
+            .unwrap()
+            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .collect::<Vec<_>>();
+        snapshot!(serde_json::to_string(&documents).unwrap(), @"[]");
+        let conf = index.embedding_configs(&rtxn).unwrap();
+        snapshot!(format!("{conf:#?}"), @r###"
+        [
+            IndexEmbeddingConfig {
+                name: "manual",
+                config: EmbeddingConfig {
+                    embedder_options: UserProvided(
+                        EmbedderOptions {
+                            dimensions: 3,
+                            distribution: None,
+                        },
+                    ),
+                    prompt: PromptData {
+                        template: "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}",
+                    },
+                },
+                user_provided: RoaringBitmap<[]>,
+            },
+        ]
+        "###);
+    }
+
+    #[test]
+    fn delete_embedder_with_user_provided_vectors() {
+        // 1. Add two embedders
+        // 2. Push two documents containing a simple vector
+        // 3. The documents must not contain the vectors after the update as they are in the vectors db
+        // 3. Delete the embedders
+        // 4. The documents contain the vectors again
+        let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
+
+        let setting = meilisearch_types::settings::Settings::<Unchecked> {
+            embedders: Setting::Set(maplit::btreemap! {
+                S("manual") => Setting::Set(EmbeddingSettings {
+                    source: Setting::Set(milli::vector::settings::EmbedderSource::UserProvided),
+                    dimensions: Setting::Set(3),
+                    ..Default::default()
+                }),
+                S("my_doggo_embedder") => Setting::Set(EmbeddingSettings {
+                    source: Setting::Set(milli::vector::settings::EmbedderSource::HuggingFace),
+                    model: Setting::Set(S("sentence-transformers/all-MiniLM-L6-v2")),
+                    revision: Setting::Set(S("e4ce9877abf3edfe10b0d82785e83bdcb973e22e")),
+                    document_template: Setting::Set(S("{{doc.doggo}}")),
+                    ..Default::default()
+                }),
+            }),
+            ..Default::default()
+        };
+        index_scheduler
+            .register(
+                KindWithContent::SettingsUpdate {
+                    index_uid: S("doggos"),
+                    new_settings: Box::new(setting),
+                    is_deletion: false,
+                    allow_index_creation: true,
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        handle.advance_one_successful_batch();
+
+        let content = serde_json::json!(
+            [
+                {
+                    "id": 0,
+                    "doggo": "kefir",
+                    "_vectors": {
+                        "manual": vec![0, 0, 0],
+                        "my_doggo_embedder": vec![1; 384],
+                    }
+                },
+                {
+                    "id": 1,
+                    "doggo": "intel",
+                    "_vectors": {
+                        "manual": vec![1, 1, 1],
+                    }
+                },
+            ]
+        );
+
+        let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0_u128).unwrap();
+        let documents_count =
+            read_json(serde_json::to_string_pretty(&content).unwrap().as_bytes(), &mut file)
+                .unwrap();
+        snapshot!(documents_count, @"2");
+        file.persist().unwrap();
+
+        index_scheduler
+            .register(
+                KindWithContent::DocumentAdditionOrUpdate {
+                    index_uid: S("doggos"),
+                    primary_key: None,
+                    method: ReplaceDocuments,
+                    content_file: uuid,
+                    documents_count,
+                    allow_index_creation: false,
+                },
+                None,
+                false,
+            )
+            .unwrap();
+        handle.advance_one_successful_batch();
+
+        {
+            let index = index_scheduler.index("doggos").unwrap();
+            let rtxn = index.read_txn().unwrap();
+            let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
+            let field_ids = field_ids_map.ids().collect::<Vec<_>>();
+            let documents = index
+                .all_documents(&rtxn)
+                .unwrap()
+                .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+                .collect::<Vec<_>>();
+            snapshot!(serde_json::to_string(&documents).unwrap(), @r###"[{"id":0,"doggo":"kefir"},{"id":1,"doggo":"intel"}]"###);
+        }
+
+        {
+            let setting = meilisearch_types::settings::Settings::<Unchecked> {
+                embedders: Setting::Set(maplit::btreemap! {
+                    S("manual") => Setting::Reset,
+                }),
+                ..Default::default()
+            };
+            index_scheduler
+                .register(
+                    KindWithContent::SettingsUpdate {
+                        index_uid: S("doggos"),
+                        new_settings: Box::new(setting),
+                        is_deletion: false,
+                        allow_index_creation: true,
+                    },
+                    None,
+                    false,
+                )
+                .unwrap();
+            handle.advance_one_successful_batch();
+        }
+
+        {
+            let index = index_scheduler.index("doggos").unwrap();
+            let rtxn = index.read_txn().unwrap();
+            let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
+            let field_ids = field_ids_map.ids().collect::<Vec<_>>();
+            let documents = index
+                .all_documents(&rtxn)
+                .unwrap()
+                .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+                .collect::<Vec<_>>();
+            snapshot!(serde_json::to_string(&documents).unwrap(), @r###"[{"id":0,"doggo":"kefir","_vectors":{"manual":{"embeddings":[[0.0,0.0,0.0]],"regenerate":false}}},{"id":1,"doggo":"intel","_vectors":{"manual":{"embeddings":[[1.0,1.0,1.0]],"regenerate":false}}}]"###);
+        }
+
+        {
+            let setting = meilisearch_types::settings::Settings::<Unchecked> {
+                embedders: Setting::Reset,
+                ..Default::default()
+            };
+            index_scheduler
+                .register(
+                    KindWithContent::SettingsUpdate {
+                        index_uid: S("doggos"),
+                        new_settings: Box::new(setting),
+                        is_deletion: false,
+                        allow_index_creation: true,
+                    },
+                    None,
+                    false,
+                )
+                .unwrap();
+            handle.advance_one_successful_batch();
+        }
+
+        {
+            let index = index_scheduler.index("doggos").unwrap();
+            let rtxn = index.read_txn().unwrap();
+            let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
+            let field_ids = field_ids_map.ids().collect::<Vec<_>>();
+            let documents = index
+                .all_documents(&rtxn)
+                .unwrap()
+                .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+                .collect::<Vec<_>>();
+
+            // FIXME: redaction
+            snapshot!(json_string!(serde_json::to_string(&documents).unwrap(), { "[]._vectors.doggo_embedder.embeddings" => "[vector]" }),  @r###""[{\"id\":0,\"doggo\":\"kefir\",\"_vectors\":{\"manual\":{\"embeddings\":[[0.0,0.0,0.0]],\"regenerate\":false},\"my_doggo_embedder\":{\"embeddings\":[[1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0]],\"regenerate\":false}}},{\"id\":1,\"doggo\":\"intel\",\"_vectors\":{\"manual\":{\"embeddings\":[[1.0,1.0,1.0]],\"regenerate\":false}}}]""###);
+        }
+    }
 }

index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-15.snap

@@ -6,10 +6,6 @@ expression: doc
   "doggo": "Intel",
   "breed": "beagle",
   "_vectors": {
-    "A_fakerest": {
-      "embeddings": "[vector]",
-      "userProvided": true
-    },
     "noise": [
       0.1,
       0.2,

index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-22.snap

@@ -6,10 +6,6 @@ expression: doc
   "doggo": "kefir",
   "breed": "patou",
   "_vectors": {
-    "A_fakerest": {
-      "embeddings": "[vector]",
-      "userProvided": true
-    },
     "noise": [
       0.1,
       0.2,

index-scheduler/src/utils.rs

@@ -238,6 +238,7 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) {
     let mut index_uids = vec![];
     match &mut task.kind {
         K::DocumentAdditionOrUpdate { index_uid, .. } => index_uids.push(index_uid),
+        K::DocumentEdition { index_uid, .. } => index_uids.push(index_uid),
         K::DocumentDeletion { index_uid, .. } => index_uids.push(index_uid),
         K::DocumentDeletionByFilter { index_uid, .. } => index_uids.push(index_uid),
         K::DocumentClear { index_uid } => index_uids.push(index_uid),
@@ -408,7 +409,26 @@ impl IndexScheduler {
                                 match status {
                                     Status::Succeeded => assert!(indexed_documents <= received_documents),
                                     Status::Failed | Status::Canceled => assert_eq!(indexed_documents, 0),
-                                    status => panic!("DocumentAddition can't have an indexed_document set if it's {}", status),
+                                    status => panic!("DocumentAddition can't have an indexed_documents set if it's {}", status),
+                                }
+                            }
+                            None => {
+                                assert!(matches!(status, Status::Enqueued | Status::Processing))
+                            }
+                        }
+                    }
+                    Details::DocumentEdition { edited_documents, .. } => {
+                        assert_eq!(kind.as_kind(), Kind::DocumentEdition);
+                        match edited_documents {
+                            Some(edited_documents) => {
+                                assert!(matches!(
+                                    status,
+                                    Status::Succeeded | Status::Failed | Status::Canceled
+                                ));
+                                match status {
+                                    Status::Succeeded => (),
+                                    Status::Failed | Status::Canceled => assert_eq!(edited_documents, 0),
+                                    status => panic!("DocumentEdition can't have an edited_documents set if it's {}", status),
                                 }
                             }
                             None => {

meili-snap/Cargo.toml

@@ -11,6 +11,6 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-insta = { version = "^1.34.0", features = ["json", "redactions"] }
+insta = { version = "^1.39.0", features = ["json", "redactions"] }
 md5 = "0.7.0"
 once_cell = "1.19"

meilisearch-auth/Cargo.toml

@@ -11,16 +11,16 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-base64 = "0.21.7"
-enum-iterator = "1.5.0"
+base64 = "0.22.1"
+enum-iterator = "2.1.0"
 hmac = "0.12.1"
 maplit = "1.0.2"
 meilisearch-types = { path = "../meilisearch-types" }
 rand = "0.8.5"
-roaring = { version = "0.10.2", features = ["serde"] }
-serde = { version = "1.0.195", features = ["derive"] }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
+roaring = { version = "0.10.6", features = ["serde"] }
+serde = { version = "1.0.204", features = ["derive"] }
+serde_json = { version = "1.0.120", features = ["preserve_order"] }
 sha2 = "0.10.8"
-thiserror = "1.0.56"
-time = { version = "0.3.31", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
+thiserror = "1.0.61"
+time = { version = "0.3.36", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+uuid = { version = "1.10.0", features = ["serde", "v4"] }

meilisearch-auth/src/lib.rs

@@ -188,6 +188,12 @@ impl AuthFilter {
         self.allow_index_creation && self.is_index_authorized(index)
     }
 
+    #[inline]
+    /// Return true if a tenant token was used to generate the search rules.
+    pub fn is_tenant_token(&self) -> bool {
+        self.search_rules.is_some()
+    }
+
     pub fn with_allowed_indexes(allowed_indexes: HashSet<IndexUidPattern>) -> Self {
         Self {
             search_rules: None,
@@ -205,6 +211,7 @@ impl AuthFilter {
                 .unwrap_or(true)
     }
 
+    /// Check if the index is authorized by the API key and the tenant token.
     pub fn is_index_authorized(&self, index: &str) -> bool {
         self.key_authorized_indexes.is_index_authorized(index)
             && self
@@ -214,6 +221,44 @@ impl AuthFilter {
                 .unwrap_or(true)
     }
 
+    /// Only check if the index is authorized by the API key
+    pub fn api_key_is_index_authorized(&self, index: &str) -> bool {
+        self.key_authorized_indexes.is_index_authorized(index)
+    }
+
+    /// Only check if the index is authorized by the tenant token
+    pub fn tenant_token_is_index_authorized(&self, index: &str) -> bool {
+        self.search_rules
+            .as_ref()
+            .map(|search_rules| search_rules.is_index_authorized(index))
+            .unwrap_or(true)
+    }
+
+    /// Return the list of authorized indexes by the tenant token if any
+    pub fn tenant_token_list_index_authorized(&self) -> Vec<String> {
+        match self.search_rules {
+            Some(ref search_rules) => {
+                let mut indexes: Vec<_> = match search_rules {
+                    SearchRules::Set(set) => set.iter().map(|s| s.to_string()).collect(),
+                    SearchRules::Map(map) => map.keys().map(|s| s.to_string()).collect(),
+                };
+                indexes.sort_unstable();
+                indexes
+            }
+            None => Vec::new(),
+        }
+    }
+
+    /// Return the list of authorized indexes by the api key if any
+    pub fn api_key_list_index_authorized(&self) -> Vec<String> {
+        let mut indexes: Vec<_> = match self.key_authorized_indexes {
+            SearchRules::Set(ref set) => set.iter().map(|s| s.to_string()).collect(),
+            SearchRules::Map(ref map) => map.keys().map(|s| s.to_string()).collect(),
+        };
+        indexes.sort_unstable();
+        indexes
+    }
+
     pub fn get_index_search_rules(&self, index: &str) -> Option<IndexSearchRules> {
         if !self.is_index_authorized(index) {
             return None;

meilisearch-types/Cargo.toml

@@ -11,36 +11,36 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-actix-web = { version = "4.6.0", default-features = false }
-anyhow = "1.0.79"
+actix-web = { version = "4.8.0", default-features = false }
+anyhow = "1.0.86"
 convert_case = "0.6.0"
 csv = "1.3.0"
-deserr = { version = "0.6.1", features = ["actix-web"] }
-either = { version = "1.9.0", features = ["serde"] }
-enum-iterator = "1.5.0"
+deserr = { version = "0.6.2", features = ["actix-web"] }
+either = { version = "1.13.0", features = ["serde"] }
+enum-iterator = "2.1.0"
 file-store = { path = "../file-store" }
-flate2 = "1.0.28"
+flate2 = "1.0.30"
 fst = "0.4.7"
-memmap2 = "0.7.1"
+memmap2 = "0.9.4"
 milli = { path = "../milli" }
-roaring = { version = "0.10.2", features = ["serde"] }
-serde = { version = "1.0.195", features = ["derive"] }
+roaring = { version = "0.10.6", features = ["serde"] }
+serde = { version = "1.0.204", features = ["derive"] }
 serde-cs = "0.2.4"
-serde_json = "1.0.111"
-tar = "0.4.40"
-tempfile = "3.9.0"
-thiserror = "1.0.56"
-time = { version = "0.3.31", features = [
+serde_json = "1.0.120"
+tar = "0.4.41"
+tempfile = "3.10.1"
+thiserror = "1.0.61"
+time = { version = "0.3.36", features = [
     "serde-well-known",
     "formatting",
     "parsing",
     "macros",
 ] }
-tokio = "1.35"
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
+tokio = "1.38"
+uuid = { version = "1.10.0", features = ["serde", "v4"] }
 
 [dev-dependencies]
-insta = "1.34.0"
+insta = "1.39.0"
 meili-snap = { path = "../meili-snap" }
 
 [features]
@@ -54,6 +54,8 @@ chinese-pinyin = ["milli/chinese-pinyin"]
 hebrew = ["milli/hebrew"]
 # japanese specialized tokenization
 japanese = ["milli/japanese"]
+# korean specialized tokenization
+korean = ["milli/korean"]
 # thai specialized tokenization
 thai = ["milli/thai"]
 # allow greek specialized tokenization

meilisearch-types/src/deserr/mod.rs

@@ -155,6 +155,10 @@ make_missing_field_convenience_builder!(
     MissingFacetSearchFacetName,
     missing_facet_search_facet_name
 );
+make_missing_field_convenience_builder!(
+    MissingDocumentEditionFunction,
+    missing_document_edition_function
+);
 
 // Integrate a sub-error into a [`DeserrError`] by taking its error message but using
 // the default error code (C) from `Self`
@@ -188,6 +192,7 @@ merge_with_error_impl_take_error_message!(ParseOffsetDateTimeError);
 merge_with_error_impl_take_error_message!(ParseTaskKindError);
 merge_with_error_impl_take_error_message!(ParseTaskStatusError);
 merge_with_error_impl_take_error_message!(IndexUidFormatError);
+merge_with_error_impl_take_error_message!(InvalidMultiSearchWeight);
 merge_with_error_impl_take_error_message!(InvalidSearchSemanticRatio);
 merge_with_error_impl_take_error_message!(InvalidSearchRankingScoreThreshold);
 merge_with_error_impl_take_error_message!(InvalidSimilarRankingScoreThreshold);

meilisearch-types/src/error.rs

@@ -222,7 +222,9 @@ InvalidApiKeyUid                      , InvalidRequest       , BAD_REQUEST ;
 InvalidContentType                    , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
 InvalidDocumentCsvDelimiter           , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentFields                 , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentRetrieveVectors        , InvalidRequest       , BAD_REQUEST ;
 MissingDocumentFilter                 , InvalidRequest       , BAD_REQUEST ;
+MissingDocumentEditionFunction        , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentFilter                 , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentGeoField               , InvalidRequest       , BAD_REQUEST ;
 InvalidVectorDimensions               , InvalidRequest       , BAD_REQUEST ;
@@ -236,13 +238,20 @@ InvalidIndexLimit                     , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexOffset                    , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexPrimaryKey                , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexUid                       , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchFederated           , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchFederationOptions   , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchQueryPagination     , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchQueryRankingRules   , InvalidRequest       , BAD_REQUEST ;
+InvalidMultiSearchWeight              , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToSearchOn     , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToCrop         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToHighlight    , InvalidRequest       , BAD_REQUEST ;
 InvalidSimilarAttributesToRetrieve    , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarRetrieveVectors         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToRetrieve     , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchRankingScoreThreshold    , InvalidRequest       , BAD_REQUEST ;
 InvalidSimilarRankingScoreThreshold   , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchRetrieveVectors          , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchCropLength               , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchCropMarker               , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchFacets                   , InvalidRequest       , BAD_REQUEST ;
@@ -270,13 +279,14 @@ InvalidSimilarShowRankingScore        , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchShowRankingScoreDetails  , InvalidRequest       , BAD_REQUEST ;
 InvalidSimilarShowRankingScoreDetails , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchSort                     , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchDistinct                 , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsDisplayedAttributes    , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsDistinctAttribute      , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsProximityPrecision     , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsFaceting               , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsFilterableAttributes   , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsPagination             , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsSearchCutoffMs           , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsSearchCutoffMs         , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsEmbedders              , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsRankingRules           , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSearchableAttributes   , InvalidRequest       , BAD_REQUEST ;
@@ -332,7 +342,10 @@ UnsupportedMediaType                  , InvalidRequest       , UNSUPPORTED_MEDIA
 
 // Experimental features
 VectorEmbeddingError                  , InvalidRequest       , BAD_REQUEST ;
-NotFoundSimilarId                     , InvalidRequest       , BAD_REQUEST
+NotFoundSimilarId                     , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentEditionContext         , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentEditionFunctionFilter  , InvalidRequest       , BAD_REQUEST ;
+EditDocumentsByFunctionError          , InvalidRequest       , BAD_REQUEST
 }
 
 impl ErrorCode for JoinError {
@@ -381,6 +394,7 @@ impl ErrorCode for milli::Error {
                         Code::IndexPrimaryKeyMultipleCandidatesFound
                     }
                     UserError::PrimaryKeyCannotBeChanged(_) => Code::IndexPrimaryKeyAlreadyExists,
+                    UserError::InvalidDistinctAttribute { .. } => Code::InvalidSearchDistinct,
                     UserError::SortRankingRuleMissing => Code::InvalidSearchSort,
                     UserError::InvalidFacetsDistribution { .. } => Code::InvalidSearchFacets,
                     UserError::InvalidSortableAttribute { .. } => Code::InvalidSearchSort,
@@ -393,7 +407,8 @@ impl ErrorCode for milli::Error {
                     UserError::CriterionError(_) => Code::InvalidSettingsRankingRules,
                     UserError::InvalidGeoField { .. } => Code::InvalidDocumentGeoField,
                     UserError::InvalidVectorDimensions { .. } => Code::InvalidVectorDimensions,
-                    UserError::InvalidVectorsMapType { .. } => Code::InvalidVectorsType,
+                    UserError::InvalidVectorsMapType { .. }
+                    | UserError::InvalidVectorsEmbedderConf { .. } => Code::InvalidVectorsType,
                     UserError::TooManyVectors(_, _) => Code::TooManyVectors,
                     UserError::SortError(_) => Code::InvalidSearchSort,
                     UserError::InvalidMinTypoWordLenSetting(_, _) => {
@@ -401,6 +416,12 @@ impl ErrorCode for milli::Error {
                     }
                     UserError::InvalidEmbedder(_) => Code::InvalidEmbedder,
                     UserError::VectorEmbeddingError(_) => Code::VectorEmbeddingError,
+                    UserError::DocumentEditionCannotModifyPrimaryKey
+                    | UserError::DocumentEditionDocumentMustBeObject
+                    | UserError::DocumentEditionRuntimeError(_)
+                    | UserError::DocumentEditionCompilationError(_) => {
+                        Code::EditDocumentsByFunctionError
+                    }
                 }
             }
         }
@@ -496,6 +517,12 @@ impl fmt::Display for deserr_codes::InvalidSearchSemanticRatio {
     }
 }
 
+impl fmt::Display for deserr_codes::InvalidMultiSearchWeight {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(f, "the value of `weight` is invalid, expected a positive float (>= 0.0).")
+    }
+}
+
 impl fmt::Display for deserr_codes::InvalidSimilarId {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(

meilisearch-types/src/features.rs

@@ -6,6 +6,7 @@ pub struct RuntimeTogglableFeatures {
     pub vector_store: bool,
     pub metrics: bool,
     pub logs_route: bool,
+    pub edit_documents_by_function: bool,
 }
 
 #[derive(Default, Debug, Clone, Copy)]

meilisearch-types/src/settings.rs

@@ -8,6 +8,7 @@ use std::str::FromStr;
 
 use deserr::{DeserializeError, Deserr, ErrorKind, MergeWithError, ValuePointerRef};
 use fst::IntoStreamer;
+use milli::index::IndexEmbeddingConfig;
 use milli::proximity::ProximityPrecision;
 use milli::update::Setting;
 use milli::{Criterion, CriterionError, Index, DEFAULT_VALUES_PER_FACET};
@@ -672,7 +673,7 @@ pub fn settings(
     let embedders: BTreeMap<_, _> = index
         .embedding_configs(rtxn)?
         .into_iter()
-        .map(|(name, config)| (name, Setting::Set(config.into())))
+        .map(|IndexEmbeddingConfig { name, config, .. }| (name, Setting::Set(config.into())))
         .collect();
     let embedders = if embedders.is_empty() { Setting::NotSet } else { Setting::Set(embedders) };
 

meilisearch-types/src/task_view.rs

@@ -1,3 +1,4 @@
+use milli::Object;
 use serde::Serialize;
 use time::{Duration, OffsetDateTime};
 
@@ -54,6 +55,8 @@ pub struct DetailsView {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub indexed_documents: Option<Option<u64>>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    pub edited_documents: Option<Option<u64>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub primary_key: Option<Option<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub provided_ids: Option<usize>,
@@ -70,6 +73,10 @@ pub struct DetailsView {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub dump_uid: Option<Option<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
+    pub context: Option<Option<Object>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub function: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(flatten)]
     pub settings: Option<Box<Settings<Unchecked>>>,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -86,6 +93,20 @@ impl From<Details> for DetailsView {
                     ..DetailsView::default()
                 }
             }
+            Details::DocumentEdition {
+                deleted_documents,
+                edited_documents,
+                original_filter,
+                context,
+                function,
+            } => DetailsView {
+                deleted_documents: Some(deleted_documents),
+                edited_documents: Some(edited_documents),
+                original_filter: Some(original_filter),
+                context: Some(context),
+                function: Some(function),
+                ..DetailsView::default()
+            },
             Details::SettingsUpdate { mut settings } => {
                 settings.hide_secrets();
                 DetailsView { settings: Some(settings), ..DetailsView::default() }

meilisearch-types/src/tasks.rs

@@ -5,6 +5,7 @@ use std::str::FromStr;
 
 use enum_iterator::Sequence;
 use milli::update::IndexDocumentsMethod;
+use milli::Object;
 use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize, Serializer};
 use time::{Duration, OffsetDateTime};
@@ -48,6 +49,7 @@ impl Task {
             | TaskDeletion { .. }
             | IndexSwap { .. } => None,
             DocumentAdditionOrUpdate { index_uid, .. }
+            | DocumentEdition { index_uid, .. }
             | DocumentDeletion { index_uid, .. }
             | DocumentDeletionByFilter { index_uid, .. }
             | DocumentClear { index_uid }
@@ -67,7 +69,8 @@ impl Task {
     pub fn content_uuid(&self) -> Option<Uuid> {
         match self.kind {
             KindWithContent::DocumentAdditionOrUpdate { content_file, .. } => Some(content_file),
-            KindWithContent::DocumentDeletion { .. }
+            KindWithContent::DocumentEdition { .. }
+            | KindWithContent::DocumentDeletion { .. }
             | KindWithContent::DocumentDeletionByFilter { .. }
             | KindWithContent::DocumentClear { .. }
             | KindWithContent::SettingsUpdate { .. }
@@ -102,6 +105,12 @@ pub enum KindWithContent {
         index_uid: String,
         filter_expr: serde_json::Value,
     },
+    DocumentEdition {
+        index_uid: String,
+        filter_expr: Option<serde_json::Value>,
+        context: Option<milli::Object>,
+        function: String,
+    },
     DocumentClear {
         index_uid: String,
     },
@@ -150,6 +159,7 @@ impl KindWithContent {
     pub fn as_kind(&self) -> Kind {
         match self {
             KindWithContent::DocumentAdditionOrUpdate { .. } => Kind::DocumentAdditionOrUpdate,
+            KindWithContent::DocumentEdition { .. } => Kind::DocumentEdition,
             KindWithContent::DocumentDeletion { .. } => Kind::DocumentDeletion,
             KindWithContent::DocumentDeletionByFilter { .. } => Kind::DocumentDeletion,
             KindWithContent::DocumentClear { .. } => Kind::DocumentDeletion,
@@ -174,6 +184,7 @@ impl KindWithContent {
             | TaskCancelation { .. }
             | TaskDeletion { .. } => vec![],
             DocumentAdditionOrUpdate { index_uid, .. }
+            | DocumentEdition { index_uid, .. }
             | DocumentDeletion { index_uid, .. }
             | DocumentDeletionByFilter { index_uid, .. }
             | DocumentClear { index_uid }
@@ -202,6 +213,15 @@ impl KindWithContent {
                     indexed_documents: None,
                 })
             }
+            KindWithContent::DocumentEdition { index_uid: _, filter_expr, context, function } => {
+                Some(Details::DocumentEdition {
+                    deleted_documents: None,
+                    edited_documents: None,
+                    original_filter: filter_expr.as_ref().map(|v| v.to_string()),
+                    context: context.clone(),
+                    function: function.clone(),
+                })
+            }
             KindWithContent::DocumentDeletion { index_uid: _, documents_ids } => {
                 Some(Details::DocumentDeletion {
                     provided_ids: documents_ids.len(),
@@ -250,6 +270,15 @@ impl KindWithContent {
                     indexed_documents: Some(0),
                 })
             }
+            KindWithContent::DocumentEdition { index_uid: _, filter_expr, context, function } => {
+                Some(Details::DocumentEdition {
+                    deleted_documents: Some(0),
+                    edited_documents: Some(0),
+                    original_filter: filter_expr.as_ref().map(|v| v.to_string()),
+                    context: context.clone(),
+                    function: function.clone(),
+                })
+            }
             KindWithContent::DocumentDeletion { index_uid: _, documents_ids } => {
                 Some(Details::DocumentDeletion {
                     provided_ids: documents_ids.len(),
@@ -301,6 +330,7 @@ impl From<&KindWithContent> for Option<Details> {
                     indexed_documents: None,
                 })
             }
+            KindWithContent::DocumentEdition { .. } => None,
             KindWithContent::DocumentDeletion { .. } => None,
             KindWithContent::DocumentDeletionByFilter { .. } => None,
             KindWithContent::DocumentClear { .. } => None,
@@ -394,6 +424,7 @@ impl std::error::Error for ParseTaskStatusError {}
 #[serde(rename_all = "camelCase")]
 pub enum Kind {
     DocumentAdditionOrUpdate,
+    DocumentEdition,
     DocumentDeletion,
     SettingsUpdate,
     IndexCreation,
@@ -410,6 +441,7 @@ impl Kind {
     pub fn related_to_one_index(&self) -> bool {
         match self {
             Kind::DocumentAdditionOrUpdate
+            | Kind::DocumentEdition
             | Kind::DocumentDeletion
             | Kind::SettingsUpdate
             | Kind::IndexCreation
@@ -427,6 +459,7 @@ impl Display for Kind {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Kind::DocumentAdditionOrUpdate => write!(f, "documentAdditionOrUpdate"),
+            Kind::DocumentEdition => write!(f, "documentEdition"),
             Kind::DocumentDeletion => write!(f, "documentDeletion"),
             Kind::SettingsUpdate => write!(f, "settingsUpdate"),
             Kind::IndexCreation => write!(f, "indexCreation"),
@@ -454,6 +487,8 @@ impl FromStr for Kind {
             Ok(Kind::IndexDeletion)
         } else if kind.eq_ignore_ascii_case("documentAdditionOrUpdate") {
             Ok(Kind::DocumentAdditionOrUpdate)
+        } else if kind.eq_ignore_ascii_case("documentEdition") {
+            Ok(Kind::DocumentEdition)
         } else if kind.eq_ignore_ascii_case("documentDeletion") {
             Ok(Kind::DocumentDeletion)
         } else if kind.eq_ignore_ascii_case("settingsUpdate") {
@@ -495,16 +530,50 @@ impl std::error::Error for ParseTaskKindError {}
 
 #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
 pub enum Details {
-    DocumentAdditionOrUpdate { received_documents: u64, indexed_documents: Option<u64> },
-    SettingsUpdate { settings: Box<Settings<Unchecked>> },
-    IndexInfo { primary_key: Option<String> },
-    DocumentDeletion { provided_ids: usize, deleted_documents: Option<u64> },
-    DocumentDeletionByFilter { original_filter: String, deleted_documents: Option<u64> },
-    ClearAll { deleted_documents: Option<u64> },
-    TaskCancelation { matched_tasks: u64, canceled_tasks: Option<u64>, original_filter: String },
-    TaskDeletion { matched_tasks: u64, deleted_tasks: Option<u64>, original_filter: String },
-    Dump { dump_uid: Option<String> },
-    IndexSwap { swaps: Vec<IndexSwap> },
+    DocumentAdditionOrUpdate {
+        received_documents: u64,
+        indexed_documents: Option<u64>,
+    },
+    SettingsUpdate {
+        settings: Box<Settings<Unchecked>>,
+    },
+    IndexInfo {
+        primary_key: Option<String>,
+    },
+    DocumentDeletion {
+        provided_ids: usize,
+        deleted_documents: Option<u64>,
+    },
+    DocumentDeletionByFilter {
+        original_filter: String,
+        deleted_documents: Option<u64>,
+    },
+    DocumentEdition {
+        deleted_documents: Option<u64>,
+        edited_documents: Option<u64>,
+        original_filter: Option<String>,
+        context: Option<Object>,
+        function: String,
+    },
+    ClearAll {
+        deleted_documents: Option<u64>,
+    },
+    TaskCancelation {
+        matched_tasks: u64,
+        canceled_tasks: Option<u64>,
+        original_filter: String,
+    },
+    TaskDeletion {
+        matched_tasks: u64,
+        deleted_tasks: Option<u64>,
+        original_filter: String,
+    },
+    Dump {
+        dump_uid: Option<String>,
+    },
+    IndexSwap {
+        swaps: Vec<IndexSwap>,
+    },
 }
 
 impl Details {
@@ -514,6 +583,7 @@ impl Details {
             Self::DocumentAdditionOrUpdate { indexed_documents, .. } => {
                 *indexed_documents = Some(0)
             }
+            Self::DocumentEdition { edited_documents, .. } => *edited_documents = Some(0),
             Self::DocumentDeletion { deleted_documents, .. } => *deleted_documents = Some(0),
             Self::DocumentDeletionByFilter { deleted_documents, .. } => {
                 *deleted_documents = Some(0)

meilisearch/Cargo.toml

@@ -14,130 +14,124 @@ default-run = "meilisearch"
 
 [dependencies]
 actix-cors = "0.7.0"
-actix-http = { version = "3.7.0", default-features = false, features = [
+actix-http = { version = "3.8.0", default-features = false, features = [
     "compress-brotli",
     "compress-gzip",
     "rustls-0_21",
 ] }
 actix-utils = "3.0.1"
-actix-web = { version = "4.6.0", default-features = false, features = [
+actix-web = { version = "4.8.0", default-features = false, features = [
     "macros",
     "compress-brotli",
     "compress-gzip",
     "cookies",
     "rustls-0_21",
 ] }
-actix-web-static-files = { version = "4.0.1", optional = true }
-anyhow = { version = "1.0.79", features = ["backtrace"] }
-async-stream = "0.3.5"
-async-trait = "0.1.77"
-bstr = "1.9.0"
-byte-unit = { version = "4.0.19", default-features = false, features = [
+anyhow = { version = "1.0.86", features = ["backtrace"] }
+async-trait = "0.1.81"
+bstr = "1.9.1"
+byte-unit = { version = "5.1.4", default-features = false, features = [
     "std",
+    "byte",
     "serde",
 ] }
-bytes = "1.5.0"
-clap = { version = "4.4.17", features = ["derive", "env"] }
-crossbeam-channel = "0.5.11"
-deserr = { version = "0.6.1", features = ["actix-web"] }
+bytes = "1.6.0"
+clap = { version = "4.5.9", features = ["derive", "env"] }
+crossbeam-channel = "0.5.13"
+deserr = { version = "0.6.2", features = ["actix-web"] }
 dump = { path = "../dump" }
-either = "1.9.0"
+either = "1.13.0"
 file-store = { path = "../file-store" }
-flate2 = "1.0.28"
+flate2 = "1.0.30"
 fst = "0.4.7"
 futures = "0.3.30"
 futures-util = "0.3.30"
-http = "0.2.11"
 index-scheduler = { path = "../index-scheduler" }
-indexmap = { version = "2.1.0", features = ["serde"] }
-is-terminal = "0.4.10"
-itertools = "0.11.0"
-jsonwebtoken = "9.2.0"
-lazy_static = "1.4.0"
+indexmap = { version = "2.2.6", features = ["serde"] }
+is-terminal = "0.4.12"
+itertools = "0.13.0"
+jsonwebtoken = "9.3.0"
+lazy_static = "1.5.0"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-mimalloc = { version = "0.1.39", default-features = false }
+mimalloc = { version = "0.1.43", default-features = false }
 mime = "0.3.17"
 num_cpus = "1.16.0"
-obkv = "0.2.1"
+obkv = "0.2.2"
 once_cell = "1.19.0"
-ordered-float = "4.2.0"
-parking_lot = "0.12.1"
+ordered-float = "4.2.1"
+parking_lot = "0.12.3"
 permissive-json-pointer = { path = "../permissive-json-pointer" }
-pin-project-lite = "0.2.13"
+pin-project-lite = "0.2.14"
 platform-dirs = "0.3.0"
-prometheus = { version = "0.13.3", features = ["process"] }
+prometheus = { version = "0.13.4", features = ["process"] }
 rand = "0.8.5"
-rayon = "1.8.0"
-regex = "1.10.2"
-reqwest = { version = "0.11.23", features = [
+rayon = "1.10.0"
+regex = "1.10.5"
+reqwest = { version = "0.12.5", features = [
     "rustls-tls",
     "json",
 ], default-features = false }
 rustls = "0.21.12"
-rustls-pemfile = "1.0.2"
-segment = { version = "0.2.3", optional = true }
-serde = { version = "1.0.195", features = ["derive"] }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
+rustls-pemfile = "1.0.4"
+segment = { version = "0.2.4", optional = true }
+serde = { version = "1.0.204", features = ["derive"] }
+serde_json = { version = "1.0.120", features = ["preserve_order"] }
 sha2 = "0.10.8"
-siphasher = "1.0.0"
+siphasher = "1.0.1"
 slice-group-by = "0.3.1"
-static-files = { version = "0.2.3", optional = true }
-sysinfo = "0.30.5"
-tar = "0.4.40"
-tempfile = "3.9.0"
-thiserror = "1.0.56"
-time = { version = "0.3.31", features = [
+static-files = { version = "0.2.4", optional = true }
+sysinfo = "0.30.13"
+tar = "0.4.41"
+tempfile = "3.10.1"
+thiserror = "1.0.61"
+time = { version = "0.3.36", features = [
     "serde-well-known",
     "formatting",
     "parsing",
     "macros",
 ] }
-tokio = { version = "1.35.1", features = ["full"] }
-tokio-stream = "0.1.14"
-toml = "0.8.8"
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
-walkdir = "2.4.0"
-yaup = "0.2.1"
+tokio = { version = "1.38.0", features = ["full"] }
+toml = "0.8.14"
+uuid = { version = "1.10.0", features = ["serde", "v4"] }
 serde_urlencoded = "0.7.1"
 termcolor = "1.4.1"
-url = { version = "2.5.0", features = ["serde"] }
+url = { version = "2.5.2", features = ["serde"] }
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["json"] }
 tracing-trace = { version = "0.1.0", path = "../tracing-trace" }
-tracing-actix-web = "0.7.10"
+tracing-actix-web = "0.7.11"
 build-info = { version = "1.7.0", path = "../build-info" }
+roaring = "0.10.2"
 
 [dev-dependencies]
-actix-rt = "2.9.0"
-assert-json-diff = "2.0.2"
+actix-rt = "2.10.0"
 brotli = "6.0.0"
-insta = "1.34.0"
+insta = "1.39.0"
 manifest-dir-macros = "0.1.18"
 maplit = "1.0.2"
 meili-snap = { path = "../meili-snap" }
 temp-env = "0.3.6"
 urlencoding = "2.1.3"
-yaup = "0.2.1"
+yaup = "0.3.1"
 
 [build-dependencies]
-anyhow = { version = "1.0.79", optional = true }
-cargo_toml = { version = "0.18.0", optional = true }
+anyhow = { version = "1.0.86", optional = true }
+cargo_toml = { version = "0.20.3", optional = true }
 hex = { version = "0.4.3", optional = true }
-reqwest = { version = "0.11.23", features = [
+reqwest = { version = "0.12.5", features = [
     "blocking",
     "rustls-tls",
 ], default-features = false, optional = true }
 sha-1 = { version = "0.10.1", optional = true }
-static-files = { version = "0.2.3", optional = true }
-tempfile = { version = "3.9.0", optional = true }
-zip = { version = "0.6.6", optional = true }
+static-files = { version = "0.2.4", optional = true }
+tempfile = { version = "3.10.1", optional = true }
+zip = { version = "2.1.3", optional = true }
 
 [features]
 default = ["analytics", "meilisearch-types/all-tokenizations", "mini-dashboard"]
 analytics = ["segment"]
 mini-dashboard = [
-    "actix-web-static-files",
     "static-files",
     "anyhow",
     "cargo_toml",
@@ -151,6 +145,7 @@ chinese = ["meilisearch-types/chinese"]
 chinese-pinyin = ["meilisearch-types/chinese-pinyin"]
 hebrew = ["meilisearch-types/hebrew"]
 japanese = ["meilisearch-types/japanese"]
+korean = ["meilisearch-types/korean"]
 thai = ["meilisearch-types/thai"]
 greek = ["meilisearch-types/greek"]
 khmer = ["meilisearch-types/khmer"]
@@ -158,5 +153,5 @@ vietnamese = ["meilisearch-types/vietnamese"]
 swedish-recomposition = ["meilisearch-types/swedish-recomposition"]
 
 [package.metadata.mini-dashboard]
-assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.13/build.zip"
-sha1 = "e20cc9b390003c6c844f4b8bcc5c5013191a77ff"
+assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.14/build.zip"
+sha1 = "592d1b5a3459d621d0aae1dded8fe3154f5c38fe"

meilisearch/src/analytics/mock_analytics.rs

@@ -6,7 +6,7 @@ use meilisearch_types::InstanceUid;
 use serde_json::Value;
 
 use super::{find_user_id, Analytics, DocumentDeletionKind, DocumentFetchKind};
-use crate::routes::indexes::documents::UpdateDocumentsQuery;
+use crate::routes::indexes::documents::{DocumentEditionByFunction, UpdateDocumentsQuery};
 use crate::Opt;
 
 pub struct MockAnalytics {
@@ -42,7 +42,7 @@ pub struct MultiSearchAggregator;
 
 #[allow(dead_code)]
 impl MultiSearchAggregator {
-    pub fn from_queries(_: &dyn Any, _: &dyn Any) -> Self {
+    pub fn from_federated_search(_: &dyn Any, _: &dyn Any) -> Self {
         Self
     }
 
@@ -97,6 +97,13 @@ impl Analytics for MockAnalytics {
         _request: &HttpRequest,
     ) {
     }
+    fn update_documents_by_function(
+        &self,
+        _documents_query: &DocumentEditionByFunction,
+        _index_creation: bool,
+        _request: &HttpRequest,
+    ) {
+    }
     fn get_fetch_documents(&self, _documents_query: &DocumentFetchKind, _request: &HttpRequest) {}
     fn post_fetch_documents(&self, _documents_query: &DocumentFetchKind, _request: &HttpRequest) {}
 }

meilisearch/src/analytics/mod.rs

@@ -13,7 +13,7 @@ use once_cell::sync::Lazy;
 use platform_dirs::AppDirs;
 use serde_json::Value;
 
-use crate::routes::indexes::documents::UpdateDocumentsQuery;
+use crate::routes::indexes::documents::{DocumentEditionByFunction, UpdateDocumentsQuery};
 
 // if the analytics feature is disabled
 // the `SegmentAnalytics` point to the mock instead of the real analytics
@@ -74,8 +74,8 @@ pub enum DocumentDeletionKind {
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum DocumentFetchKind {
-    PerDocumentId,
-    Normal { with_filter: bool, limit: usize, offset: usize },
+    PerDocumentId { retrieve_vectors: bool },
+    Normal { with_filter: bool, limit: usize, offset: usize, retrieve_vectors: bool },
 }
 
 pub trait Analytics: Sync + Send {
@@ -119,11 +119,19 @@ pub trait Analytics: Sync + Send {
     // this method should be called to aggregate a add documents request
     fn delete_documents(&self, kind: DocumentDeletionKind, request: &HttpRequest);
 
-    // this method should be called to batch a update documents request
+    // this method should be called to batch an update documents request
     fn update_documents(
         &self,
         documents_query: &UpdateDocumentsQuery,
         index_creation: bool,
         request: &HttpRequest,
     );
+
+    // this method should be called to batch an update documents by function request
+    fn update_documents_by_function(
+        &self,
+        documents_query: &DocumentEditionByFunction,
+        index_creation: bool,
+        request: &HttpRequest,
+    );
 }

meilisearch/src/analytics/segment_analytics.rs

@@ -5,10 +5,9 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-use actix_web::http::header::USER_AGENT;
+use actix_web::http::header::{CONTENT_TYPE, USER_AGENT};
 use actix_web::HttpRequest;
 use byte_unit::Byte;
-use http::header::CONTENT_TYPE;
 use index_scheduler::IndexScheduler;
 use meilisearch_auth::{AuthController, AuthFilter};
 use meilisearch_types::InstanceUid;
@@ -31,12 +30,12 @@ use crate::analytics::Analytics;
 use crate::option::{
     default_http_addr, IndexerOpts, LogMode, MaxMemory, MaxThreads, ScheduleSnapshot,
 };
-use crate::routes::indexes::documents::UpdateDocumentsQuery;
+use crate::routes::indexes::documents::{DocumentEditionByFunction, UpdateDocumentsQuery};
 use crate::routes::indexes::facet_search::FacetSearchQuery;
 use crate::routes::{create_all_stats, Stats};
 use crate::search::{
-    FacetSearchResult, MatchingStrategy, SearchQuery, SearchQueryWithIndex, SearchResult,
-    SimilarQuery, SimilarResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
+    FacetSearchResult, FederatedSearch, MatchingStrategy, SearchQuery, SearchQueryWithIndex,
+    SearchResult, SimilarQuery, SimilarResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
     DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
     DEFAULT_SEMANTIC_RATIO,
 };
@@ -81,6 +80,7 @@ pub enum AnalyticsMsg {
     AggregateAddDocuments(DocumentsAggregator),
     AggregateDeleteDocuments(DocumentsDeletionAggregator),
     AggregateUpdateDocuments(DocumentsAggregator),
+    AggregateEditDocumentsByFunction(EditDocumentsByFunctionAggregator),
     AggregateGetFetchDocuments(DocumentsFetchAggregator),
     AggregatePostFetchDocuments(DocumentsFetchAggregator),
 }
@@ -150,6 +150,7 @@ impl SegmentAnalytics {
             add_documents_aggregator: DocumentsAggregator::default(),
             delete_documents_aggregator: DocumentsDeletionAggregator::default(),
             update_documents_aggregator: DocumentsAggregator::default(),
+            edit_documents_by_function_aggregator: EditDocumentsByFunctionAggregator::default(),
             get_fetch_documents_aggregator: DocumentsFetchAggregator::default(),
             post_fetch_documents_aggregator: DocumentsFetchAggregator::default(),
             get_similar_aggregator: SimilarAggregator::default(),
@@ -230,6 +231,17 @@ impl super::Analytics for SegmentAnalytics {
         let _ = self.sender.try_send(AnalyticsMsg::AggregateUpdateDocuments(aggregate));
     }
 
+    fn update_documents_by_function(
+        &self,
+        documents_query: &DocumentEditionByFunction,
+        index_creation: bool,
+        request: &HttpRequest,
+    ) {
+        let aggregate =
+            EditDocumentsByFunctionAggregator::from_query(documents_query, index_creation, request);
+        let _ = self.sender.try_send(AnalyticsMsg::AggregateEditDocumentsByFunction(aggregate));
+    }
+
     fn get_fetch_documents(&self, documents_query: &DocumentFetchKind, request: &HttpRequest) {
         let aggregate = DocumentsFetchAggregator::from_query(documents_query, request);
         let _ = self.sender.try_send(AnalyticsMsg::AggregateGetFetchDocuments(aggregate));
@@ -390,6 +402,7 @@ pub struct Segment {
     add_documents_aggregator: DocumentsAggregator,
     delete_documents_aggregator: DocumentsDeletionAggregator,
     update_documents_aggregator: DocumentsAggregator,
+    edit_documents_by_function_aggregator: EditDocumentsByFunctionAggregator,
     get_fetch_documents_aggregator: DocumentsFetchAggregator,
     post_fetch_documents_aggregator: DocumentsFetchAggregator,
     get_similar_aggregator: SimilarAggregator,
@@ -454,6 +467,7 @@ impl Segment {
                         Some(AnalyticsMsg::AggregateAddDocuments(agreg)) => self.add_documents_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregateDeleteDocuments(agreg)) => self.delete_documents_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregateUpdateDocuments(agreg)) => self.update_documents_aggregator.aggregate(agreg),
+                        Some(AnalyticsMsg::AggregateEditDocumentsByFunction(agreg)) => self.edit_documents_by_function_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregateGetFetchDocuments(agreg)) => self.get_fetch_documents_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregatePostFetchDocuments(agreg)) => self.post_fetch_documents_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregateGetSimilar(agreg)) => self.get_similar_aggregator.aggregate(agreg),
@@ -509,6 +523,7 @@ impl Segment {
             add_documents_aggregator,
             delete_documents_aggregator,
             update_documents_aggregator,
+            edit_documents_by_function_aggregator,
             get_fetch_documents_aggregator,
             post_fetch_documents_aggregator,
             get_similar_aggregator,
@@ -550,6 +565,11 @@ impl Segment {
         {
             let _ = self.batcher.push(update_documents).await;
         }
+        if let Some(edit_documents_by_function) = take(edit_documents_by_function_aggregator)
+            .into_event(user, "Documents Edited By Function")
+        {
+            let _ = self.batcher.push(edit_documents_by_function).await;
+        }
         if let Some(get_fetch_documents) =
             take(get_fetch_documents_aggregator).into_event(user, "Documents Fetched GET")
         {
@@ -597,6 +617,9 @@ pub struct SearchAggregator {
     // every time a request has a filter, this field must be incremented by one
     sort_total_number_of_criteria: usize,
 
+    // distinct
+    distinct: bool,
+
     // filter
     filter_with_geo_radius: bool,
     filter_with_geo_bounding_box: bool,
@@ -622,6 +645,7 @@ pub struct SearchAggregator {
     // Whether a non-default embedder was specified
     embedder: bool,
     hybrid: bool,
+    retrieve_vectors: bool,
 
     // every time a search is done, we increment the counter linked to the used settings
     matching_strategy: HashMap<String, usize>,
@@ -662,6 +686,7 @@ impl SearchAggregator {
             page,
             hits_per_page,
             attributes_to_retrieve: _,
+            retrieve_vectors,
             attributes_to_crop: _,
             crop_length,
             attributes_to_highlight: _,
@@ -670,6 +695,7 @@ impl SearchAggregator {
             show_ranking_score_details,
             filter,
             sort,
+            distinct,
             facets: _,
             highlight_pre_tag,
             highlight_post_tag,
@@ -692,6 +718,8 @@ impl SearchAggregator {
             ret.sort_sum_of_criteria_terms = sort.len();
         }
 
+        ret.distinct = distinct.is_some();
+
         if let Some(ref filter) = filter {
             static RE: Lazy<Regex> = Lazy::new(|| Regex::new("AND | OR").unwrap());
             ret.filter_total_number_of_criteria = 1;
@@ -728,6 +756,7 @@ impl SearchAggregator {
         if let Some(ref vector) = vector {
             ret.max_vector_size = vector.len();
         }
+        ret.retrieve_vectors |= retrieve_vectors;
 
         if query.is_finite_pagination() {
             let limit = hits_per_page.unwrap_or_else(DEFAULT_SEARCH_LIMIT);
@@ -795,6 +824,7 @@ impl SearchAggregator {
             sort_with_geo_point,
             sort_sum_of_criteria_terms,
             sort_total_number_of_criteria,
+            distinct,
             filter_with_geo_radius,
             filter_with_geo_bounding_box,
             filter_sum_of_criteria_terms,
@@ -803,6 +833,7 @@ impl SearchAggregator {
             attributes_to_search_on_total_number_of_uses,
             max_terms_number,
             max_vector_size,
+            retrieve_vectors,
             matching_strategy,
             max_limit,
             max_offset,
@@ -851,6 +882,9 @@ impl SearchAggregator {
         self.sort_total_number_of_criteria =
             self.sort_total_number_of_criteria.saturating_add(sort_total_number_of_criteria);
 
+        // distinct
+        self.distinct |= distinct;
+
         // filter
         self.filter_with_geo_radius |= filter_with_geo_radius;
         self.filter_with_geo_bounding_box |= filter_with_geo_bounding_box;
@@ -873,6 +907,7 @@ impl SearchAggregator {
 
         // vector
         self.max_vector_size = self.max_vector_size.max(max_vector_size);
+        self.retrieve_vectors |= retrieve_vectors;
         self.semantic_ratio |= semantic_ratio;
         self.hybrid |= hybrid;
         self.embedder |= embedder;
@@ -921,6 +956,7 @@ impl SearchAggregator {
             sort_with_geo_point,
             sort_sum_of_criteria_terms,
             sort_total_number_of_criteria,
+            distinct,
             filter_with_geo_radius,
             filter_with_geo_bounding_box,
             filter_sum_of_criteria_terms,
@@ -929,6 +965,7 @@ impl SearchAggregator {
             attributes_to_search_on_total_number_of_uses,
             max_terms_number,
             max_vector_size,
+            retrieve_vectors,
             matching_strategy,
             max_limit,
             max_offset,
@@ -977,6 +1014,7 @@ impl SearchAggregator {
                     "with_geoPoint": sort_with_geo_point,
                     "avg_criteria_number": format!("{:.2}", sort_sum_of_criteria_terms as f64 / sort_total_number_of_criteria as f64),
                 },
+                "distinct": distinct,
                 "filter": {
                    "with_geoRadius": filter_with_geo_radius,
                    "with_geoBoundingBox": filter_with_geo_bounding_box,
@@ -991,6 +1029,7 @@ impl SearchAggregator {
                 },
                 "vector": {
                     "max_vector_size": max_vector_size,
+                    "retrieve_vectors": retrieve_vectors,
                 },
                 "hybrid": {
                     "enabled": hybrid,
@@ -1056,22 +1095,33 @@ pub struct MultiSearchAggregator {
     show_ranking_score: bool,
     show_ranking_score_details: bool,
 
+    // federation
+    use_federation: bool,
+
     // context
     user_agents: HashSet<String>,
 }
 
 impl MultiSearchAggregator {
-    pub fn from_queries(query: &[SearchQueryWithIndex], request: &HttpRequest) -> Self {
+    pub fn from_federated_search(
+        federated_search: &FederatedSearch,
+        request: &HttpRequest,
+    ) -> Self {
         let timestamp = Some(OffsetDateTime::now_utc());
 
         let user_agents = extract_user_agents(request).into_iter().collect();
 
-        let distinct_indexes: HashSet<_> = query
+        let use_federation = federated_search.federation.is_some();
+
+        let distinct_indexes: HashSet<_> = federated_search
+            .queries
             .iter()
             .map(|query| {
+                let query = &query;
                 // make sure we get a compilation error if a field gets added to / removed from SearchQueryWithIndex
                 let SearchQueryWithIndex {
                     index_uid,
+                    federation_options: _,
                     q: _,
                     vector: _,
                     offset: _,
@@ -1079,6 +1129,7 @@ impl MultiSearchAggregator {
                     page: _,
                     hits_per_page: _,
                     attributes_to_retrieve: _,
+                    retrieve_vectors: _,
                     attributes_to_crop: _,
                     crop_length: _,
                     attributes_to_highlight: _,
@@ -1087,6 +1138,7 @@ impl MultiSearchAggregator {
                     show_matches_position: _,
                     filter: _,
                     sort: _,
+                    distinct: _,
                     facets: _,
                     highlight_pre_tag: _,
                     highlight_post_tag: _,
@@ -1101,8 +1153,10 @@ impl MultiSearchAggregator {
             })
             .collect();
 
-        let show_ranking_score = query.iter().any(|query| query.show_ranking_score);
-        let show_ranking_score_details = query.iter().any(|query| query.show_ranking_score_details);
+        let show_ranking_score =
+            federated_search.queries.iter().any(|query| query.show_ranking_score);
+        let show_ranking_score_details =
+            federated_search.queries.iter().any(|query| query.show_ranking_score_details);
 
         Self {
             timestamp,
@@ -1110,10 +1164,11 @@ impl MultiSearchAggregator {
             total_succeeded: 0,
             total_distinct_index_count: distinct_indexes.len(),
             total_single_index: if distinct_indexes.len() == 1 { 1 } else { 0 },
-            total_search_count: query.len(),
+            total_search_count: federated_search.queries.len(),
             show_ranking_score,
             show_ranking_score_details,
             user_agents,
+            use_federation,
         }
     }
 
@@ -1139,6 +1194,7 @@ impl MultiSearchAggregator {
         let show_ranking_score_details =
             this.show_ranking_score_details || other.show_ranking_score_details;
         let mut user_agents = this.user_agents;
+        let use_federation = this.use_federation || other.use_federation;
 
         for user_agent in other.user_agents.into_iter() {
             user_agents.insert(user_agent);
@@ -1155,6 +1211,7 @@ impl MultiSearchAggregator {
             user_agents,
             show_ranking_score,
             show_ranking_score_details,
+            use_federation,
             // do not add _ or ..Default::default() here
         };
 
@@ -1173,6 +1230,7 @@ impl MultiSearchAggregator {
             user_agents,
             show_ranking_score,
             show_ranking_score_details,
+            use_federation,
         } = self;
 
         if total_received == 0 {
@@ -1197,6 +1255,9 @@ impl MultiSearchAggregator {
                 "scoring": {
                     "show_ranking_score": show_ranking_score,
                     "show_ranking_score_details": show_ranking_score_details,
+                },
+                "federation": {
+                    "use_federation": use_federation,
                 }
             });
 
@@ -1445,6 +1506,75 @@ impl DocumentsAggregator {
     }
 }
 
+#[derive(Default)]
+pub struct EditDocumentsByFunctionAggregator {
+    timestamp: Option<OffsetDateTime>,
+
+    // Set to true if at least one request was filtered
+    filtered: bool,
+    // Set to true if at least one request contained a context
+    with_context: bool,
+
+    // context
+    user_agents: HashSet<String>,
+
+    index_creation: bool,
+}
+
+impl EditDocumentsByFunctionAggregator {
+    pub fn from_query(
+        documents_query: &DocumentEditionByFunction,
+        index_creation: bool,
+        request: &HttpRequest,
+    ) -> Self {
+        let DocumentEditionByFunction { filter, context, function: _ } = documents_query;
+
+        Self {
+            timestamp: Some(OffsetDateTime::now_utc()),
+            user_agents: extract_user_agents(request).into_iter().collect(),
+            filtered: filter.is_some(),
+            with_context: context.is_some(),
+            index_creation,
+        }
+    }
+
+    /// Aggregate one [DocumentsAggregator] into another.
+    pub fn aggregate(&mut self, other: Self) {
+        let Self { timestamp, user_agents, index_creation, filtered, with_context } = other;
+
+        if self.timestamp.is_none() {
+            self.timestamp = timestamp;
+        }
+
+        // we can't create a union because there is no `into_union` method
+        for user_agent in user_agents {
+            self.user_agents.insert(user_agent);
+        }
+        self.index_creation |= index_creation;
+        self.filtered |= filtered;
+        self.with_context |= with_context;
+    }
+
+    pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
+        let Self { timestamp, user_agents, index_creation, filtered, with_context } = self;
+
+        let properties = json!({
+            "user-agent": user_agents,
+            "filtered": filtered,
+            "with_context": with_context,
+            "index_creation": index_creation,
+        });
+
+        Some(Track {
+            timestamp,
+            user: user.clone(),
+            event: event_name.to_string(),
+            properties,
+            ..Default::default()
+        })
+    }
+}
+
 #[derive(Default, Serialize)]
 pub struct DocumentsDeletionAggregator {
     #[serde(skip)]
@@ -1534,6 +1664,9 @@ pub struct DocumentsFetchAggregator {
     // if a filter was used
     per_filter: bool,
 
+    #[serde(rename = "vector.retrieve_vectors")]
+    retrieve_vectors: bool,
+
     // pagination
     #[serde(rename = "pagination.max_limit")]
     max_limit: usize,
@@ -1543,18 +1676,21 @@ pub struct DocumentsFetchAggregator {
 
 impl DocumentsFetchAggregator {
     pub fn from_query(query: &DocumentFetchKind, request: &HttpRequest) -> Self {
-        let (limit, offset) = match query {
-            DocumentFetchKind::PerDocumentId => (1, 0),
-            DocumentFetchKind::Normal { limit, offset, .. } => (*limit, *offset),
+        let (limit, offset, retrieve_vectors) = match query {
+            DocumentFetchKind::PerDocumentId { retrieve_vectors } => (1, 0, *retrieve_vectors),
+            DocumentFetchKind::Normal { limit, offset, retrieve_vectors, .. } => {
+                (*limit, *offset, *retrieve_vectors)
+            }
         };
         Self {
             timestamp: Some(OffsetDateTime::now_utc()),
             user_agents: extract_user_agents(request).into_iter().collect(),
             total_received: 1,
-            per_document_id: matches!(query, DocumentFetchKind::PerDocumentId),
+            per_document_id: matches!(query, DocumentFetchKind::PerDocumentId { .. }),
             per_filter: matches!(query, DocumentFetchKind::Normal { with_filter, .. } if *with_filter),
             max_limit: limit,
             max_offset: offset,
+            retrieve_vectors,
         }
     }
 
@@ -1568,6 +1704,7 @@ impl DocumentsFetchAggregator {
             per_filter,
             max_limit,
             max_offset,
+            retrieve_vectors,
         } = other;
 
         if self.timestamp.is_none() {
@@ -1583,6 +1720,8 @@ impl DocumentsFetchAggregator {
 
         self.max_limit = self.max_limit.max(max_limit);
         self.max_offset = self.max_offset.max(max_offset);
+
+        self.retrieve_vectors |= retrieve_vectors;
     }
 
     pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
@@ -1623,6 +1762,7 @@ pub struct SimilarAggregator {
 
     // Whether a non-default embedder was specified
     embedder: bool,
+    retrieve_vectors: bool,
 
     // pagination
     max_limit: usize,
@@ -1646,6 +1786,7 @@ impl SimilarAggregator {
             offset,
             limit,
             attributes_to_retrieve: _,
+            retrieve_vectors,
             show_ranking_score,
             show_ranking_score_details,
             filter,
@@ -1690,6 +1831,7 @@ impl SimilarAggregator {
         ret.ranking_score_threshold = ranking_score_threshold.is_some();
 
         ret.embedder = embedder.is_some();
+        ret.retrieve_vectors = *retrieve_vectors;
 
         ret
     }
@@ -1722,6 +1864,7 @@ impl SimilarAggregator {
             show_ranking_score_details,
             embedder,
             ranking_score_threshold,
+            retrieve_vectors,
         } = other;
 
         if self.timestamp.is_none() {
@@ -1751,6 +1894,7 @@ impl SimilarAggregator {
         }
 
         self.embedder |= embedder;
+        self.retrieve_vectors |= retrieve_vectors;
 
         // pagination
         self.max_limit = self.max_limit.max(max_limit);
@@ -1785,6 +1929,7 @@ impl SimilarAggregator {
             show_ranking_score_details,
             embedder,
             ranking_score_threshold,
+            retrieve_vectors,
         } = self;
 
         if total_received == 0 {
@@ -1811,6 +1956,9 @@ impl SimilarAggregator {
                    "avg_criteria_number": format!("{:.2}", filter_sum_of_criteria_terms as f64 / filter_total_number_of_criteria as f64),
                    "most_used_syntax": used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)),
                 },
+                "vector": {
+                    "retrieve_vectors": retrieve_vectors,
+                },
                 "hybrid": {
                     "embedder": embedder,
                 },

meilisearch/src/error.rs

@@ -1,6 +1,6 @@
 use actix_web as aweb;
 use aweb::error::{JsonPayloadError, QueryPayloadError};
-use byte_unit::Byte;
+use byte_unit::{Byte, UnitType};
 use meilisearch_types::document_formats::{DocumentFormatError, PayloadType};
 use meilisearch_types::error::{Code, ErrorCode, ResponseError};
 use meilisearch_types::index_uid::{IndexUid, IndexUidFormatError};
@@ -25,6 +25,10 @@ pub enum MeilisearchHttpError {
     DocumentNotFound(String),
     #[error("Sending an empty filter is forbidden.")]
     EmptyFilter,
+    #[error("Using `federationOptions` is not allowed in a non-federated search.\n Hint: remove `federationOptions` from query #{0} or add `federation: {{}}` to the request.")]
+    FederationOptionsInNonFederatedRequest(usize),
+    #[error("Inside `.queries[{0}]`: Using pagination options is not allowed in federated queries.\n Hint: remove `{1}` from query #{0} or remove `federation: {{}}` from the request")]
+    PaginationInFederatedQuery(usize, &'static str),
     #[error("Invalid syntax for the filter parameter: `expected {}, found: {1}`.", .0.join(", "))]
     InvalidExpression(&'static [&'static str], Value),
     #[error("A {0} payload is missing.")]
@@ -33,7 +37,7 @@ pub enum MeilisearchHttpError {
     TooManySearchRequests(usize),
     #[error("Internal error: Search limiter is down.")]
     SearchLimiterIsDown,
-    #[error("The provided payload reached the size limit. The maximum accepted payload size is {}.",  Byte::from_bytes(*.0 as u64).get_appropriate_unit(true))]
+    #[error("The provided payload reached the size limit. The maximum accepted payload size is {}.",  Byte::from_u64(*.0 as u64).get_appropriate_unit(UnitType::Binary))]
     PayloadTooLarge(usize),
     #[error("Two indexes must be given for each swap. The list `[{}]` contains {} indexes.",
         .0.iter().map(|uid| format!("\"{uid}\"")).collect::<Vec<_>>().join(", "), .0.len()
@@ -86,6 +90,12 @@ impl ErrorCode for MeilisearchHttpError {
             MeilisearchHttpError::DocumentFormat(e) => e.error_code(),
             MeilisearchHttpError::Join(_) => Code::Internal,
             MeilisearchHttpError::MissingSearchHybrid => Code::MissingSearchHybrid,
+            MeilisearchHttpError::FederationOptionsInNonFederatedRequest(_) => {
+                Code::InvalidMultiSearchFederationOptions
+            }
+            MeilisearchHttpError::PaginationInFederatedQuery(_, _) => {
+                Code::InvalidMultiSearchQueryPagination
+            }
         }
     }
 }
@@ -98,14 +108,29 @@ impl From<MeilisearchHttpError> for aweb::Error {
 
 impl From<aweb::error::PayloadError> for MeilisearchHttpError {
     fn from(error: aweb::error::PayloadError) -> Self {
-        MeilisearchHttpError::Payload(PayloadError::Payload(error))
+        match error {
+            aweb::error::PayloadError::Incomplete(_) => MeilisearchHttpError::Payload(
+                PayloadError::Payload(ActixPayloadError::IncompleteError),
+            ),
+            _ => MeilisearchHttpError::Payload(PayloadError::Payload(
+                ActixPayloadError::OtherError(error),
+            )),
+        }
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+pub enum ActixPayloadError {
+    #[error("The provided payload is incomplete and cannot be parsed")]
+    IncompleteError,
+    #[error(transparent)]
+    OtherError(aweb::error::PayloadError),
+}
+
 #[derive(Debug, thiserror::Error)]
 pub enum PayloadError {
     #[error(transparent)]
-    Payload(aweb::error::PayloadError),
+    Payload(ActixPayloadError),
     #[error(transparent)]
     Json(JsonPayloadError),
     #[error(transparent)]
@@ -122,13 +147,15 @@ impl ErrorCode for PayloadError {
     fn error_code(&self) -> Code {
         match self {
             PayloadError::Payload(e) => match e {
-                aweb::error::PayloadError::Incomplete(_) => Code::Internal,
-                aweb::error::PayloadError::EncodingCorrupted => Code::Internal,
-                aweb::error::PayloadError::Overflow => Code::PayloadTooLarge,
-                aweb::error::PayloadError::UnknownLength => Code::Internal,
-                aweb::error::PayloadError::Http2Payload(_) => Code::Internal,
-                aweb::error::PayloadError::Io(_) => Code::Internal,
-                _ => todo!(),
+                ActixPayloadError::IncompleteError => Code::BadRequest,
+                ActixPayloadError::OtherError(error) => match error {
+                    aweb::error::PayloadError::EncodingCorrupted => Code::Internal,
+                    aweb::error::PayloadError::Overflow => Code::PayloadTooLarge,
+                    aweb::error::PayloadError::UnknownLength => Code::Internal,
+                    aweb::error::PayloadError::Http2Payload(_) => Code::Internal,
+                    aweb::error::PayloadError::Io(_) => Code::Internal,
+                    _ => todo!(),
+                },
             },
             PayloadError::Json(err) => match err {
                 JsonPayloadError::Overflow { .. } => Code::PayloadTooLarge,

meilisearch/src/extractors/authentication/mod.rs

@@ -12,6 +12,8 @@ use futures::Future;
 use meilisearch_auth::{AuthController, AuthFilter};
 use meilisearch_types::error::{Code, ResponseError};
 
+use self::policies::AuthError;
+
 pub struct GuardedData<P, D> {
     data: D,
     filters: AuthFilter,
@@ -35,12 +37,12 @@ impl<P, D> GuardedData<P, D> {
         let missing_master_key = auth.get_master_key().is_none();
 
         match Self::authenticate(auth, token, index).await? {
-            Some(filters) => match data {
+            Ok(filters) => match data {
                 Some(data) => Ok(Self { data, filters, _marker: PhantomData }),
                 None => Err(AuthenticationError::IrretrievableState.into()),
             },
-            None if missing_master_key => Err(AuthenticationError::MissingMasterKey.into()),
-            None => Err(AuthenticationError::InvalidToken.into()),
+            Err(_) if missing_master_key => Err(AuthenticationError::MissingMasterKey.into()),
+            Err(e) => Err(ResponseError::from_msg(e.to_string(), Code::InvalidApiKey)),
         }
     }
 
@@ -51,12 +53,12 @@ impl<P, D> GuardedData<P, D> {
         let missing_master_key = auth.get_master_key().is_none();
 
         match Self::authenticate(auth, String::new(), None).await? {
-            Some(filters) => match data {
+            Ok(filters) => match data {
                 Some(data) => Ok(Self { data, filters, _marker: PhantomData }),
                 None => Err(AuthenticationError::IrretrievableState.into()),
             },
-            None if missing_master_key => Err(AuthenticationError::MissingMasterKey.into()),
-            None => Err(AuthenticationError::MissingAuthorizationHeader.into()),
+            Err(_) if missing_master_key => Err(AuthenticationError::MissingMasterKey.into()),
+            Err(_) => Err(AuthenticationError::MissingAuthorizationHeader.into()),
         }
     }
 
@@ -64,7 +66,7 @@ impl<P, D> GuardedData<P, D> {
         auth: Data<AuthController>,
         token: String,
         index: Option<String>,
-    ) -> Result<Option<AuthFilter>, ResponseError>
+    ) -> Result<Result<AuthFilter, AuthError>, ResponseError>
     where
         P: Policy + 'static,
     {
@@ -127,13 +129,14 @@ pub trait Policy {
         auth: Data<AuthController>,
         token: &str,
         index: Option<&str>,
-    ) -> Option<AuthFilter>;
+    ) -> Result<AuthFilter, policies::AuthError>;
 }
 
 pub mod policies {
     use actix_web::web::Data;
     use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation};
     use meilisearch_auth::{AuthController, AuthFilter, SearchRules};
+    use meilisearch_types::error::{Code, ErrorCode};
     // reexport actions in policies in order to be used in routes configuration.
     pub use meilisearch_types::keys::{actions, Action};
     use serde::{Deserialize, Serialize};
@@ -144,11 +147,53 @@ pub mod policies {
 
     enum TenantTokenOutcome {
         NotATenantToken,
-        Invalid,
-        Expired,
         Valid(Uuid, SearchRules),
     }
 
+    #[derive(thiserror::Error, Debug)]
+    pub enum AuthError {
+        #[error("Tenant token expired. Was valid up to `{exp}` and we're now `{now}`.")]
+        ExpiredTenantToken { exp: i64, now: i64 },
+        #[error("The provided API key is invalid.")]
+        InvalidApiKey,
+        #[error("The provided tenant token cannot acces the index `{index}`, allowed indexes are {allowed:?}.")]
+        TenantTokenAccessingnUnauthorizedIndex { index: String, allowed: Vec<String> },
+        #[error(
+            "The API key used to generate this tenant token cannot acces the index `{index}`."
+        )]
+        TenantTokenApiKeyAccessingnUnauthorizedIndex { index: String },
+        #[error(
+            "The API key cannot acces the index `{index}`, authorized indexes are {allowed:?}."
+        )]
+        ApiKeyAccessingnUnauthorizedIndex { index: String, allowed: Vec<String> },
+        #[error("The provided tenant token is invalid.")]
+        InvalidTenantToken,
+        #[error("Could not decode tenant token, {0}.")]
+        CouldNotDecodeTenantToken(jsonwebtoken::errors::Error),
+        #[error("Invalid action `{0}`.")]
+        InternalInvalidAction(u8),
+    }
+
+    impl From<jsonwebtoken::errors::Error> for AuthError {
+        fn from(error: jsonwebtoken::errors::Error) -> Self {
+            use jsonwebtoken::errors::ErrorKind;
+
+            match error.kind() {
+                ErrorKind::InvalidToken => AuthError::InvalidTenantToken,
+                _ => AuthError::CouldNotDecodeTenantToken(error),
+            }
+        }
+    }
+
+    impl ErrorCode for AuthError {
+        fn error_code(&self) -> Code {
+            match self {
+                AuthError::InternalInvalidAction(_) => Code::Internal,
+                _ => Code::InvalidApiKey,
+            }
+        }
+    }
+
     fn tenant_token_validation() -> Validation {
         let mut validation = Validation::default();
         validation.validate_exp = false;
@@ -158,15 +203,15 @@ pub mod policies {
     }
 
     /// Extracts the key id used to sign the payload, without performing any validation.
-    fn extract_key_id(token: &str) -> Option<Uuid> {
+    fn extract_key_id(token: &str) -> Result<Uuid, AuthError> {
         let mut validation = tenant_token_validation();
         validation.insecure_disable_signature_validation();
         let dummy_key = DecodingKey::from_secret(b"secret");
-        let token_data = decode::<Claims>(token, &dummy_key, &validation).ok()?;
+        let token_data = decode::<Claims>(token, &dummy_key, &validation)?;
 
         // get token fields without validating it.
         let Claims { api_key_uid, .. } = token_data.claims;
-        Some(api_key_uid)
+        Ok(api_key_uid)
     }
 
     fn is_keys_action(action: u8) -> bool {
@@ -187,76 +232,102 @@ pub mod policies {
             auth: Data<AuthController>,
             token: &str,
             index: Option<&str>,
-        ) -> Option<AuthFilter> {
+        ) -> Result<AuthFilter, AuthError> {
             // authenticate if token is the master key.
             // Without a master key, all routes are accessible except the key-related routes.
             if auth.get_master_key().map_or_else(|| !is_keys_action(A), |mk| mk == token) {
-                return Some(AuthFilter::default());
+                return Ok(AuthFilter::default());
             }
 
             let (key_uuid, search_rules) =
                 match ActionPolicy::<A>::authenticate_tenant_token(&auth, token) {
-                    TenantTokenOutcome::Valid(key_uuid, search_rules) => {
+                    Ok(TenantTokenOutcome::Valid(key_uuid, search_rules)) => {
                         (key_uuid, Some(search_rules))
                     }
-                    TenantTokenOutcome::Expired => return None,
-                    TenantTokenOutcome::Invalid => return None,
-                    TenantTokenOutcome::NotATenantToken => {
-                        (auth.get_optional_uid_from_encoded_key(token.as_bytes()).ok()??, None)
-                    }
+                    Ok(TenantTokenOutcome::NotATenantToken)
+                    | Err(AuthError::InvalidTenantToken) => (
+                        auth.get_optional_uid_from_encoded_key(token.as_bytes())
+                            .map_err(|_e| AuthError::InvalidApiKey)?
+                            .ok_or(AuthError::InvalidApiKey)?,
+                        None,
+                    ),
+                    Err(e) => return Err(e),
                 };
 
             // check that the indexes are allowed
-            let action = Action::from_repr(A)?;
-            let auth_filter = auth.get_key_filters(key_uuid, search_rules).ok()?;
-            if auth.is_key_authorized(key_uuid, action, index).unwrap_or(false)
-                && index.map(|index| auth_filter.is_index_authorized(index)).unwrap_or(true)
-            {
-                return Some(auth_filter);
+            let action = Action::from_repr(A).ok_or(AuthError::InternalInvalidAction(A))?;
+            let auth_filter = auth
+                .get_key_filters(key_uuid, search_rules)
+                .map_err(|_e| AuthError::InvalidApiKey)?;
+
+            // First check if the index is authorized in the tenant token, this is a public
+            // information, we can return a nice error message.
+            if let Some(index) = index {
+                if !auth_filter.tenant_token_is_index_authorized(index) {
+                    return Err(AuthError::TenantTokenAccessingnUnauthorizedIndex {
+                        index: index.to_string(),
+                        allowed: auth_filter.tenant_token_list_index_authorized(),
+                    });
+                }
+                if !auth_filter.api_key_is_index_authorized(index) {
+                    if auth_filter.is_tenant_token() {
+                        // If the error comes from a tenant token we cannot share the list
+                        // of authorized indexes in the API key. This is not public information.
+                        return Err(AuthError::TenantTokenApiKeyAccessingnUnauthorizedIndex {
+                            index: index.to_string(),
+                        });
+                    } else {
+                        // Otherwise we can share the list
+                        // of authorized indexes in the API key.
+                        return Err(AuthError::ApiKeyAccessingnUnauthorizedIndex {
+                            index: index.to_string(),
+                            allowed: auth_filter.api_key_list_index_authorized(),
+                        });
+                    }
+                }
+            }
+            if auth.is_key_authorized(key_uuid, action, index).unwrap_or(false) {
+                return Ok(auth_filter);
             }
 
-            None
+            Err(AuthError::InvalidApiKey)
         }
     }
 
     impl<const A: u8> ActionPolicy<A> {
-        fn authenticate_tenant_token(auth: &AuthController, token: &str) -> TenantTokenOutcome {
+        fn authenticate_tenant_token(
+            auth: &AuthController,
+            token: &str,
+        ) -> Result<TenantTokenOutcome, AuthError> {
             // Only search action can be accessed by a tenant token.
             if A != actions::SEARCH {
-                return TenantTokenOutcome::NotATenantToken;
+                return Ok(TenantTokenOutcome::NotATenantToken);
             }
 
-            let uid = if let Some(uid) = extract_key_id(token) {
-                uid
-            } else {
-                return TenantTokenOutcome::NotATenantToken;
-            };
+            let uid = extract_key_id(token)?;
 
             // Check if tenant token is valid.
             let key = if let Some(key) = auth.generate_key(uid) {
                 key
             } else {
-                return TenantTokenOutcome::Invalid;
+                return Err(AuthError::InvalidTenantToken);
             };
 
-            let data = if let Ok(data) = decode::<Claims>(
+            let data = decode::<Claims>(
                 token,
                 &DecodingKey::from_secret(key.as_bytes()),
                 &tenant_token_validation(),
-            ) {
-                data
-            } else {
-                return TenantTokenOutcome::Invalid;
-            };
+            )?;
 
             // Check if token is expired.
             if let Some(exp) = data.claims.exp {
-                if OffsetDateTime::now_utc().unix_timestamp() > exp {
-                    return TenantTokenOutcome::Expired;
+                let now = OffsetDateTime::now_utc().unix_timestamp();
+                if now > exp {
+                    return Err(AuthError::ExpiredTenantToken { exp, now });
                 }
             }
 
-            TenantTokenOutcome::Valid(uid, data.claims.search_rules)
+            Ok(TenantTokenOutcome::Valid(uid, data.claims.search_rules))
         }
     }
 

meilisearch/src/lib.rs

@@ -15,6 +15,7 @@ use std::fs::File;
 use std::io::{BufReader, BufWriter};
 use std::num::NonZeroUsize;
 use std::path::Path;
+use std::str::FromStr;
 use std::sync::Arc;
 use std::thread::{self, available_parallelism};
 use std::time::Duration;
@@ -23,13 +24,13 @@ use actix_cors::Cors;
 use actix_http::body::MessageBody;
 use actix_web::dev::{ServiceFactory, ServiceResponse};
 use actix_web::error::JsonPayloadError;
+use actix_web::http::header::{CONTENT_TYPE, USER_AGENT};
 use actix_web::web::Data;
 use actix_web::{web, HttpRequest};
 use analytics::Analytics;
 use anyhow::bail;
 use error::PayloadError;
 use extractors::payload::PayloadConfig;
-use http::header::CONTENT_TYPE;
 use index_scheduler::{IndexScheduler, IndexSchedulerOptions};
 use meilisearch_auth::AuthController;
 use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
@@ -167,7 +168,7 @@ impl tracing_actix_web::RootSpanBuilder for AwebTracingLogger {
         let conn_info = request.connection_info();
         let headers = request.headers();
         let user_agent = headers
-            .get(http::header::USER_AGENT)
+            .get(USER_AGENT)
             .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned())
             .unwrap_or_default();
         info_span!("HTTP request", method = %request.method(), host = conn_info.host(), route = %request.path(), query_parameters = %request.query_string(), %user_agent, status_code = Empty, error = Empty)
@@ -300,15 +301,15 @@ fn open_or_create_database_unchecked(
             dumps_path: opt.dump_dir.clone(),
             webhook_url: opt.task_webhook_url.as_ref().map(|url| url.to_string()),
             webhook_authorization_header: opt.task_webhook_authorization_header.clone(),
-            task_db_size: opt.max_task_db_size.get_bytes() as usize,
-            index_base_map_size: opt.max_index_size.get_bytes() as usize,
+            task_db_size: opt.max_task_db_size.as_u64() as usize,
+            index_base_map_size: opt.max_index_size.as_u64() as usize,
             enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage,
             indexer_config: (&opt.indexer_options).try_into()?,
             autobatching_enabled: true,
             cleanup_enabled: !opt.experimental_replication_parameters,
             max_number_of_tasks: 1_000_000,
             max_number_of_batched_tasks: opt.experimental_max_number_of_batched_tasks,
-            index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().get_bytes() as usize,
+            index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().as_u64() as usize,
             index_count: DEFAULT_INDEX_COUNT,
             instance_features,
         })?)
@@ -476,7 +477,7 @@ pub fn configure_data(
         opt.experimental_search_queue_size,
         available_parallelism().unwrap_or(NonZeroUsize::new(2).unwrap()),
     );
-    let http_payload_size_limit = opt.http_payload_size_limit.get_bytes() as usize;
+    let http_payload_size_limit = opt.http_payload_size_limit.as_u64() as usize;
     config
         .app_data(index_scheduler)
         .app_data(auth)

meilisearch/src/option.rs

@@ -9,7 +9,7 @@ use std::str::FromStr;
 use std::sync::Arc;
 use std::{env, fmt, fs};
 
-use byte_unit::{Byte, ByteError};
+use byte_unit::{Byte, ParseError, UnitType};
 use clap::Parser;
 use meilisearch_types::features::InstanceTogglableFeatures;
 use meilisearch_types::milli::update::IndexerConfig;
@@ -674,7 +674,7 @@ impl TryFrom<&IndexerOpts> for IndexerConfig {
 
         Ok(Self {
             log_every_n: Some(DEFAULT_LOG_EVERY_N),
-            max_memory: other.max_indexing_memory.map(|b| b.get_bytes() as usize),
+            max_memory: other.max_indexing_memory.map(|b| b.as_u64() as usize),
             thread_pool: Some(thread_pool),
             max_positions_per_attributes: None,
             skip_index_budget: other.skip_index_budget,
@@ -688,23 +688,25 @@ impl TryFrom<&IndexerOpts> for IndexerConfig {
 pub struct MaxMemory(Option<Byte>);
 
 impl FromStr for MaxMemory {
-    type Err = ByteError;
+    type Err = ParseError;
 
-    fn from_str(s: &str) -> Result<MaxMemory, ByteError> {
+    fn from_str(s: &str) -> Result<MaxMemory, Self::Err> {
         Byte::from_str(s).map(Some).map(MaxMemory)
     }
 }
 
 impl Default for MaxMemory {
     fn default() -> MaxMemory {
-        MaxMemory(total_memory_bytes().map(|bytes| bytes * 2 / 3).map(Byte::from_bytes))
+        MaxMemory(total_memory_bytes().map(|bytes| bytes * 2 / 3).map(Byte::from_u64))
     }
 }
 
 impl fmt::Display for MaxMemory {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self.0 {
-            Some(memory) => write!(f, "{}", memory.get_appropriate_unit(true)),
+            Some(memory) => {
+                write!(f, "{}", memory.get_appropriate_unit(UnitType::Binary))
+            }
             None => f.write_str("unknown"),
         }
     }
@@ -844,11 +846,11 @@ fn default_env() -> String {
 }
 
 fn default_max_index_size() -> Byte {
-    Byte::from_bytes(INDEX_SIZE)
+    Byte::from_u64(INDEX_SIZE)
 }
 
 fn default_max_task_db_size() -> Byte {
-    Byte::from_bytes(TASK_DB_SIZE)
+    Byte::from_u64(TASK_DB_SIZE)
 }
 
 fn default_http_payload_size_limit() -> Byte {

meilisearch/src/routes/features.rs

@@ -47,6 +47,8 @@ pub struct RuntimeTogglableFeatures {
     pub metrics: Option<bool>,
     #[deserr(default)]
     pub logs_route: Option<bool>,
+    #[deserr(default)]
+    pub edit_documents_by_function: Option<bool>,
 }
 
 async fn patch_features(
@@ -66,13 +68,21 @@ async fn patch_features(
         vector_store: new_features.0.vector_store.unwrap_or(old_features.vector_store),
         metrics: new_features.0.metrics.unwrap_or(old_features.metrics),
         logs_route: new_features.0.logs_route.unwrap_or(old_features.logs_route),
+        edit_documents_by_function: new_features
+            .0
+            .edit_documents_by_function
+            .unwrap_or(old_features.edit_documents_by_function),
     };
 
     // explicitly destructure for analytics rather than using the `Serialize` implementation, because
     // the it renames to camelCase, which we don't want for analytics.
     // **Do not** ignore fields with `..` or `_` here, because we want to add them in the future.
-    let meilisearch_types::features::RuntimeTogglableFeatures { vector_store, metrics, logs_route } =
-        new_features;
+    let meilisearch_types::features::RuntimeTogglableFeatures {
+        vector_store,
+        metrics,
+        logs_route,
+        edit_documents_by_function,
+    } = new_features;
 
     analytics.publish(
         "Experimental features Updated".to_string(),
@@ -80,6 +90,7 @@ async fn patch_features(
             "vector_store": vector_store,
             "metrics": metrics,
             "logs_route": logs_route,
+            "edit_documents_by_function": edit_documents_by_function,
         }),
         Some(&req),
     );

meilisearch/src/routes/indexes/documents.rs

@@ -16,6 +16,7 @@ use meilisearch_types::error::{Code, ResponseError};
 use meilisearch_types::heed::RoTxn;
 use meilisearch_types::index_uid::IndexUid;
 use meilisearch_types::milli::update::IndexDocumentsMethod;
+use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors;
 use meilisearch_types::milli::DocumentId;
 use meilisearch_types::star_or::OptionStarOrList;
 use meilisearch_types::tasks::KindWithContent;
@@ -39,7 +40,7 @@ use crate::extractors::sequential_extractor::SeqHandler;
 use crate::routes::{
     get_task_id, is_dry_run, PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT,
 };
-use crate::search::parse_filter;
+use crate::search::{parse_filter, RetrieveVectors};
 use crate::Opt;
 
 static ACCEPTED_CONTENT_TYPE: Lazy<Vec<String>> = Lazy::new(|| {
@@ -81,6 +82,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         web::resource("/delete-batch").route(web::post().to(SeqHandler(delete_documents_batch))),
     )
     .service(web::resource("/delete").route(web::post().to(SeqHandler(delete_documents_by_filter))))
+    .service(web::resource("/edit").route(web::post().to(SeqHandler(edit_documents_by_function))))
     .service(web::resource("/fetch").route(web::post().to(SeqHandler(documents_by_query_post))))
     .service(
         web::resource("/{document_id}")
@@ -94,6 +96,8 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
 pub struct GetDocument {
     #[deserr(default, error = DeserrQueryParamError<InvalidDocumentFields>)]
     fields: OptionStarOrList<String>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidDocumentRetrieveVectors>)]
+    retrieve_vectors: Param<bool>,
 }
 
 pub async fn get_document(
@@ -107,13 +111,20 @@ pub async fn get_document(
     debug!(parameters = ?params, "Get document");
     let index_uid = IndexUid::try_from(index_uid)?;
 
-    analytics.get_fetch_documents(&DocumentFetchKind::PerDocumentId, &req);
-
-    let GetDocument { fields } = params.into_inner();
+    let GetDocument { fields, retrieve_vectors: param_retrieve_vectors } = params.into_inner();
     let attributes_to_retrieve = fields.merge_star_and_none();
 
+    let features = index_scheduler.features();
+    let retrieve_vectors = RetrieveVectors::new(param_retrieve_vectors.0, features)?;
+
+    analytics.get_fetch_documents(
+        &DocumentFetchKind::PerDocumentId { retrieve_vectors: param_retrieve_vectors.0 },
+        &req,
+    );
+
     let index = index_scheduler.index(&index_uid)?;
-    let document = retrieve_document(&index, &document_id, attributes_to_retrieve)?;
+    let document =
+        retrieve_document(&index, &document_id, attributes_to_retrieve, retrieve_vectors)?;
     debug!(returns = ?document, "Get document");
     Ok(HttpResponse::Ok().json(document))
 }
@@ -153,6 +164,8 @@ pub struct BrowseQueryGet {
     limit: Param<usize>,
     #[deserr(default, error = DeserrQueryParamError<InvalidDocumentFields>)]
     fields: OptionStarOrList<String>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidDocumentRetrieveVectors>)]
+    retrieve_vectors: Param<bool>,
     #[deserr(default, error = DeserrQueryParamError<InvalidDocumentFilter>)]
     filter: Option<String>,
 }
@@ -166,6 +179,8 @@ pub struct BrowseQuery {
     limit: usize,
     #[deserr(default, error = DeserrJsonError<InvalidDocumentFields>)]
     fields: Option<Vec<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidDocumentRetrieveVectors>)]
+    retrieve_vectors: bool,
     #[deserr(default, error = DeserrJsonError<InvalidDocumentFilter>)]
     filter: Option<Value>,
 }
@@ -185,6 +200,7 @@ pub async fn documents_by_query_post(
             with_filter: body.filter.is_some(),
             limit: body.limit,
             offset: body.offset,
+            retrieve_vectors: body.retrieve_vectors,
         },
         &req,
     );
@@ -201,7 +217,7 @@ pub async fn get_documents(
 ) -> Result<HttpResponse, ResponseError> {
     debug!(parameters = ?params, "Get documents GET");
 
-    let BrowseQueryGet { limit, offset, fields, filter } = params.into_inner();
+    let BrowseQueryGet { limit, offset, fields, retrieve_vectors, filter } = params.into_inner();
 
     let filter = match filter {
         Some(f) => match serde_json::from_str(&f) {
@@ -215,6 +231,7 @@ pub async fn get_documents(
         offset: offset.0,
         limit: limit.0,
         fields: fields.merge_star_and_none(),
+        retrieve_vectors: retrieve_vectors.0,
         filter,
     };
 
@@ -223,6 +240,7 @@ pub async fn get_documents(
             with_filter: query.filter.is_some(),
             limit: query.limit,
             offset: query.offset,
+            retrieve_vectors: query.retrieve_vectors,
         },
         &req,
     );
@@ -236,10 +254,14 @@ fn documents_by_query(
     query: BrowseQuery,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
-    let BrowseQuery { offset, limit, fields, filter } = query;
+    let BrowseQuery { offset, limit, fields, retrieve_vectors, filter } = query;
+
+    let features = index_scheduler.features();
+    let retrieve_vectors = RetrieveVectors::new(retrieve_vectors, features)?;
 
     let index = index_scheduler.index(&index_uid)?;
-    let (total, documents) = retrieve_documents(&index, offset, limit, filter, fields)?;
+    let (total, documents) =
+        retrieve_documents(&index, offset, limit, filter, fields, retrieve_vectors)?;
 
     let ret = PaginationView::new(offset, limit, total as usize, documents);
 
@@ -553,6 +575,82 @@ pub async fn delete_documents_by_filter(
     Ok(HttpResponse::Accepted().json(task))
 }
 
+#[derive(Debug, Deserr)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
+pub struct DocumentEditionByFunction {
+    #[deserr(default, error = DeserrJsonError<InvalidDocumentFilter>)]
+    pub filter: Option<Value>,
+    #[deserr(default, error = DeserrJsonError<InvalidDocumentEditionContext>)]
+    pub context: Option<Value>,
+    #[deserr(error = DeserrJsonError<InvalidDocumentEditionFunctionFilter>, missing_field_error = DeserrJsonError::missing_document_edition_function)]
+    pub function: String,
+}
+
+pub async fn edit_documents_by_function(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::DOCUMENTS_ALL }>, Data<IndexScheduler>>,
+    index_uid: web::Path<String>,
+    params: AwebJson<DocumentEditionByFunction, DeserrJsonError>,
+    req: HttpRequest,
+    opt: web::Data<Opt>,
+    analytics: web::Data<dyn Analytics>,
+) -> Result<HttpResponse, ResponseError> {
+    debug!(parameters = ?params, "Edit documents by function");
+
+    index_scheduler
+        .features()
+        .check_edit_documents_by_function("Using the documents edit route")?;
+
+    let index_uid = IndexUid::try_from(index_uid.into_inner())?;
+    let index_uid = index_uid.into_inner();
+    let params = params.into_inner();
+
+    analytics.update_documents_by_function(
+        &params,
+        index_scheduler.index(&index_uid).is_err(),
+        &req,
+    );
+
+    let DocumentEditionByFunction { filter, context, function } = params;
+    let engine = milli::rhai::Engine::new();
+    if let Err(e) = engine.compile(&function) {
+        return Err(ResponseError::from_msg(e.to_string(), Code::BadRequest));
+    }
+
+    if let Some(ref filter) = filter {
+        // we ensure the filter is well formed before enqueuing it
+        || -> Result<_, ResponseError> {
+            Ok(crate::search::parse_filter(filter)?.ok_or(MeilisearchHttpError::EmptyFilter)?)
+        }()
+        // and whatever was the error, the error code should always be an InvalidDocumentFilter
+        .map_err(|err| ResponseError::from_msg(err.message, Code::InvalidDocumentFilter))?;
+    }
+    let task = KindWithContent::DocumentEdition {
+        index_uid,
+        filter_expr: filter,
+        context: match context {
+            Some(Value::Object(m)) => Some(m),
+            None => None,
+            _ => {
+                return Err(ResponseError::from_msg(
+                    "The context must be an object".to_string(),
+                    Code::InvalidDocumentEditionContext,
+                ))
+            }
+        },
+        function,
+    };
+
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
+    let task: SummarizedTaskView =
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
+
+    debug!(returns = ?task, "Edit documents by function");
+    Ok(HttpResponse::Accepted().json(task))
+}
+
 pub async fn clear_all_documents(
     index_scheduler: GuardedData<ActionPolicy<{ actions::DOCUMENTS_DELETE }>, Data<IndexScheduler>>,
     index_uid: web::Path<String>,
@@ -579,13 +677,46 @@ fn some_documents<'a, 't: 'a>(
     index: &'a Index,
     rtxn: &'t RoTxn,
     doc_ids: impl IntoIterator<Item = DocumentId> + 'a,
+    retrieve_vectors: RetrieveVectors,
 ) -> Result<impl Iterator<Item = Result<Document, ResponseError>> + 'a, ResponseError> {
     let fields_ids_map = index.fields_ids_map(rtxn)?;
     let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
+    let embedding_configs = index.embedding_configs(rtxn)?;
 
     Ok(index.iter_documents(rtxn, doc_ids)?.map(move |ret| {
-        ret.map_err(ResponseError::from).and_then(|(_key, document)| -> Result<_, ResponseError> {
-            Ok(milli::obkv_to_json(&all_fields, &fields_ids_map, document)?)
+        ret.map_err(ResponseError::from).and_then(|(key, document)| -> Result<_, ResponseError> {
+            let mut document = milli::obkv_to_json(&all_fields, &fields_ids_map, document)?;
+            match retrieve_vectors {
+                RetrieveVectors::Ignore => {}
+                RetrieveVectors::Hide => {
+                    document.remove("_vectors");
+                }
+                RetrieveVectors::Retrieve => {
+                    // Clippy is simply wrong
+                    #[allow(clippy::manual_unwrap_or_default)]
+                    let mut vectors = match document.remove("_vectors") {
+                        Some(Value::Object(map)) => map,
+                        _ => Default::default(),
+                    };
+                    for (name, vector) in index.embeddings(rtxn, key)? {
+                        let user_provided = embedding_configs
+                            .iter()
+                            .find(|conf| conf.name == name)
+                            .is_some_and(|conf| conf.user_provided.contains(key));
+                        let embeddings = ExplicitVectors {
+                            embeddings: Some(vector.into()),
+                            regenerate: !user_provided,
+                        };
+                        vectors.insert(
+                            name,
+                            serde_json::to_value(embeddings).map_err(MeilisearchHttpError::from)?,
+                        );
+                    }
+                    document.insert("_vectors".into(), vectors.into());
+                }
+            }
+
+            Ok(document)
         })
     }))
 }
@@ -596,6 +727,7 @@ fn retrieve_documents<S: AsRef<str>>(
     limit: usize,
     filter: Option<Value>,
     attributes_to_retrieve: Option<Vec<S>>,
+    retrieve_vectors: RetrieveVectors,
 ) -> Result<(u64, Vec<Document>), ResponseError> {
     let rtxn = index.read_txn()?;
     let filter = &filter;
@@ -620,53 +752,57 @@ fn retrieve_documents<S: AsRef<str>>(
     let (it, number_of_documents) = {
         let number_of_documents = candidates.len();
         (
-            some_documents(index, &rtxn, candidates.into_iter().skip(offset).take(limit))?,
+            some_documents(
+                index,
+                &rtxn,
+                candidates.into_iter().skip(offset).take(limit),
+                retrieve_vectors,
+            )?,
             number_of_documents,
         )
     };
 
-    let documents: Result<Vec<_>, ResponseError> = it
+    let documents: Vec<_> = it
         .map(|document| {
             Ok(match &attributes_to_retrieve {
                 Some(attributes_to_retrieve) => permissive_json_pointer::select_values(
                     &document?,
-                    attributes_to_retrieve.iter().map(|s| s.as_ref()),
+                    attributes_to_retrieve.iter().map(|s| s.as_ref()).chain(
+                        (retrieve_vectors == RetrieveVectors::Retrieve).then_some("_vectors"),
+                    ),
                 ),
                 None => document?,
             })
         })
-        .collect();
+        .collect::<Result<_, ResponseError>>()?;
 
-    Ok((number_of_documents, documents?))
+    Ok((number_of_documents, documents))
 }
 
 fn retrieve_document<S: AsRef<str>>(
     index: &Index,
     doc_id: &str,
     attributes_to_retrieve: Option<Vec<S>>,
+    retrieve_vectors: RetrieveVectors,
 ) -> Result<Document, ResponseError> {
     let txn = index.read_txn()?;
 
-    let fields_ids_map = index.fields_ids_map(&txn)?;
-    let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
-
     let internal_id = index
         .external_documents_ids()
         .get(&txn, doc_id)?
         .ok_or_else(|| MeilisearchHttpError::DocumentNotFound(doc_id.to_string()))?;
 
-    let document = index
-        .documents(&txn, std::iter::once(internal_id))?
-        .into_iter()
+    let document = some_documents(index, &txn, Some(internal_id), retrieve_vectors)?
         .next()
-        .map(|(_, d)| d)
-        .ok_or_else(|| MeilisearchHttpError::DocumentNotFound(doc_id.to_string()))?;
+        .ok_or_else(|| MeilisearchHttpError::DocumentNotFound(doc_id.to_string()))??;
 
-    let document = meilisearch_types::milli::obkv_to_json(&all_fields, &fields_ids_map, document)?;
     let document = match &attributes_to_retrieve {
         Some(attributes_to_retrieve) => permissive_json_pointer::select_values(
             &document,
-            attributes_to_retrieve.iter().map(|s| s.as_ref()),
+            attributes_to_retrieve
+                .iter()
+                .map(|s| s.as_ref())
+                .chain((retrieve_vectors == RetrieveVectors::Retrieve).then_some("_vectors")),
         ),
         None => document,
     };

meilisearch/src/routes/indexes/facet_search.rs

@@ -115,6 +115,7 @@ impl From<FacetSearchQuery> for SearchQuery {
             page: None,
             hits_per_page: None,
             attributes_to_retrieve: None,
+            retrieve_vectors: false,
             attributes_to_crop: None,
             crop_length: DEFAULT_CROP_LENGTH(),
             attributes_to_highlight: None,
@@ -123,6 +124,7 @@ impl From<FacetSearchQuery> for SearchQuery {
             show_ranking_score_details: false,
             filter,
             sort: None,
+            distinct: None,
             facets: None,
             highlight_pre_tag: DEFAULT_HIGHLIGHT_PRE_TAG(),
             highlight_post_tag: DEFAULT_HIGHLIGHT_POST_TAG(),

meilisearch/src/routes/indexes/search.rs

@@ -20,9 +20,9 @@ use crate::extractors::sequential_extractor::SeqHandler;
 use crate::metrics::MEILISEARCH_DEGRADED_SEARCH_REQUESTS;
 use crate::search::{
     add_search_rules, perform_search, HybridQuery, MatchingStrategy, RankingScoreThreshold,
-    SearchKind, SearchQuery, SemanticRatio, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
-    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
-    DEFAULT_SEARCH_OFFSET, DEFAULT_SEMANTIC_RATIO,
+    RetrieveVectors, SearchKind, SearchQuery, SemanticRatio, DEFAULT_CROP_LENGTH,
+    DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG,
+    DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET, DEFAULT_SEMANTIC_RATIO,
 };
 use crate::search_queue::SearchQueue;
 
@@ -51,6 +51,8 @@ pub struct SearchQueryGet {
     hits_per_page: Option<Param<usize>>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchAttributesToRetrieve>)]
     attributes_to_retrieve: Option<CS<String>>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSearchRetrieveVectors>)]
+    retrieve_vectors: Param<bool>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchAttributesToCrop>)]
     attributes_to_crop: Option<CS<String>>,
     #[deserr(default = Param(DEFAULT_CROP_LENGTH()), error = DeserrQueryParamError<InvalidSearchCropLength>)]
@@ -61,6 +63,8 @@ pub struct SearchQueryGet {
     filter: Option<String>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchSort>)]
     sort: Option<String>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSearchDistinct>)]
+    distinct: Option<String>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchShowMatchesPosition>)]
     show_matches_position: Param<bool>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchShowRankingScore>)]
@@ -153,11 +157,13 @@ impl From<SearchQueryGet> for SearchQuery {
             page: other.page.as_deref().copied(),
             hits_per_page: other.hits_per_page.as_deref().copied(),
             attributes_to_retrieve: other.attributes_to_retrieve.map(|o| o.into_iter().collect()),
+            retrieve_vectors: other.retrieve_vectors.0,
             attributes_to_crop: other.attributes_to_crop.map(|o| o.into_iter().collect()),
             crop_length: other.crop_length.0,
             attributes_to_highlight: other.attributes_to_highlight.map(|o| o.into_iter().collect()),
             filter,
             sort: other.sort.map(|attr| fix_sort_query_parameters(&attr)),
+            distinct: other.distinct,
             show_matches_position: other.show_matches_position.0,
             show_ranking_score: other.show_ranking_score.0,
             show_ranking_score_details: other.show_ranking_score_details.0,
@@ -222,10 +228,12 @@ pub async fn search_with_url_query(
     let features = index_scheduler.features();
 
     let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)?;
-
+    let retrieve_vector = RetrieveVectors::new(query.retrieve_vectors, features)?;
     let _permit = search_queue.try_get_search_permit().await?;
-    let search_result =
-        tokio::task::spawn_blocking(move || perform_search(&index, query, search_kind)).await?;
+    let search_result = tokio::task::spawn_blocking(move || {
+        perform_search(&index, query, search_kind, retrieve_vector)
+    })
+    .await?;
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
     }
@@ -262,10 +270,13 @@ pub async fn search_with_post(
     let features = index_scheduler.features();
 
     let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)?;
+    let retrieve_vectors = RetrieveVectors::new(query.retrieve_vectors, features)?;
 
     let _permit = search_queue.try_get_search_permit().await?;
-    let search_result =
-        tokio::task::spawn_blocking(move || perform_search(&index, query, search_kind)).await?;
+    let search_result = tokio::task::spawn_blocking(move || {
+        perform_search(&index, query, search_kind, retrieve_vectors)
+    })
+    .await?;
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
         if search_result.degraded {
@@ -287,11 +298,10 @@ pub fn search_kind(
     features: RoFeatures,
 ) -> Result<SearchKind, ResponseError> {
     if query.vector.is_some() {
-        features.check_vector("Passing `vector` as a query parameter")?;
+        features.check_vector("Passing `vector` as a parameter")?;
     }
-
     if query.hybrid.is_some() {
-        features.check_vector("Passing `hybrid` as a query parameter")?;
+        features.check_vector("Passing `hybrid` as a parameter")?;
     }
 
     // regardless of anything, always do a keyword search when we don't have a vector and the query is whitespace or missing

meilisearch/src/routes/indexes/similar.rs

@@ -4,11 +4,7 @@ use deserr::actix_web::{AwebJson, AwebQueryParameter};
 use index_scheduler::IndexScheduler;
 use meilisearch_types::deserr::query_params::Param;
 use meilisearch_types::deserr::{DeserrJsonError, DeserrQueryParamError};
-use meilisearch_types::error::deserr_codes::{
-    InvalidEmbedder, InvalidSimilarAttributesToRetrieve, InvalidSimilarFilter, InvalidSimilarId,
-    InvalidSimilarLimit, InvalidSimilarOffset, InvalidSimilarRankingScoreThreshold,
-    InvalidSimilarShowRankingScore, InvalidSimilarShowRankingScoreDetails,
-};
+use meilisearch_types::error::deserr_codes::*;
 use meilisearch_types::error::{ErrorCode as _, ResponseError};
 use meilisearch_types::index_uid::IndexUid;
 use meilisearch_types::keys::actions;
@@ -21,8 +17,8 @@ use crate::analytics::{Analytics, SimilarAggregator};
 use crate::extractors::authentication::GuardedData;
 use crate::extractors::sequential_extractor::SeqHandler;
 use crate::search::{
-    add_search_rules, perform_similar, RankingScoreThresholdSimilar, SearchKind, SimilarQuery,
-    SimilarResult, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET,
+    add_search_rules, perform_similar, RankingScoreThresholdSimilar, RetrieveVectors, SearchKind,
+    SimilarQuery, SimilarResult, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET,
 };
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
@@ -97,6 +93,8 @@ async fn similar(
 
     features.check_vector("Using the similar API")?;
 
+    let retrieve_vectors = RetrieveVectors::new(query.retrieve_vectors, features)?;
+
     // Tenant token search_rules.
     if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(&index_uid) {
         add_search_rules(&mut query.filter, search_rules);
@@ -107,8 +105,10 @@ async fn similar(
     let (embedder_name, embedder) =
         SearchKind::embedder(&index_scheduler, &index, query.embedder.as_deref(), None)?;
 
-    tokio::task::spawn_blocking(move || perform_similar(&index, query, embedder_name, embedder))
-        .await?
+    tokio::task::spawn_blocking(move || {
+        perform_similar(&index, query, embedder_name, embedder, retrieve_vectors)
+    })
+    .await?
 }
 
 #[derive(Debug, deserr::Deserr)]
@@ -122,6 +122,8 @@ pub struct SimilarQueryGet {
     limit: Param<usize>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSimilarAttributesToRetrieve>)]
     attributes_to_retrieve: Option<CS<String>>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSimilarRetrieveVectors>)]
+    retrieve_vectors: Param<bool>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSimilarFilter>)]
     filter: Option<String>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSimilarShowRankingScore>)]
@@ -156,6 +158,7 @@ impl TryFrom<SimilarQueryGet> for SimilarQuery {
             offset,
             limit,
             attributes_to_retrieve,
+            retrieve_vectors,
             filter,
             show_ranking_score,
             show_ranking_score_details,
@@ -180,6 +183,7 @@ impl TryFrom<SimilarQueryGet> for SimilarQuery {
             filter,
             embedder,
             attributes_to_retrieve: attributes_to_retrieve.map(|o| o.into_iter().collect()),
+            retrieve_vectors: retrieve_vectors.0,
             show_ranking_score: show_ranking_score.0,
             show_ranking_score_details: show_ranking_score_details.0,
             ranking_score_threshold: ranking_score_threshold.map(|x| x.0),

meilisearch/src/routes/multi_search.rs

@@ -10,12 +10,14 @@ use serde::Serialize;
 use tracing::debug;
 
 use crate::analytics::{Analytics, MultiSearchAggregator};
+use crate::error::MeilisearchHttpError;
 use crate::extractors::authentication::policies::ActionPolicy;
 use crate::extractors::authentication::{AuthenticationError, GuardedData};
 use crate::extractors::sequential_extractor::SeqHandler;
 use crate::routes::indexes::search::search_kind;
 use crate::search::{
-    add_search_rules, perform_search, SearchQueryWithIndex, SearchResultWithIndex,
+    add_search_rules, perform_federated_search, perform_search, FederatedSearch, RetrieveVectors,
+    SearchQueryWithIndex, SearchResultWithIndex,
 };
 use crate::search_queue::SearchQueue;
 
@@ -28,82 +30,44 @@ struct SearchResults {
     results: Vec<SearchResultWithIndex>,
 }
 
-#[derive(Debug, deserr::Deserr)]
-#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
-pub struct SearchQueries {
-    queries: Vec<SearchQueryWithIndex>,
-}
-
 pub async fn multi_search_with_post(
     index_scheduler: GuardedData<ActionPolicy<{ actions::SEARCH }>, Data<IndexScheduler>>,
     search_queue: Data<SearchQueue>,
-    params: AwebJson<SearchQueries, DeserrJsonError>,
+    params: AwebJson<FederatedSearch, DeserrJsonError>,
     req: HttpRequest,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    let queries = params.into_inner().queries;
-
-    let mut multi_aggregate = MultiSearchAggregator::from_queries(&queries, &req);
-    let features = index_scheduler.features();
-
     // Since we don't want to process half of the search requests and then get a permit refused
     // we're going to get one permit for the whole duration of the multi-search request.
     let _permit = search_queue.try_get_search_permit().await?;
 
-    // Explicitly expect a `(ResponseError, usize)` for the error type rather than `ResponseError` only,
-    // so that `?` doesn't work if it doesn't use `with_index`, ensuring that it is not forgotten in case of code
-    // changes.
-    let search_results: Result<_, (ResponseError, usize)> = async {
-        let mut search_results = Vec::with_capacity(queries.len());
-        for (query_index, (index_uid, mut query)) in
-            queries.into_iter().map(SearchQueryWithIndex::into_index_query).enumerate()
-        {
-            debug!(on_index = query_index, parameters = ?query, "Multi-search");
+    let federated_search = params.into_inner();
 
+    let mut multi_aggregate = MultiSearchAggregator::from_federated_search(&federated_search, &req);
+
+    let FederatedSearch { mut queries, federation } = federated_search;
+
+    let features = index_scheduler.features();
+
+    // regardless of federation, check authorization and apply search rules
+    let auth = 'check_authorization: {
+        for (query_index, federated_query) in queries.iter_mut().enumerate() {
+            let index_uid = federated_query.index_uid.as_str();
             // Check index from API key
-            if !index_scheduler.filters().is_index_authorized(&index_uid) {
-                return Err(AuthenticationError::InvalidToken).with_index(query_index);
+            if !index_scheduler.filters().is_index_authorized(index_uid) {
+                break 'check_authorization Err(AuthenticationError::InvalidToken)
+                    .with_index(query_index);
             }
             // Apply search rules from tenant token
-            if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(&index_uid)
+            if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(index_uid)
             {
-                add_search_rules(&mut query.filter, search_rules);
+                add_search_rules(&mut federated_query.filter, search_rules);
             }
-
-            let index = index_scheduler
-                .index(&index_uid)
-                .map_err(|err| {
-                    let mut err = ResponseError::from(err);
-                    // Patch the HTTP status code to 400 as it defaults to 404 for `index_not_found`, but
-                    // here the resource not found is not part of the URL.
-                    err.code = StatusCode::BAD_REQUEST;
-                    err
-                })
-                .with_index(query_index)?;
-
-            let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)
-                .with_index(query_index)?;
-
-            let search_result =
-                tokio::task::spawn_blocking(move || perform_search(&index, query, search_kind))
-                    .await
-                    .with_index(query_index)?;
-
-            search_results.push(SearchResultWithIndex {
-                index_uid: index_uid.into_inner(),
-                result: search_result.with_index(query_index)?,
-            });
         }
-        Ok(search_results)
-    }
-    .await;
+        Ok(())
+    };
 
-    if search_results.is_ok() {
-        multi_aggregate.succeed();
-    }
-    analytics.post_multi_search(multi_aggregate);
-
-    let search_results = search_results.map_err(|(mut err, query_index)| {
+    auth.map_err(|(mut err, query_index)| {
         // Add the query index that failed as context for the error message.
         // We're doing it only here and not directly in the `WithIndex` trait so that the `with_index` function returns a different type
         // of result and we can benefit from static typing.
@@ -111,9 +75,95 @@ pub async fn multi_search_with_post(
         err
     })?;
 
-    debug!(returns = ?search_results, "Multi-search");
+    let response = match federation {
+        Some(federation) => {
+            let search_result = tokio::task::spawn_blocking(move || {
+                perform_federated_search(&index_scheduler, queries, federation, features)
+            })
+            .await;
 
-    Ok(HttpResponse::Ok().json(SearchResults { results: search_results }))
+            if let Ok(Ok(_)) = search_result {
+                multi_aggregate.succeed();
+            }
+
+            analytics.post_multi_search(multi_aggregate);
+            HttpResponse::Ok().json(search_result??)
+        }
+        None => {
+            // Explicitly expect a `(ResponseError, usize)` for the error type rather than `ResponseError` only,
+            // so that `?` doesn't work if it doesn't use `with_index`, ensuring that it is not forgotten in case of code
+            // changes.
+            let search_results: Result<_, (ResponseError, usize)> = async {
+                let mut search_results = Vec::with_capacity(queries.len());
+                for (query_index, (index_uid, query, federation_options)) in queries
+                    .into_iter()
+                    .map(SearchQueryWithIndex::into_index_query_federation)
+                    .enumerate()
+                {
+                    debug!(on_index = query_index, parameters = ?query, "Multi-search");
+
+                    if federation_options.is_some() {
+                        return Err((
+                            MeilisearchHttpError::FederationOptionsInNonFederatedRequest(
+                                query_index,
+                            )
+                            .into(),
+                            query_index,
+                        ));
+                    }
+
+                    let index = index_scheduler
+                        .index(&index_uid)
+                        .map_err(|err| {
+                            let mut err = ResponseError::from(err);
+                            // Patch the HTTP status code to 400 as it defaults to 404 for `index_not_found`, but
+                            // here the resource not found is not part of the URL.
+                            err.code = StatusCode::BAD_REQUEST;
+                            err
+                        })
+                        .with_index(query_index)?;
+
+                    let search_kind =
+                        search_kind(&query, index_scheduler.get_ref(), &index, features)
+                            .with_index(query_index)?;
+                    let retrieve_vector = RetrieveVectors::new(query.retrieve_vectors, features)
+                        .with_index(query_index)?;
+
+                    let search_result = tokio::task::spawn_blocking(move || {
+                        perform_search(&index, query, search_kind, retrieve_vector)
+                    })
+                    .await
+                    .with_index(query_index)?;
+
+                    search_results.push(SearchResultWithIndex {
+                        index_uid: index_uid.into_inner(),
+                        result: search_result.with_index(query_index)?,
+                    });
+                }
+                Ok(search_results)
+            }
+            .await;
+
+            if search_results.is_ok() {
+                multi_aggregate.succeed();
+            }
+            analytics.post_multi_search(multi_aggregate);
+
+            let search_results = search_results.map_err(|(mut err, query_index)| {
+                // Add the query index that failed as context for the error message.
+                // We're doing it only here and not directly in the `WithIndex` trait so that the `with_index` function returns a different type
+                // of result and we can benefit from static typing.
+                err.message = format!("Inside `.queries[{query_index}]`: {}", err.message);
+                err
+            })?;
+
+            debug!(returns = ?search_results, "Multi-search");
+
+            HttpResponse::Ok().json(SearchResults { results: search_results })
+        }
+    };
+
+    Ok(response)
 }
 
 /// Local `Result` extension trait to avoid `map_err` boilerplate.

meilisearch/src/routes/tasks.rs

@@ -591,7 +591,7 @@ mod tests {
             let err = deserr_query_params::<TaskDeletionOrCancelationQuery>(params).unwrap_err();
             snapshot!(meili_snap::json_string!(err), @r###"
             {
-              "message": "Invalid value in parameter `types`: `createIndex` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.",
+              "message": "Invalid value in parameter `types`: `createIndex` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.",
               "code": "invalid_task_types",
               "type": "invalid_request",
               "link": "https://docs.meilisearch.com/errors#invalid_task_types"

meilisearch/src/search/federated.rs

@@ -0,0 +1,624 @@
+use std::cmp::Ordering;
+use std::collections::BTreeMap;
+use std::fmt;
+use std::iter::Zip;
+use std::rc::Rc;
+use std::str::FromStr as _;
+use std::time::Duration;
+use std::vec::{IntoIter, Vec};
+
+use actix_http::StatusCode;
+use index_scheduler::{IndexScheduler, RoFeatures};
+use meilisearch_types::deserr::DeserrJsonError;
+use meilisearch_types::error::deserr_codes::{
+    InvalidMultiSearchWeight, InvalidSearchLimit, InvalidSearchOffset,
+};
+use meilisearch_types::error::ResponseError;
+use meilisearch_types::milli::score_details::{ScoreDetails, ScoreValue};
+use meilisearch_types::milli::{self, DocumentId, TimeBudget};
+use roaring::RoaringBitmap;
+use serde::Serialize;
+
+use super::ranking_rules::{self, RankingRules};
+use super::{
+    prepare_search, AttributesFormat, HitMaker, HitsInfo, RetrieveVectors, SearchHit, SearchKind,
+    SearchQuery, SearchQueryWithIndex,
+};
+use crate::error::MeilisearchHttpError;
+use crate::routes::indexes::search::search_kind;
+
+pub const DEFAULT_FEDERATED_WEIGHT: fn() -> f64 = || 1.0;
+
+#[derive(Debug, Default, Clone, Copy, PartialEq, deserr::Deserr)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
+pub struct FederationOptions {
+    #[deserr(default, error = DeserrJsonError<InvalidMultiSearchWeight>)]
+    pub weight: Weight,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, deserr::Deserr)]
+#[deserr(try_from(f64) = TryFrom::try_from -> InvalidMultiSearchWeight)]
+pub struct Weight(f64);
+
+impl Default for Weight {
+    fn default() -> Self {
+        Weight(DEFAULT_FEDERATED_WEIGHT())
+    }
+}
+
+impl std::convert::TryFrom<f64> for Weight {
+    type Error = InvalidMultiSearchWeight;
+
+    fn try_from(f: f64) -> Result<Self, Self::Error> {
+        // the suggested "fix" is: `!(0.0..=1.0).contains(&f)`` which is allegedly less readable
+        #[allow(clippy::manual_range_contains)]
+        if f < 0.0 {
+            Err(InvalidMultiSearchWeight)
+        } else {
+            Ok(Weight(f))
+        }
+    }
+}
+
+impl std::ops::Deref for Weight {
+    type Target = f64;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+#[derive(Debug, deserr::Deserr)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
+pub struct Federation {
+    #[deserr(default = super::DEFAULT_SEARCH_LIMIT(), error = DeserrJsonError<InvalidSearchLimit>)]
+    pub limit: usize,
+    #[deserr(default = super::DEFAULT_SEARCH_OFFSET(), error = DeserrJsonError<InvalidSearchOffset>)]
+    pub offset: usize,
+}
+
+#[derive(Debug, deserr::Deserr)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
+pub struct FederatedSearch {
+    pub queries: Vec<SearchQueryWithIndex>,
+    #[deserr(default)]
+    pub federation: Option<Federation>,
+}
+#[derive(Serialize, Clone, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct FederatedSearchResult {
+    pub hits: Vec<SearchHit>,
+    pub processing_time_ms: u128,
+    #[serde(flatten)]
+    pub hits_info: HitsInfo,
+
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub semantic_hit_count: Option<u32>,
+
+    // These fields are only used for analytics purposes
+    #[serde(skip)]
+    pub degraded: bool,
+    #[serde(skip)]
+    pub used_negative_operator: bool,
+}
+
+impl fmt::Debug for FederatedSearchResult {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let FederatedSearchResult {
+            hits,
+            processing_time_ms,
+            hits_info,
+            semantic_hit_count,
+            degraded,
+            used_negative_operator,
+        } = self;
+
+        let mut debug = f.debug_struct("SearchResult");
+        // The most important thing when looking at a search result is the time it took to process
+        debug.field("processing_time_ms", &processing_time_ms);
+        debug.field("hits", &format!("[{} hits returned]", hits.len()));
+        debug.field("hits_info", &hits_info);
+        if *used_negative_operator {
+            debug.field("used_negative_operator", used_negative_operator);
+        }
+        if *degraded {
+            debug.field("degraded", degraded);
+        }
+        if let Some(semantic_hit_count) = semantic_hit_count {
+            debug.field("semantic_hit_count", &semantic_hit_count);
+        }
+
+        debug.finish()
+    }
+}
+
+struct WeightedScore<'a> {
+    details: &'a [ScoreDetails],
+    weight: f64,
+}
+
+impl<'a> WeightedScore<'a> {
+    pub fn new(details: &'a [ScoreDetails], weight: f64) -> Self {
+        Self { details, weight }
+    }
+
+    pub fn weighted_global_score(&self) -> f64 {
+        ScoreDetails::global_score(self.details.iter()) * self.weight
+    }
+
+    pub fn compare_weighted_global_scores(&self, other: &Self) -> Ordering {
+        self.weighted_global_score()
+            .partial_cmp(&other.weighted_global_score())
+            // both are numbers, possibly infinite
+            .unwrap()
+    }
+
+    pub fn compare(&self, other: &Self) -> Ordering {
+        let mut left_it = ScoreDetails::score_values(self.details.iter());
+        let mut right_it = ScoreDetails::score_values(other.details.iter());
+
+        loop {
+            let left = left_it.next();
+            let right = right_it.next();
+
+            match (left, right) {
+                (None, None) => return Ordering::Equal,
+                (None, Some(_)) => return Ordering::Less,
+                (Some(_), None) => return Ordering::Greater,
+                (Some(ScoreValue::Score(left)), Some(ScoreValue::Score(right))) => {
+                    let left = left * self.weight;
+                    let right = right * other.weight;
+                    if (left - right).abs() <= f64::EPSILON {
+                        continue;
+                    }
+                    return left.partial_cmp(&right).unwrap();
+                }
+                (Some(ScoreValue::Sort(left)), Some(ScoreValue::Sort(right))) => {
+                    match left.partial_cmp(right) {
+                        Some(Ordering::Equal) => continue,
+                        Some(order) => return order,
+                        None => return self.compare_weighted_global_scores(other),
+                    }
+                }
+                (Some(ScoreValue::GeoSort(left)), Some(ScoreValue::GeoSort(right))) => {
+                    match left.partial_cmp(right) {
+                        Some(Ordering::Equal) => continue,
+                        Some(order) => return order,
+                        None => {
+                            return self.compare_weighted_global_scores(other);
+                        }
+                    }
+                }
+                // not comparable details, use global
+                (Some(ScoreValue::Score(_)), Some(_))
+                | (Some(_), Some(ScoreValue::Score(_)))
+                | (Some(ScoreValue::GeoSort(_)), Some(ScoreValue::Sort(_)))
+                | (Some(ScoreValue::Sort(_)), Some(ScoreValue::GeoSort(_))) => {
+                    return self.compare_weighted_global_scores(other);
+                }
+            }
+        }
+    }
+}
+
+struct QueryByIndex {
+    query: SearchQuery,
+    federation_options: FederationOptions,
+    query_index: usize,
+}
+
+struct SearchResultByQuery<'a> {
+    documents_ids: Vec<DocumentId>,
+    document_scores: Vec<Vec<ScoreDetails>>,
+    federation_options: FederationOptions,
+    hit_maker: HitMaker<'a>,
+    query_index: usize,
+}
+
+struct SearchResultByQueryIter<'a> {
+    it: Zip<IntoIter<DocumentId>, IntoIter<Vec<ScoreDetails>>>,
+    federation_options: FederationOptions,
+    hit_maker: Rc<HitMaker<'a>>,
+    query_index: usize,
+}
+
+impl<'a> SearchResultByQueryIter<'a> {
+    fn new(
+        SearchResultByQuery {
+            documents_ids,
+            document_scores,
+            federation_options,
+            hit_maker,
+            query_index,
+        }: SearchResultByQuery<'a>,
+    ) -> Self {
+        let it = documents_ids.into_iter().zip(document_scores);
+        Self { it, federation_options, hit_maker: Rc::new(hit_maker), query_index }
+    }
+}
+
+struct SearchResultByQueryIterItem<'a> {
+    docid: DocumentId,
+    score: Vec<ScoreDetails>,
+    federation_options: FederationOptions,
+    hit_maker: Rc<HitMaker<'a>>,
+    query_index: usize,
+}
+
+fn merge_index_local_results(
+    results_by_query: Vec<SearchResultByQuery<'_>>,
+) -> impl Iterator<Item = SearchResultByQueryIterItem> + '_ {
+    itertools::kmerge_by(
+        results_by_query.into_iter().map(SearchResultByQueryIter::new),
+        |left: &SearchResultByQueryIterItem, right: &SearchResultByQueryIterItem| {
+            let left_score = WeightedScore::new(&left.score, *left.federation_options.weight);
+            let right_score = WeightedScore::new(&right.score, *right.federation_options.weight);
+
+            match left_score.compare(&right_score) {
+                // the biggest score goes first
+                Ordering::Greater => true,
+                // break ties using query index
+                Ordering::Equal => left.query_index < right.query_index,
+                Ordering::Less => false,
+            }
+        },
+    )
+}
+
+fn merge_index_global_results(
+    results_by_index: Vec<SearchResultByIndex>,
+) -> impl Iterator<Item = SearchHitByIndex> {
+    itertools::kmerge_by(
+        results_by_index.into_iter().map(|result_by_index| result_by_index.hits.into_iter()),
+        |left: &SearchHitByIndex, right: &SearchHitByIndex| {
+            let left_score = WeightedScore::new(&left.score, *left.federation_options.weight);
+            let right_score = WeightedScore::new(&right.score, *right.federation_options.weight);
+
+            match left_score.compare(&right_score) {
+                // the biggest score goes first
+                Ordering::Greater => true,
+                // break ties using query index
+                Ordering::Equal => left.query_index < right.query_index,
+                Ordering::Less => false,
+            }
+        },
+    )
+}
+
+impl<'a> Iterator for SearchResultByQueryIter<'a> {
+    type Item = SearchResultByQueryIterItem<'a>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let (docid, score) = self.it.next()?;
+        Some(SearchResultByQueryIterItem {
+            docid,
+            score,
+            federation_options: self.federation_options,
+            hit_maker: Rc::clone(&self.hit_maker),
+            query_index: self.query_index,
+        })
+    }
+}
+
+struct SearchHitByIndex {
+    hit: SearchHit,
+    score: Vec<ScoreDetails>,
+    federation_options: FederationOptions,
+    query_index: usize,
+}
+
+struct SearchResultByIndex {
+    hits: Vec<SearchHitByIndex>,
+    candidates: RoaringBitmap,
+    degraded: bool,
+    used_negative_operator: bool,
+}
+
+pub fn perform_federated_search(
+    index_scheduler: &IndexScheduler,
+    queries: Vec<SearchQueryWithIndex>,
+    federation: Federation,
+    features: RoFeatures,
+) -> Result<FederatedSearchResult, ResponseError> {
+    let before_search = std::time::Instant::now();
+
+    // this implementation partition the queries by index to guarantee an important property:
+    // - all the queries to a particular index use the same read transaction.
+    // This is an important property, otherwise we cannot guarantee the self-consistency of the results.
+
+    // 1. partition queries by index
+    let mut queries_by_index: BTreeMap<String, Vec<QueryByIndex>> = Default::default();
+    for (query_index, federated_query) in queries.into_iter().enumerate() {
+        if let Some(pagination_field) = federated_query.has_pagination() {
+            return Err(MeilisearchHttpError::PaginationInFederatedQuery(
+                query_index,
+                pagination_field,
+            )
+            .into());
+        }
+
+        let (index_uid, query, federation_options) = federated_query.into_index_query_federation();
+
+        queries_by_index.entry(index_uid.into_inner()).or_default().push(QueryByIndex {
+            query,
+            federation_options: federation_options.unwrap_or_default(),
+            query_index,
+        })
+    }
+
+    // 2. perform queries, merge and make hits index by index
+    let required_hit_count = federation.limit + federation.offset;
+    // In step (2), semantic_hit_count will be set to Some(0) if any search kind uses semantic
+    // Then in step (3), we'll update its value if there is any semantic search
+    let mut semantic_hit_count = None;
+    let mut results_by_index = Vec::with_capacity(queries_by_index.len());
+    let mut previous_query_data: Option<(RankingRules, usize, String)> = None;
+
+    for (index_uid, queries) in queries_by_index {
+        let index = match index_scheduler.index(&index_uid) {
+            Ok(index) => index,
+            Err(err) => {
+                let mut err = ResponseError::from(err);
+                // Patch the HTTP status code to 400 as it defaults to 404 for `index_not_found`, but
+                // here the resource not found is not part of the URL.
+                err.code = StatusCode::BAD_REQUEST;
+                if let Some(query) = queries.first() {
+                    err.message =
+                        format!("Inside `.queries[{}]`: {}", query.query_index, err.message);
+                }
+                return Err(err);
+            }
+        };
+
+        // Important: this is the only transaction we'll use for this index during this federated search
+        let rtxn = index.read_txn()?;
+
+        let criteria = index.criteria(&rtxn)?;
+
+        // stuff we need for the hitmaker
+        let script_lang_map = index.script_language(&rtxn)?;
+
+        let dictionary = index.dictionary(&rtxn)?;
+        let dictionary: Option<Vec<_>> =
+            dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
+        let separators = index.allowed_separators(&rtxn)?;
+        let separators: Option<Vec<_>> =
+            separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
+
+        // each query gets its individual cutoff
+        let cutoff = index.search_cutoff(&rtxn)?;
+
+        let mut degraded = false;
+        let mut used_negative_operator = false;
+        let mut candidates = RoaringBitmap::new();
+
+        // 2.1. Compute all candidates for each query in the index
+        let mut results_by_query = Vec::with_capacity(queries.len());
+
+        for QueryByIndex { query, federation_options, query_index } in queries {
+            // use an immediately invoked lambda to capture the result without returning from the function
+
+            let res: Result<(), ResponseError> = (|| {
+                let search_kind = search_kind(&query, index_scheduler, &index, features)?;
+
+                let canonicalization_kind = match (&search_kind, &query.q) {
+                    (SearchKind::SemanticOnly { .. }, _) => {
+                        ranking_rules::CanonicalizationKind::Vector
+                    }
+                    (_, Some(q)) if !q.is_empty() => ranking_rules::CanonicalizationKind::Keyword,
+                    _ => ranking_rules::CanonicalizationKind::Placeholder,
+                };
+
+                let sort = if let Some(sort) = &query.sort {
+                    let sorts: Vec<_> =
+                        match sort.iter().map(|s| milli::AscDesc::from_str(s)).collect() {
+                            Ok(sorts) => sorts,
+                            Err(asc_desc_error) => {
+                                return Err(milli::Error::from(milli::SortError::from(
+                                    asc_desc_error,
+                                ))
+                                .into())
+                            }
+                        };
+                    Some(sorts)
+                } else {
+                    None
+                };
+
+                let ranking_rules = ranking_rules::RankingRules::new(
+                    criteria.clone(),
+                    sort,
+                    query.matching_strategy.into(),
+                    canonicalization_kind,
+                );
+
+                if let Some((previous_ranking_rules, previous_query_index, previous_index_uid)) =
+                    previous_query_data.take()
+                {
+                    if let Err(error) = ranking_rules.is_compatible_with(&previous_ranking_rules) {
+                        return Err(error.to_response_error(
+                            &ranking_rules,
+                            &previous_ranking_rules,
+                            query_index,
+                            previous_query_index,
+                            &index_uid,
+                            &previous_index_uid,
+                        ));
+                    }
+                    previous_query_data = if previous_ranking_rules.constraint_count()
+                        > ranking_rules.constraint_count()
+                    {
+                        Some((previous_ranking_rules, previous_query_index, previous_index_uid))
+                    } else {
+                        Some((ranking_rules, query_index, index_uid.clone()))
+                    };
+                } else {
+                    previous_query_data = Some((ranking_rules, query_index, index_uid.clone()));
+                }
+
+                match search_kind {
+                    SearchKind::KeywordOnly => {}
+                    _ => semantic_hit_count = Some(0),
+                }
+
+                let retrieve_vectors = RetrieveVectors::new(query.retrieve_vectors, features)?;
+
+                let time_budget = match cutoff {
+                    Some(cutoff) => TimeBudget::new(Duration::from_millis(cutoff)),
+                    None => TimeBudget::default(),
+                };
+
+                let (mut search, _is_finite_pagination, _max_total_hits, _offset) =
+                    prepare_search(&index, &rtxn, &query, &search_kind, time_budget)?;
+
+                search.scoring_strategy(milli::score_details::ScoringStrategy::Detailed);
+                search.offset(0);
+                search.limit(required_hit_count);
+
+                let (result, _semantic_hit_count) = super::search_from_kind(search_kind, search)?;
+                let format = AttributesFormat {
+                    attributes_to_retrieve: query.attributes_to_retrieve,
+                    retrieve_vectors,
+                    attributes_to_highlight: query.attributes_to_highlight,
+                    attributes_to_crop: query.attributes_to_crop,
+                    crop_length: query.crop_length,
+                    crop_marker: query.crop_marker,
+                    highlight_pre_tag: query.highlight_pre_tag,
+                    highlight_post_tag: query.highlight_post_tag,
+                    show_matches_position: query.show_matches_position,
+                    sort: query.sort,
+                    show_ranking_score: query.show_ranking_score,
+                    show_ranking_score_details: query.show_ranking_score_details,
+                };
+
+                let milli::SearchResult {
+                    matching_words,
+                    candidates: query_candidates,
+                    documents_ids,
+                    document_scores,
+                    degraded: query_degraded,
+                    used_negative_operator: query_used_negative_operator,
+                } = result;
+
+                candidates |= query_candidates;
+                degraded |= query_degraded;
+                used_negative_operator |= query_used_negative_operator;
+
+                let tokenizer = HitMaker::tokenizer(
+                    &script_lang_map,
+                    dictionary.as_deref(),
+                    separators.as_deref(),
+                );
+
+                let formatter_builder = HitMaker::formatter_builder(matching_words, tokenizer);
+
+                let hit_maker = HitMaker::new(&index, &rtxn, format, formatter_builder)?;
+
+                results_by_query.push(SearchResultByQuery {
+                    federation_options,
+                    hit_maker,
+                    query_index,
+                    documents_ids,
+                    document_scores,
+                });
+                Ok(())
+            })();
+
+            if let Err(mut error) = res {
+                error.message = format!("Inside `.queries[{query_index}]`: {}", error.message);
+                return Err(error);
+            }
+        }
+        // 2.2. merge inside index
+        let mut documents_seen = RoaringBitmap::new();
+        let merged_result: Result<Vec<_>, ResponseError> =
+            merge_index_local_results(results_by_query)
+                // skip documents we've already seen & mark that we saw the current document
+                .filter(|SearchResultByQueryIterItem { docid, .. }| documents_seen.insert(*docid))
+                .take(required_hit_count)
+                // 2.3 make hits
+                .map(
+                    |SearchResultByQueryIterItem {
+                         docid,
+                         score,
+                         federation_options,
+                         hit_maker,
+                         query_index,
+                     }| {
+                        let mut hit = hit_maker.make_hit(docid, &score)?;
+                        let weighted_score =
+                            ScoreDetails::global_score(score.iter()) * (*federation_options.weight);
+
+                        let _federation = serde_json::json!(
+                            {
+                                "indexUid": index_uid,
+                                "queriesPosition": query_index,
+                                "weightedRankingScore": weighted_score,
+                            }
+                        );
+                        hit.document.insert("_federation".to_string(), _federation);
+                        Ok(SearchHitByIndex { hit, score, federation_options, query_index })
+                    },
+                )
+                .collect();
+
+        let merged_result = merged_result?;
+        results_by_index.push(SearchResultByIndex {
+            hits: merged_result,
+            candidates,
+            degraded,
+            used_negative_operator,
+        });
+    }
+
+    // 3. merge hits and metadata across indexes
+    // 3.1 merge metadata
+    let (estimated_total_hits, degraded, used_negative_operator) = {
+        let mut estimated_total_hits = 0;
+        let mut degraded = false;
+        let mut used_negative_operator = false;
+
+        for SearchResultByIndex {
+            hits: _,
+            candidates,
+            degraded: degraded_by_index,
+            used_negative_operator: used_negative_operator_by_index,
+        } in &results_by_index
+        {
+            estimated_total_hits += candidates.len() as usize;
+            degraded |= *degraded_by_index;
+            used_negative_operator |= *used_negative_operator_by_index;
+        }
+
+        (estimated_total_hits, degraded, used_negative_operator)
+    };
+
+    // 3.2 merge hits
+    let merged_hits: Vec<_> = merge_index_global_results(results_by_index)
+        .skip(federation.offset)
+        .take(federation.limit)
+        .inspect(|hit| {
+            if let Some(semantic_hit_count) = &mut semantic_hit_count {
+                if hit.score.iter().any(|score| matches!(&score, ScoreDetails::Vector(_))) {
+                    *semantic_hit_count += 1;
+                }
+            }
+        })
+        .map(|hit| hit.hit)
+        .collect();
+
+    let search_result = FederatedSearchResult {
+        hits: merged_hits,
+        processing_time_ms: before_search.elapsed().as_millis(),
+        hits_info: HitsInfo::OffsetLimit {
+            limit: federation.limit,
+            offset: federation.offset,
+            estimated_total_hits,
+        },
+        semantic_hit_count,
+        degraded,
+        used_negative_operator,
+    };
+
+    Ok(search_result)
+}

meilisearch/src/search/mod.rs

@@ -1,6 +1,6 @@
 use core::fmt;
 use std::cmp::min;
-use std::collections::{BTreeMap, BTreeSet, HashSet};
+use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
 use std::str::FromStr;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
@@ -15,6 +15,7 @@ use meilisearch_types::error::{Code, ResponseError};
 use meilisearch_types::heed::RoTxn;
 use meilisearch_types::index_uid::IndexUid;
 use meilisearch_types::milli::score_details::{ScoreDetails, ScoringStrategy};
+use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors;
 use meilisearch_types::milli::vector::Embedder;
 use meilisearch_types::milli::{FacetValueHit, OrderBy, SearchForFacetValues, TimeBudget};
 use meilisearch_types::settings::DEFAULT_PAGINATION_MAX_TOTAL_HITS;
@@ -30,6 +31,11 @@ use serde_json::{json, Value};
 
 use crate::error::MeilisearchHttpError;
 
+mod federated;
+pub use federated::{perform_federated_search, FederatedSearch, Federation, FederationOptions};
+
+mod ranking_rules;
+
 type MatchesPosition = BTreeMap<String, Vec<MatchBounds>>;
 
 pub const DEFAULT_SEARCH_OFFSET: fn() -> usize = || 0;
@@ -59,6 +65,8 @@ pub struct SearchQuery {
     pub hits_per_page: Option<usize>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToRetrieve>)]
     pub attributes_to_retrieve: Option<BTreeSet<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchRetrieveVectors>)]
+    pub retrieve_vectors: bool,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToCrop>)]
     pub attributes_to_crop: Option<Vec<String>>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchCropLength>, default = DEFAULT_CROP_LENGTH())]
@@ -75,6 +83,8 @@ pub struct SearchQuery {
     pub filter: Option<Value>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchSort>)]
     pub sort: Option<Vec<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchDistinct>)]
+    pub distinct: Option<String>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchFacets>)]
     pub facets: Option<Vec<String>>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchHighlightPreTag>, default = DEFAULT_HIGHLIGHT_PRE_TAG())]
@@ -141,6 +151,7 @@ impl fmt::Debug for SearchQuery {
             page,
             hits_per_page,
             attributes_to_retrieve,
+            retrieve_vectors,
             attributes_to_crop,
             crop_length,
             attributes_to_highlight,
@@ -149,6 +160,7 @@ impl fmt::Debug for SearchQuery {
             show_ranking_score_details,
             filter,
             sort,
+            distinct,
             facets,
             highlight_pre_tag,
             highlight_post_tag,
@@ -173,6 +185,9 @@ impl fmt::Debug for SearchQuery {
         if let Some(q) = q {
             debug.field("q", &q);
         }
+        if *retrieve_vectors {
+            debug.field("retrieve_vectors", &retrieve_vectors);
+        }
         if let Some(v) = vector {
             if v.len() < 10 {
                 debug.field("vector", &v);
@@ -195,6 +210,9 @@ impl fmt::Debug for SearchQuery {
         if let Some(sort) = sort {
             debug.field("sort", &sort);
         }
+        if let Some(distinct) = distinct {
+            debug.field("distinct", &distinct);
+        }
         if let Some(facets) = facets {
             debug.field("facets", &facets);
         }
@@ -244,11 +262,13 @@ pub struct HybridQuery {
     pub embedder: Option<String>,
 }
 
+#[derive(Clone)]
 pub enum SearchKind {
     KeywordOnly,
     SemanticOnly { embedder_name: String, embedder: Arc<Embedder> },
     Hybrid { embedder_name: String, embedder: Arc<Embedder>, semantic_ratio: f32 },
 }
+
 impl SearchKind {
     pub(crate) fn semantic(
         index_scheduler: &index_scheduler::IndexScheduler,
@@ -345,7 +365,7 @@ impl SearchQuery {
     }
 }
 
-/// A `SearchQuery` + an index UID.
+/// A `SearchQuery` + an index UID and optional FederationOptions.
 // This struct contains the fields of `SearchQuery` inline.
 // This is because neither deserr nor serde support `flatten` when using `deny_unknown_fields.
 // The `From<SearchQueryWithIndex>` implementation ensures both structs remain up to date.
@@ -360,16 +380,18 @@ pub struct SearchQueryWithIndex {
     pub vector: Option<Vec<f32>>,
     #[deserr(default, error = DeserrJsonError<InvalidHybridQuery>)]
     pub hybrid: Option<HybridQuery>,
-    #[deserr(default = DEFAULT_SEARCH_OFFSET(), error = DeserrJsonError<InvalidSearchOffset>)]
-    pub offset: usize,
-    #[deserr(default = DEFAULT_SEARCH_LIMIT(), error = DeserrJsonError<InvalidSearchLimit>)]
-    pub limit: usize,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchOffset>)]
+    pub offset: Option<usize>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchLimit>)]
+    pub limit: Option<usize>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchPage>)]
     pub page: Option<usize>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchHitsPerPage>)]
     pub hits_per_page: Option<usize>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToRetrieve>)]
     pub attributes_to_retrieve: Option<BTreeSet<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchRetrieveVectors>)]
+    pub retrieve_vectors: bool,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToCrop>)]
     pub attributes_to_crop: Option<Vec<String>>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchCropLength>, default = DEFAULT_CROP_LENGTH())]
@@ -386,6 +408,8 @@ pub struct SearchQueryWithIndex {
     pub filter: Option<Value>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchSort>)]
     pub sort: Option<Vec<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchDistinct>)]
+    pub distinct: Option<String>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchFacets>)]
     pub facets: Option<Vec<String>>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchHighlightPreTag>, default = DEFAULT_HIGHLIGHT_PRE_TAG())]
@@ -400,12 +424,33 @@ pub struct SearchQueryWithIndex {
     pub attributes_to_search_on: Option<Vec<String>>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchRankingScoreThreshold>, default)]
     pub ranking_score_threshold: Option<RankingScoreThreshold>,
+
+    #[deserr(default)]
+    pub federation_options: Option<FederationOptions>,
 }
 
 impl SearchQueryWithIndex {
-    pub fn into_index_query(self) -> (IndexUid, SearchQuery) {
+    pub fn has_federation_options(&self) -> bool {
+        self.federation_options.is_some()
+    }
+    pub fn has_pagination(&self) -> Option<&'static str> {
+        if self.offset.is_some() {
+            Some("offset")
+        } else if self.limit.is_some() {
+            Some("limit")
+        } else if self.page.is_some() {
+            Some("page")
+        } else if self.hits_per_page.is_some() {
+            Some("hitsPerPage")
+        } else {
+            None
+        }
+    }
+
+    pub fn into_index_query_federation(self) -> (IndexUid, SearchQuery, Option<FederationOptions>) {
         let SearchQueryWithIndex {
             index_uid,
+            federation_options,
             q,
             vector,
             offset,
@@ -413,6 +458,7 @@ impl SearchQueryWithIndex {
             page,
             hits_per_page,
             attributes_to_retrieve,
+            retrieve_vectors,
             attributes_to_crop,
             crop_length,
             attributes_to_highlight,
@@ -421,6 +467,7 @@ impl SearchQueryWithIndex {
             show_matches_position,
             filter,
             sort,
+            distinct,
             facets,
             highlight_pre_tag,
             highlight_post_tag,
@@ -435,11 +482,12 @@ impl SearchQueryWithIndex {
             SearchQuery {
                 q,
                 vector,
-                offset,
-                limit,
+                offset: offset.unwrap_or(DEFAULT_SEARCH_OFFSET()),
+                limit: limit.unwrap_or(DEFAULT_SEARCH_LIMIT()),
                 page,
                 hits_per_page,
                 attributes_to_retrieve,
+                retrieve_vectors,
                 attributes_to_crop,
                 crop_length,
                 attributes_to_highlight,
@@ -448,6 +496,7 @@ impl SearchQueryWithIndex {
                 show_matches_position,
                 filter,
                 sort,
+                distinct,
                 facets,
                 highlight_pre_tag,
                 highlight_post_tag,
@@ -459,6 +508,7 @@ impl SearchQueryWithIndex {
                 // do not use ..Default::default() here,
                 // rather add any missing field from `SearchQuery` to `SearchQueryWithIndex`
             },
+            federation_options,
         )
     }
 }
@@ -478,6 +528,8 @@ pub struct SimilarQuery {
     pub embedder: Option<String>,
     #[deserr(default, error = DeserrJsonError<InvalidSimilarAttributesToRetrieve>)]
     pub attributes_to_retrieve: Option<BTreeSet<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSimilarRetrieveVectors>)]
+    pub retrieve_vectors: bool,
     #[deserr(default, error = DeserrJsonError<InvalidSimilarShowRankingScore>, default)]
     pub show_ranking_score: bool,
     #[deserr(default, error = DeserrJsonError<InvalidSimilarShowRankingScoreDetails>, default)]
@@ -716,6 +768,10 @@ fn prepare_search<'t>(
         search.ranking_score_threshold(ranking_score_threshold.0);
     }
 
+    if let Some(distinct) = &query.distinct {
+        search.distinct(distinct.clone());
+    }
+
     match search_kind {
         SearchKind::KeywordOnly => {
             if let Some(q) = &query.q {
@@ -725,10 +781,15 @@ fn prepare_search<'t>(
         SearchKind::SemanticOnly { embedder_name, embedder } => {
             let vector = match query.vector.clone() {
                 Some(vector) => vector,
-                None => embedder
-                    .embed_one(query.q.clone().unwrap())
-                    .map_err(milli::vector::Error::from)
-                    .map_err(milli::Error::from)?,
+                None => {
+                    let span = tracing::trace_span!(target: "search::vector", "embed_one");
+                    let _entered = span.enter();
+
+                    embedder
+                        .embed_one(query.q.clone().unwrap())
+                        .map_err(milli::vector::Error::from)
+                        .map_err(milli::Error::from)?
+                }
             };
 
             search.semantic(embedder_name.clone(), embedder.clone(), Some(vector));
@@ -810,6 +871,7 @@ pub fn perform_search(
     index: &Index,
     query: SearchQuery,
     search_kind: SearchKind,
+    retrieve_vectors: RetrieveVectors,
 ) -> Result<SearchResult, MeilisearchHttpError> {
     let before_search = Instant::now();
     let rtxn = index.read_txn()?;
@@ -831,15 +893,7 @@ pub fn perform_search(
             used_negative_operator,
         },
         semantic_hit_count,
-    ) = match &search_kind {
-        SearchKind::KeywordOnly => (search.execute()?, None),
-        SearchKind::SemanticOnly { .. } => {
-            let results = search.execute()?;
-            let semantic_hit_count = results.document_scores.len() as u32;
-            (results, Some(semantic_hit_count))
-        }
-        SearchKind::Hybrid { semantic_ratio, .. } => search.execute_hybrid(*semantic_ratio)?,
-    };
+    ) = search_from_kind(search_kind, search)?;
 
     let SearchQuery {
         q,
@@ -847,6 +901,8 @@ pub fn perform_search(
         page,
         hits_per_page,
         attributes_to_retrieve,
+        // use the enum passed as parameter
+        retrieve_vectors: _,
         attributes_to_crop,
         crop_length,
         attributes_to_highlight,
@@ -866,10 +922,12 @@ pub fn perform_search(
         matching_strategy: _,
         attributes_to_search_on: _,
         filter: _,
+        distinct: _,
     } = query;
 
     let format = AttributesFormat {
         attributes_to_retrieve,
+        retrieve_vectors,
         attributes_to_highlight,
         attributes_to_crop,
         crop_length,
@@ -882,8 +940,13 @@ pub fn perform_search(
         show_ranking_score_details,
     };
 
-    let documents =
-        make_hits(index, &rtxn, format, matching_words, documents_ids, document_scores)?;
+    let documents = make_hits(
+        index,
+        &rtxn,
+        format,
+        matching_words,
+        documents_ids.iter().copied().zip(document_scores.iter()),
+    )?;
 
     let number_of_hits = min(candidates.len() as usize, max_total_hits);
     let hits_info = if is_finite_pagination {
@@ -951,8 +1014,25 @@ pub fn perform_search(
     Ok(result)
 }
 
+pub fn search_from_kind(
+    search_kind: SearchKind,
+    search: milli::Search<'_>,
+) -> Result<(milli::SearchResult, Option<u32>), MeilisearchHttpError> {
+    let (milli_result, semantic_hit_count) = match &search_kind {
+        SearchKind::KeywordOnly => (search.execute()?, None),
+        SearchKind::SemanticOnly { .. } => {
+            let results = search.execute()?;
+            let semantic_hit_count = results.document_scores.len() as u32;
+            (results, Some(semantic_hit_count))
+        }
+        SearchKind::Hybrid { semantic_ratio, .. } => search.execute_hybrid(*semantic_ratio)?,
+    };
+    Ok((milli_result, semantic_hit_count))
+}
+
 struct AttributesFormat {
     attributes_to_retrieve: Option<BTreeSet<String>>,
+    retrieve_vectors: RetrieveVectors,
     attributes_to_highlight: Option<HashSet<String>>,
     attributes_to_crop: Option<Vec<String>>,
     crop_length: usize,
@@ -965,103 +1045,248 @@ struct AttributesFormat {
     show_ranking_score_details: bool,
 }
 
-fn make_hits(
-    index: &Index,
-    rtxn: &RoTxn<'_>,
-    format: AttributesFormat,
-    matching_words: milli::MatchingWords,
-    documents_ids: Vec<u32>,
-    document_scores: Vec<Vec<ScoreDetails>>,
-) -> Result<Vec<SearchHit>, MeilisearchHttpError> {
-    let fields_ids_map = index.fields_ids_map(rtxn).unwrap();
-    let displayed_ids = index
-        .displayed_fields_ids(rtxn)?
-        .map(|fields| fields.into_iter().collect::<BTreeSet<_>>())
-        .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect());
-    let fids = |attrs: &BTreeSet<String>| {
-        let mut ids = BTreeSet::new();
-        for attr in attrs {
-            if attr == "*" {
-                ids.clone_from(&displayed_ids);
-                break;
-            }
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RetrieveVectors {
+    /// Do not touch the `_vectors` field
+    ///
+    /// this is the behavior when the vectorStore feature is disabled
+    Ignore,
+    /// Remove the `_vectors` field
+    ///
+    /// this is the behavior when the vectorStore feature is enabled, and `retrieveVectors` is `false`
+    Hide,
+    /// Retrieve vectors from the DB and merge them into the `_vectors` field
+    ///
+    /// this is the behavior when the vectorStore feature is enabled, and `retrieveVectors` is `true`
+    Retrieve,
+}
 
-            if let Some(id) = fields_ids_map.id(attr) {
-                ids.insert(id);
-            }
+impl RetrieveVectors {
+    pub fn new(
+        retrieve_vector: bool,
+        features: index_scheduler::RoFeatures,
+    ) -> Result<Self, index_scheduler::Error> {
+        match (retrieve_vector, features.check_vector("Passing `retrieveVectors` as a parameter")) {
+            (true, Ok(())) => Ok(Self::Retrieve),
+            (true, Err(error)) => Err(error),
+            (false, Ok(())) => Ok(Self::Hide),
+            (false, Err(_)) => Ok(Self::Ignore),
         }
-        ids
-    };
-    let to_retrieve_ids: BTreeSet<_> = format
-        .attributes_to_retrieve
-        .as_ref()
-        .map(fids)
-        .unwrap_or_else(|| displayed_ids.clone())
-        .intersection(&displayed_ids)
-        .cloned()
-        .collect();
-    let attr_to_highlight = format.attributes_to_highlight.unwrap_or_default();
-    let attr_to_crop = format.attributes_to_crop.unwrap_or_default();
-    let formatted_options = compute_formatted_options(
-        &attr_to_highlight,
-        &attr_to_crop,
-        format.crop_length,
-        &to_retrieve_ids,
-        &fields_ids_map,
-        &displayed_ids,
-    );
-    let mut tokenizer_builder = TokenizerBuilder::default();
-    tokenizer_builder.create_char_map(true);
-    let script_lang_map = index.script_language(rtxn)?;
-    if !script_lang_map.is_empty() {
-        tokenizer_builder.allow_list(&script_lang_map);
     }
-    let separators = index.allowed_separators(rtxn)?;
-    let separators: Option<Vec<_>> =
-        separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
-    if let Some(ref separators) = separators {
-        tokenizer_builder.separators(separators);
+}
+
+struct HitMaker<'a> {
+    index: &'a Index,
+    rtxn: &'a RoTxn<'a>,
+    fields_ids_map: FieldsIdsMap,
+    displayed_ids: BTreeSet<FieldId>,
+    vectors_fid: Option<FieldId>,
+    retrieve_vectors: RetrieveVectors,
+    to_retrieve_ids: BTreeSet<FieldId>,
+    embedding_configs: Vec<milli::index::IndexEmbeddingConfig>,
+    formatter_builder: MatcherBuilder<'a>,
+    formatted_options: BTreeMap<FieldId, FormatOptions>,
+    show_ranking_score: bool,
+    show_ranking_score_details: bool,
+    sort: Option<Vec<String>>,
+    show_matches_position: bool,
+}
+
+impl<'a> HitMaker<'a> {
+    pub fn tokenizer<'b>(
+        script_lang_map: &'b HashMap<milli::tokenizer::Script, Vec<milli::tokenizer::Language>>,
+        dictionary: Option<&'b [&'b str]>,
+        separators: Option<&'b [&'b str]>,
+    ) -> milli::tokenizer::Tokenizer<'b> {
+        let mut tokenizer_builder = TokenizerBuilder::default();
+        tokenizer_builder.create_char_map(true);
+        if !script_lang_map.is_empty() {
+            tokenizer_builder.allow_list(script_lang_map);
+        }
+
+        if let Some(separators) = separators {
+            tokenizer_builder.separators(separators);
+        }
+
+        if let Some(dictionary) = dictionary {
+            tokenizer_builder.words_dict(dictionary);
+        }
+
+        tokenizer_builder.into_tokenizer()
     }
-    let dictionary = index.dictionary(rtxn)?;
-    let dictionary: Option<Vec<_>> =
-        dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
-    if let Some(ref dictionary) = dictionary {
-        tokenizer_builder.words_dict(dictionary);
+
+    pub fn formatter_builder(
+        matching_words: milli::MatchingWords,
+        tokenizer: milli::tokenizer::Tokenizer<'_>,
+    ) -> MatcherBuilder<'_> {
+        let formatter_builder = MatcherBuilder::new(matching_words, tokenizer);
+
+        formatter_builder
     }
-    let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer_builder.build());
-    formatter_builder.crop_marker(format.crop_marker);
-    formatter_builder.highlight_prefix(format.highlight_pre_tag);
-    formatter_builder.highlight_suffix(format.highlight_post_tag);
-    let mut documents = Vec::new();
-    let documents_iter = index.documents(rtxn, documents_ids)?;
-    for ((_id, obkv), score) in documents_iter.into_iter().zip(document_scores.into_iter()) {
+
+    pub fn new(
+        index: &'a Index,
+        rtxn: &'a RoTxn<'a>,
+        format: AttributesFormat,
+        mut formatter_builder: MatcherBuilder<'a>,
+    ) -> Result<Self, MeilisearchHttpError> {
+        formatter_builder.crop_marker(format.crop_marker);
+        formatter_builder.highlight_prefix(format.highlight_pre_tag);
+        formatter_builder.highlight_suffix(format.highlight_post_tag);
+
+        let fields_ids_map = index.fields_ids_map(rtxn)?;
+        let displayed_ids = index
+            .displayed_fields_ids(rtxn)?
+            .map(|fields| fields.into_iter().collect::<BTreeSet<_>>());
+
+        let vectors_fid =
+            fields_ids_map.id(milli::vector::parsed_vectors::RESERVED_VECTORS_FIELD_NAME);
+
+        let vectors_is_hidden = match (&displayed_ids, vectors_fid) {
+            // displayed_ids is a wildcard, so `_vectors` can be displayed regardless of its fid
+            (None, _) => false,
+            // displayed_ids is a finite list, and `_vectors` cannot be part of it because it is not an existing field
+            (Some(_), None) => true,
+            // displayed_ids is a finit list, so hide if `_vectors` is not part of it
+            (Some(map), Some(vectors_fid)) => map.contains(&vectors_fid),
+        };
+
+        let displayed_ids =
+            displayed_ids.unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect());
+
+        let retrieve_vectors = if let RetrieveVectors::Retrieve = format.retrieve_vectors {
+            if vectors_is_hidden {
+                RetrieveVectors::Hide
+            } else {
+                RetrieveVectors::Retrieve
+            }
+        } else {
+            format.retrieve_vectors
+        };
+
+        let fids = |attrs: &BTreeSet<String>| {
+            let mut ids = BTreeSet::new();
+            for attr in attrs {
+                if attr == "*" {
+                    ids.clone_from(&displayed_ids);
+                    break;
+                }
+
+                if let Some(id) = fields_ids_map.id(attr) {
+                    ids.insert(id);
+                }
+            }
+            ids
+        };
+        let to_retrieve_ids: BTreeSet<_> = format
+            .attributes_to_retrieve
+            .as_ref()
+            .map(fids)
+            .unwrap_or_else(|| displayed_ids.clone())
+            .intersection(&displayed_ids)
+            .cloned()
+            .collect();
+
+        let attr_to_highlight = format.attributes_to_highlight.unwrap_or_default();
+        let attr_to_crop = format.attributes_to_crop.unwrap_or_default();
+        let formatted_options = compute_formatted_options(
+            &attr_to_highlight,
+            &attr_to_crop,
+            format.crop_length,
+            &to_retrieve_ids,
+            &fields_ids_map,
+            &displayed_ids,
+        );
+
+        let embedding_configs = index.embedding_configs(rtxn)?;
+
+        Ok(Self {
+            index,
+            rtxn,
+            fields_ids_map,
+            displayed_ids,
+            vectors_fid,
+            retrieve_vectors,
+            to_retrieve_ids,
+            embedding_configs,
+            formatter_builder,
+            formatted_options,
+            show_ranking_score: format.show_ranking_score,
+            show_ranking_score_details: format.show_ranking_score_details,
+            show_matches_position: format.show_matches_position,
+            sort: format.sort,
+        })
+    }
+
+    pub fn make_hit(
+        &self,
+        id: u32,
+        score: &[ScoreDetails],
+    ) -> Result<SearchHit, MeilisearchHttpError> {
+        let (_, obkv) =
+            self.index.iter_documents(self.rtxn, std::iter::once(id))?.next().unwrap()?;
+
         // First generate a document with all the displayed fields
-        let displayed_document = make_document(&displayed_ids, &fields_ids_map, obkv)?;
+        let displayed_document = make_document(&self.displayed_ids, &self.fields_ids_map, obkv)?;
+
+        let add_vectors_fid =
+            self.vectors_fid.filter(|_fid| self.retrieve_vectors == RetrieveVectors::Retrieve);
 
         // select the attributes to retrieve
-        let attributes_to_retrieve = to_retrieve_ids
+        let attributes_to_retrieve = self
+            .to_retrieve_ids
             .iter()
-            .map(|&fid| fields_ids_map.name(fid).expect("Missing field name"));
+            // skip the vectors_fid if RetrieveVectors::Hide
+            .filter(|fid| match self.vectors_fid {
+                Some(vectors_fid) => {
+                    !(self.retrieve_vectors == RetrieveVectors::Hide && **fid == vectors_fid)
+                }
+                None => true,
+            })
+            // need to retrieve the existing `_vectors` field if the `RetrieveVectors::Retrieve`
+            .chain(add_vectors_fid.iter())
+            .map(|&fid| self.fields_ids_map.name(fid).expect("Missing field name"));
+
         let mut document =
             permissive_json_pointer::select_values(&displayed_document, attributes_to_retrieve);
 
+        if self.retrieve_vectors == RetrieveVectors::Retrieve {
+            // Clippy is wrong
+            #[allow(clippy::manual_unwrap_or_default)]
+            let mut vectors = match document.remove("_vectors") {
+                Some(Value::Object(map)) => map,
+                _ => Default::default(),
+            };
+            for (name, vector) in self.index.embeddings(self.rtxn, id)? {
+                let user_provided = self
+                    .embedding_configs
+                    .iter()
+                    .find(|conf| conf.name == name)
+                    .is_some_and(|conf| conf.user_provided.contains(id));
+                let embeddings =
+                    ExplicitVectors { embeddings: Some(vector.into()), regenerate: !user_provided };
+                vectors.insert(name, serde_json::to_value(embeddings)?);
+            }
+            document.insert("_vectors".into(), vectors.into());
+        }
+
         let (matches_position, formatted) = format_fields(
             &displayed_document,
-            &fields_ids_map,
-            &formatter_builder,
-            &formatted_options,
-            format.show_matches_position,
-            &displayed_ids,
+            &self.fields_ids_map,
+            &self.formatter_builder,
+            &self.formatted_options,
+            self.show_matches_position,
+            &self.displayed_ids,
         )?;
 
-        if let Some(sort) = format.sort.as_ref() {
+        if let Some(sort) = self.sort.as_ref() {
             insert_geo_distance(sort, &mut document);
         }
 
         let ranking_score =
-            format.show_ranking_score.then(|| ScoreDetails::global_score(score.iter()));
+            self.show_ranking_score.then(|| ScoreDetails::global_score(score.iter()));
         let ranking_score_details =
-            format.show_ranking_score_details.then(|| ScoreDetails::to_json_map(score.iter()));
+            self.show_ranking_score_details.then(|| ScoreDetails::to_json_map(score.iter()));
 
         let hit = SearchHit {
             document,
@@ -1070,7 +1295,38 @@ fn make_hits(
             ranking_score_details,
             ranking_score,
         };
-        documents.push(hit);
+
+        Ok(hit)
+    }
+}
+
+fn make_hits<'a>(
+    index: &Index,
+    rtxn: &RoTxn<'_>,
+    format: AttributesFormat,
+    matching_words: milli::MatchingWords,
+    documents_ids_scores: impl Iterator<Item = (u32, &'a Vec<ScoreDetails>)> + 'a,
+) -> Result<Vec<SearchHit>, MeilisearchHttpError> {
+    let mut documents = Vec::new();
+
+    let script_lang_map = index.script_language(rtxn)?;
+
+    let dictionary = index.dictionary(rtxn)?;
+    let dictionary: Option<Vec<_>> =
+        dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
+    let separators = index.allowed_separators(rtxn)?;
+    let separators: Option<Vec<_>> =
+        separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
+
+    let tokenizer =
+        HitMaker::tokenizer(&script_lang_map, dictionary.as_deref(), separators.as_deref());
+
+    let formatter_builder = HitMaker::formatter_builder(matching_words, tokenizer);
+
+    let hit_maker = HitMaker::new(index, rtxn, format, formatter_builder)?;
+
+    for (id, score) in documents_ids_scores {
+        documents.push(hit_maker.make_hit(id, score)?);
     }
     Ok(documents)
 }
@@ -1114,6 +1370,7 @@ pub fn perform_similar(
     query: SimilarQuery,
     embedder_name: String,
     embedder: Arc<Embedder>,
+    retrieve_vectors: RetrieveVectors,
 ) -> Result<SimilarResult, ResponseError> {
     let before_search = Instant::now();
     let rtxn = index.read_txn()?;
@@ -1125,6 +1382,7 @@ pub fn perform_similar(
         filter: _,
         embedder: _,
         attributes_to_retrieve,
+        retrieve_vectors: _,
         show_ranking_score,
         show_ranking_score_details,
         ranking_score_threshold,
@@ -1171,6 +1429,7 @@ pub fn perform_similar(
 
     let format = AttributesFormat {
         attributes_to_retrieve,
+        retrieve_vectors,
         attributes_to_highlight: None,
         attributes_to_crop: None,
         crop_length: DEFAULT_CROP_LENGTH(),
@@ -1183,7 +1442,13 @@ pub fn perform_similar(
         show_ranking_score_details,
     };
 
-    let hits = make_hits(index, &rtxn, format, Default::default(), documents_ids, document_scores)?;
+    let hits = make_hits(
+        index,
+        &rtxn,
+        format,
+        Default::default(),
+        documents_ids.iter().copied().zip(document_scores.iter()),
+    )?;
 
     let max_total_hits = index
         .pagination_max_total_hits(&rtxn)
@@ -1212,13 +1477,23 @@ fn insert_geo_distance(sorts: &[String], document: &mut Document) {
         // TODO: TAMO: milli encountered an internal error, what do we want to do?
         let base = [capture_group[1].parse().unwrap(), capture_group[2].parse().unwrap()];
         let geo_point = &document.get("_geo").unwrap_or(&json!(null));
-        if let Some((lat, lng)) = geo_point["lat"].as_f64().zip(geo_point["lng"].as_f64()) {
+        if let Some((lat, lng)) =
+            extract_geo_value(&geo_point["lat"]).zip(extract_geo_value(&geo_point["lng"]))
+        {
             let distance = milli::distance_between_two_points(&base, &[lat, lng]);
             document.insert("_geoDistance".to_string(), json!(distance.round() as usize));
         }
     }
 }
 
+fn extract_geo_value(value: &Value) -> Option<f64> {
+    match value {
+        Value::Number(n) => n.as_f64(),
+        Value::String(s) => s.parse().ok(),
+        _ => None,
+    }
+}
+
 fn compute_formatted_options(
     attr_to_highlight: &HashSet<String>,
     attr_to_crop: &[String],
@@ -1346,10 +1621,10 @@ fn make_document(
     Ok(document)
 }
 
-fn format_fields<'a>(
+fn format_fields(
     document: &Document,
     field_ids_map: &FieldsIdsMap,
-    builder: &'a MatcherBuilder<'a>,
+    builder: &MatcherBuilder<'_>,
     formatted_options: &BTreeMap<FieldId, FormatOptions>,
     compute_matches: bool,
     displayable_ids: &BTreeSet<FieldId>,
@@ -1404,9 +1679,9 @@ fn format_fields<'a>(
     Ok((matches_position, document))
 }
 
-fn format_value<'a>(
+fn format_value(
     value: Value,
-    builder: &'a MatcherBuilder<'a>,
+    builder: &MatcherBuilder<'_>,
     format_options: Option<FormatOptions>,
     infos: &mut Vec<MatchBounds>,
     compute_matches: bool,
@@ -1592,4 +1867,54 @@ mod test {
         insert_geo_distance(sorters, &mut document);
         assert_eq!(document.get("_geoDistance"), None);
     }
+
+    #[test]
+    fn test_insert_geo_distance_with_coords_as_string() {
+        let value: Document = serde_json::from_str(
+            r#"{
+              "_geo": {
+                "lat": "50",
+                "lng": 3
+              }
+            }"#,
+        )
+        .unwrap();
+
+        let sorters = &["_geoPoint(50,3):desc".to_string()];
+        let mut document = value.clone();
+        insert_geo_distance(sorters, &mut document);
+        assert_eq!(document.get("_geoDistance"), Some(&json!(0)));
+
+        let value: Document = serde_json::from_str(
+            r#"{
+              "_geo": {
+                "lat": "50",
+                "lng": "3"
+              },
+              "id": "1"
+            }"#,
+        )
+        .unwrap();
+
+        let sorters = &["_geoPoint(50,3):desc".to_string()];
+        let mut document = value.clone();
+        insert_geo_distance(sorters, &mut document);
+        assert_eq!(document.get("_geoDistance"), Some(&json!(0)));
+
+        let value: Document = serde_json::from_str(
+            r#"{
+              "_geo": {
+                "lat": 50,
+                "lng": "3"
+              },
+              "id": "1"
+            }"#,
+        )
+        .unwrap();
+
+        let sorters = &["_geoPoint(50,3):desc".to_string()];
+        let mut document = value.clone();
+        insert_geo_distance(sorters, &mut document);
+        assert_eq!(document.get("_geoDistance"), Some(&json!(0)));
+    }
 }

meilisearch/src/search/ranking_rules.rs

@@ -0,0 +1,823 @@
+use std::collections::HashMap;
+use std::fmt::Write;
+
+use itertools::Itertools as _;
+use meilisearch_types::error::{Code, ResponseError};
+use meilisearch_types::milli::{AscDesc, Criterion, Member, TermsMatchingStrategy};
+
+pub struct RankingRules {
+    canonical_criteria: Vec<Criterion>,
+    canonical_sort: Option<Vec<AscDesc>>,
+    canonicalization_actions: Vec<CanonicalizationAction>,
+    source_criteria: Vec<Criterion>,
+    source_sort: Option<Vec<AscDesc>>,
+}
+
+pub enum CanonicalizationAction {
+    PrependedWords {
+        prepended_index: RankingRuleSource,
+    },
+    RemovedDuplicate {
+        earlier_occurrence: RankingRuleSource,
+        removed_occurrence: RankingRuleSource,
+    },
+    RemovedWords {
+        reason: RemoveWords,
+        removed_occurrence: RankingRuleSource,
+    },
+    RemovedPlaceholder {
+        removed_occurrence: RankingRuleSource,
+    },
+    TruncatedVector {
+        vector_rule: RankingRuleSource,
+        truncated_from: RankingRuleSource,
+    },
+    RemovedVector {
+        vector_rule: RankingRuleSource,
+        removed_occurrence: RankingRuleSource,
+    },
+    RemovedSort {
+        removed_occurrence: RankingRuleSource,
+    },
+}
+
+pub enum RemoveWords {
+    WasPrepended,
+    MatchingStrategyAll,
+}
+
+impl std::fmt::Display for RemoveWords {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let reason = match self {
+            RemoveWords::WasPrepended => "it was previously prepended",
+            RemoveWords::MatchingStrategyAll => "`query.matchingWords` is set to `all`",
+        };
+        f.write_str(reason)
+    }
+}
+
+pub enum CanonicalizationKind {
+    Placeholder,
+    Keyword,
+    Vector,
+}
+
+pub struct CompatibilityError {
+    previous: RankingRule,
+    current: RankingRule,
+}
+impl CompatibilityError {
+    pub(crate) fn to_response_error(
+        &self,
+        ranking_rules: &RankingRules,
+        previous_ranking_rules: &RankingRules,
+        query_index: usize,
+        previous_query_index: usize,
+        index_uid: &str,
+        previous_index_uid: &str,
+    ) -> meilisearch_types::error::ResponseError {
+        let rule = self.current.as_string(
+            &ranking_rules.canonical_criteria,
+            &ranking_rules.canonical_sort,
+            query_index,
+            index_uid,
+        );
+        let previous_rule = self.previous.as_string(
+            &previous_ranking_rules.canonical_criteria,
+            &previous_ranking_rules.canonical_sort,
+            previous_query_index,
+            previous_index_uid,
+        );
+
+        let canonicalization_actions = ranking_rules.canonicalization_notes();
+        let previous_canonicalization_actions = previous_ranking_rules.canonicalization_notes();
+
+        let mut msg = String::new();
+        let reason = self.reason();
+        let _ = writeln!(
+            &mut msg,
+            "The results of queries #{previous_query_index} and #{query_index} are incompatible: "
+        );
+        let _ = writeln!(&mut msg, "  1. {previous_rule}");
+        let _ = writeln!(&mut msg, "  2. {rule}");
+        let _ = writeln!(&mut msg, "  - {reason}");
+
+        if !previous_canonicalization_actions.is_empty() {
+            let _ = write!(&mut msg, "  - note: The ranking rules of query #{previous_query_index} were modified during canonicalization:\n{previous_canonicalization_actions}");
+        }
+
+        if !canonicalization_actions.is_empty() {
+            let _ = write!(&mut msg, "  - note: The ranking rules of query #{query_index} were modified during canonicalization:\n{canonicalization_actions}");
+        }
+
+        ResponseError::from_msg(msg, Code::InvalidMultiSearchQueryRankingRules)
+    }
+    pub fn reason(&self) -> &'static str {
+        match (self.previous.kind, self.current.kind) {
+            (RankingRuleKind::Relevancy, RankingRuleKind::AscendingSort)
+            | (RankingRuleKind::Relevancy, RankingRuleKind::DescendingSort)
+            | (RankingRuleKind::AscendingSort, RankingRuleKind::Relevancy)
+            | (RankingRuleKind::DescendingSort, RankingRuleKind::Relevancy) => {
+                "cannot compare a relevancy rule with a sort rule"
+            }
+
+            (RankingRuleKind::Relevancy, RankingRuleKind::AscendingGeoSort)
+            | (RankingRuleKind::Relevancy, RankingRuleKind::DescendingGeoSort)
+            | (RankingRuleKind::AscendingGeoSort, RankingRuleKind::Relevancy)
+            | (RankingRuleKind::DescendingGeoSort, RankingRuleKind::Relevancy) => {
+                "cannot compare a relevancy rule with a sort rule"
+            }
+
+            (RankingRuleKind::AscendingSort, RankingRuleKind::DescendingSort)
+            | (RankingRuleKind::DescendingSort, RankingRuleKind::AscendingSort) => {
+                "cannot compare two sort rules in opposite directions"
+            }
+
+            (RankingRuleKind::AscendingSort, RankingRuleKind::AscendingGeoSort)
+            | (RankingRuleKind::AscendingSort, RankingRuleKind::DescendingGeoSort)
+            | (RankingRuleKind::DescendingSort, RankingRuleKind::AscendingGeoSort)
+            | (RankingRuleKind::DescendingSort, RankingRuleKind::DescendingGeoSort)
+            | (RankingRuleKind::AscendingGeoSort, RankingRuleKind::AscendingSort)
+            | (RankingRuleKind::AscendingGeoSort, RankingRuleKind::DescendingSort)
+            | (RankingRuleKind::DescendingGeoSort, RankingRuleKind::AscendingSort)
+            | (RankingRuleKind::DescendingGeoSort, RankingRuleKind::DescendingSort) => {
+                "cannot compare a sort rule with a geosort rule"
+            }
+
+            (RankingRuleKind::AscendingGeoSort, RankingRuleKind::DescendingGeoSort)
+            | (RankingRuleKind::DescendingGeoSort, RankingRuleKind::AscendingGeoSort) => {
+                "cannot compare two geosort rules in opposite directions"
+            }
+            (RankingRuleKind::Relevancy, RankingRuleKind::Relevancy)
+            | (RankingRuleKind::AscendingSort, RankingRuleKind::AscendingSort)
+            | (RankingRuleKind::DescendingSort, RankingRuleKind::DescendingSort)
+            | (RankingRuleKind::AscendingGeoSort, RankingRuleKind::AscendingGeoSort)
+            | (RankingRuleKind::DescendingGeoSort, RankingRuleKind::DescendingGeoSort) => {
+                "internal error, comparison should be possible"
+            }
+        }
+    }
+}
+
+impl RankingRules {
+    pub fn new(
+        criteria: Vec<Criterion>,
+        sort: Option<Vec<AscDesc>>,
+        terms_matching_strategy: TermsMatchingStrategy,
+        canonicalization_kind: CanonicalizationKind,
+    ) -> Self {
+        let (canonical_criteria, canonical_sort, canonicalization_actions) =
+            Self::canonicalize(&criteria, &sort, terms_matching_strategy, canonicalization_kind);
+        Self {
+            canonical_criteria,
+            canonical_sort,
+            canonicalization_actions,
+            source_criteria: criteria,
+            source_sort: sort,
+        }
+    }
+
+    fn canonicalize(
+        criteria: &[Criterion],
+        sort: &Option<Vec<AscDesc>>,
+        terms_matching_strategy: TermsMatchingStrategy,
+        canonicalization_kind: CanonicalizationKind,
+    ) -> (Vec<Criterion>, Option<Vec<AscDesc>>, Vec<CanonicalizationAction>) {
+        match canonicalization_kind {
+            CanonicalizationKind::Placeholder => Self::canonicalize_placeholder(criteria, sort),
+            CanonicalizationKind::Keyword => {
+                Self::canonicalize_keyword(criteria, sort, terms_matching_strategy)
+            }
+            CanonicalizationKind::Vector => Self::canonicalize_vector(criteria, sort),
+        }
+    }
+
+    fn canonicalize_placeholder(
+        criteria: &[Criterion],
+        sort_query: &Option<Vec<AscDesc>>,
+    ) -> (Vec<Criterion>, Option<Vec<AscDesc>>, Vec<CanonicalizationAction>) {
+        let mut sort = None;
+
+        let mut sorted_fields = HashMap::new();
+        let mut canonicalization_actions = Vec::new();
+        let mut canonical_criteria = Vec::new();
+        let mut canonical_sort = None;
+
+        for (criterion_index, criterion) in criteria.iter().enumerate() {
+            match criterion.clone() {
+                Criterion::Words
+                | Criterion::Typo
+                | Criterion::Proximity
+                | Criterion::Attribute
+                | Criterion::Exactness => {
+                    canonicalization_actions.push(CanonicalizationAction::RemovedPlaceholder {
+                        removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                    })
+                }
+
+                Criterion::Sort => {
+                    if let Some(previous_index) = sort {
+                        canonicalization_actions.push(CanonicalizationAction::RemovedDuplicate {
+                            earlier_occurrence: RankingRuleSource::Criterion(previous_index),
+                            removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                        });
+                    } else if let Some(sort_query) = sort_query {
+                        sort = Some(criterion_index);
+                        canonical_criteria.push(criterion.clone());
+                        canonical_sort = Some(canonicalize_sort(
+                            &mut sorted_fields,
+                            sort_query.as_slice(),
+                            criterion_index,
+                            &mut canonicalization_actions,
+                        ));
+                    } else {
+                        canonicalization_actions.push(CanonicalizationAction::RemovedSort {
+                            removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                        })
+                    }
+                }
+                Criterion::Asc(s) | Criterion::Desc(s) => match sorted_fields.entry(s) {
+                    std::collections::hash_map::Entry::Occupied(entry) => canonicalization_actions
+                        .push(CanonicalizationAction::RemovedDuplicate {
+                            earlier_occurrence: *entry.get(),
+                            removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                        }),
+                    std::collections::hash_map::Entry::Vacant(entry) => {
+                        entry.insert(RankingRuleSource::Criterion(criterion_index));
+                        canonical_criteria.push(criterion.clone())
+                    }
+                },
+            }
+        }
+
+        (canonical_criteria, canonical_sort, canonicalization_actions)
+    }
+
+    fn canonicalize_vector(
+        criteria: &[Criterion],
+        sort_query: &Option<Vec<AscDesc>>,
+    ) -> (Vec<Criterion>, Option<Vec<AscDesc>>, Vec<CanonicalizationAction>) {
+        let mut sort = None;
+
+        let mut sorted_fields = HashMap::new();
+        let mut canonicalization_actions = Vec::new();
+        let mut canonical_criteria = Vec::new();
+        let mut canonical_sort = None;
+
+        let mut vector = None;
+
+        'criteria: for (criterion_index, criterion) in criteria.iter().enumerate() {
+            match criterion.clone() {
+                Criterion::Words
+                | Criterion::Typo
+                | Criterion::Proximity
+                | Criterion::Attribute
+                | Criterion::Exactness => match vector {
+                    Some(previous_occurrence) => {
+                        if sorted_fields.is_empty() {
+                            canonicalization_actions.push(CanonicalizationAction::RemovedVector {
+                                vector_rule: RankingRuleSource::Criterion(previous_occurrence),
+                                removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                            });
+                        } else {
+                            canonicalization_actions.push(
+                                CanonicalizationAction::TruncatedVector {
+                                    vector_rule: RankingRuleSource::Criterion(previous_occurrence),
+                                    truncated_from: RankingRuleSource::Criterion(criterion_index),
+                                },
+                            );
+                            break 'criteria;
+                        }
+                    }
+                    None => {
+                        canonical_criteria.push(criterion.clone());
+                        vector = Some(criterion_index);
+                    }
+                },
+
+                Criterion::Sort => {
+                    if let Some(previous_index) = sort {
+                        canonicalization_actions.push(CanonicalizationAction::RemovedDuplicate {
+                            earlier_occurrence: RankingRuleSource::Criterion(previous_index),
+                            removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                        });
+                    } else if let Some(sort_query) = sort_query {
+                        sort = Some(criterion_index);
+                        canonical_criteria.push(criterion.clone());
+                        canonical_sort = Some(canonicalize_sort(
+                            &mut sorted_fields,
+                            sort_query.as_slice(),
+                            criterion_index,
+                            &mut canonicalization_actions,
+                        ));
+                    } else {
+                        canonicalization_actions.push(CanonicalizationAction::RemovedSort {
+                            removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                        })
+                    }
+                }
+                Criterion::Asc(s) | Criterion::Desc(s) => match sorted_fields.entry(s) {
+                    std::collections::hash_map::Entry::Occupied(entry) => canonicalization_actions
+                        .push(CanonicalizationAction::RemovedDuplicate {
+                            earlier_occurrence: *entry.get(),
+                            removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                        }),
+                    std::collections::hash_map::Entry::Vacant(entry) => {
+                        entry.insert(RankingRuleSource::Criterion(criterion_index));
+                        canonical_criteria.push(criterion.clone())
+                    }
+                },
+            }
+        }
+
+        (canonical_criteria, canonical_sort, canonicalization_actions)
+    }
+
+    fn canonicalize_keyword(
+        criteria: &[Criterion],
+        sort_query: &Option<Vec<AscDesc>>,
+        terms_matching_strategy: TermsMatchingStrategy,
+    ) -> (Vec<Criterion>, Option<Vec<AscDesc>>, Vec<CanonicalizationAction>) {
+        let mut words = None;
+        let mut typo = None;
+        let mut proximity = None;
+        let mut sort = None;
+        let mut attribute = None;
+        let mut exactness = None;
+        let mut sorted_fields = HashMap::new();
+
+        let mut canonical_criteria = Vec::new();
+        let mut canonical_sort = None;
+
+        let mut canonicalization_actions = Vec::new();
+
+        for (criterion_index, criterion) in criteria.iter().enumerate() {
+            let criterion = criterion.clone();
+            match criterion.clone() {
+                Criterion::Words => {
+                    if let TermsMatchingStrategy::All = terms_matching_strategy {
+                        canonicalization_actions.push(CanonicalizationAction::RemovedWords {
+                            reason: RemoveWords::MatchingStrategyAll,
+                            removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                        });
+                        continue;
+                    }
+                    if let Some(maybe_previous_index) = words {
+                        if let Some(previous_index) = maybe_previous_index {
+                            canonicalization_actions.push(
+                                CanonicalizationAction::RemovedDuplicate {
+                                    earlier_occurrence: RankingRuleSource::Criterion(
+                                        previous_index,
+                                    ),
+                                    removed_occurrence: RankingRuleSource::Criterion(
+                                        criterion_index,
+                                    ),
+                                },
+                            );
+                            continue;
+                        }
+                        canonicalization_actions.push(CanonicalizationAction::RemovedWords {
+                            reason: RemoveWords::WasPrepended,
+                            removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                        })
+                    }
+                    words = Some(Some(criterion_index));
+                    canonical_criteria.push(criterion);
+                }
+                Criterion::Typo => {
+                    canonicalize_criterion(
+                        criterion,
+                        criterion_index,
+                        terms_matching_strategy,
+                        &mut words,
+                        &mut canonicalization_actions,
+                        &mut canonical_criteria,
+                        &mut typo,
+                    );
+                }
+                Criterion::Proximity => {
+                    canonicalize_criterion(
+                        criterion,
+                        criterion_index,
+                        terms_matching_strategy,
+                        &mut words,
+                        &mut canonicalization_actions,
+                        &mut canonical_criteria,
+                        &mut proximity,
+                    );
+                }
+                Criterion::Attribute => {
+                    canonicalize_criterion(
+                        criterion,
+                        criterion_index,
+                        terms_matching_strategy,
+                        &mut words,
+                        &mut canonicalization_actions,
+                        &mut canonical_criteria,
+                        &mut attribute,
+                    );
+                }
+                Criterion::Exactness => {
+                    canonicalize_criterion(
+                        criterion,
+                        criterion_index,
+                        terms_matching_strategy,
+                        &mut words,
+                        &mut canonicalization_actions,
+                        &mut canonical_criteria,
+                        &mut exactness,
+                    );
+                }
+
+                Criterion::Sort => {
+                    if let Some(previous_index) = sort {
+                        canonicalization_actions.push(CanonicalizationAction::RemovedDuplicate {
+                            earlier_occurrence: RankingRuleSource::Criterion(previous_index),
+                            removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                        });
+                    } else if let Some(sort_query) = sort_query {
+                        sort = Some(criterion_index);
+                        canonical_criteria.push(criterion);
+                        canonical_sort = Some(canonicalize_sort(
+                            &mut sorted_fields,
+                            sort_query.as_slice(),
+                            criterion_index,
+                            &mut canonicalization_actions,
+                        ));
+                    } else {
+                        canonicalization_actions.push(CanonicalizationAction::RemovedSort {
+                            removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                        })
+                    }
+                }
+                Criterion::Asc(s) | Criterion::Desc(s) => match sorted_fields.entry(s) {
+                    std::collections::hash_map::Entry::Occupied(entry) => canonicalization_actions
+                        .push(CanonicalizationAction::RemovedDuplicate {
+                            earlier_occurrence: *entry.get(),
+                            removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+                        }),
+                    std::collections::hash_map::Entry::Vacant(entry) => {
+                        entry.insert(RankingRuleSource::Criterion(criterion_index));
+                        canonical_criteria.push(criterion)
+                    }
+                },
+            }
+        }
+
+        (canonical_criteria, canonical_sort, canonicalization_actions)
+    }
+
+    pub fn is_compatible_with(&self, previous: &Self) -> Result<(), CompatibilityError> {
+        for (current, previous) in self.coalesce_iterator().zip(previous.coalesce_iterator()) {
+            if current.kind != previous.kind {
+                return Err(CompatibilityError { current, previous });
+            }
+        }
+        Ok(())
+    }
+
+    pub fn constraint_count(&self) -> usize {
+        self.coalesce_iterator().count()
+    }
+
+    fn coalesce_iterator(&self) -> impl Iterator<Item = RankingRule> + '_ {
+        self.canonical_criteria
+            .iter()
+            .enumerate()
+            .flat_map(|(criterion_index, criterion)| {
+                RankingRule::from_criterion(criterion_index, criterion, &self.canonical_sort)
+            })
+            .coalesce(
+                |previous @ RankingRule { source: previous_source, kind: previous_kind },
+                 current @ RankingRule { source, kind }| {
+                    match (previous_kind, kind) {
+                        (RankingRuleKind::Relevancy, RankingRuleKind::Relevancy) => {
+                            let merged_source = match (previous_source, source) {
+                                (
+                                    RankingRuleSource::Criterion(previous),
+                                    RankingRuleSource::Criterion(current),
+                                ) => RankingRuleSource::CoalescedCriteria(previous, current),
+                                (
+                                    RankingRuleSource::CoalescedCriteria(begin, _end),
+                                    RankingRuleSource::Criterion(current),
+                                ) => RankingRuleSource::CoalescedCriteria(begin, current),
+                                (_previous, current) => current,
+                            };
+                            Ok(RankingRule { source: merged_source, kind })
+                        }
+                        _ => Err((previous, current)),
+                    }
+                },
+            )
+    }
+
+    fn canonicalization_notes(&self) -> String {
+        use CanonicalizationAction::*;
+        let mut notes = String::new();
+        for (index, action) in self.canonicalization_actions.iter().enumerate() {
+            let index = index + 1;
+            let _ = match action {
+                PrependedWords { prepended_index } => writeln!(
+                    &mut notes,
+                    "    {index}. Prepended rule `words` before first relevancy rule `{}` at position {}",
+                    prepended_index.rule_name(&self.source_criteria, &self.source_sort),
+                    prepended_index.rule_position()
+                ),
+                RemovedDuplicate { earlier_occurrence, removed_occurrence } => writeln!(
+                    &mut notes,
+                    "    {index}. Removed duplicate rule `{}` at position {} as it already appears at position {}",
+                    earlier_occurrence.rule_name(&self.source_criteria, &self.source_sort),
+                    removed_occurrence.rule_position(),
+                    earlier_occurrence.rule_position(),
+                ),
+                RemovedWords { reason, removed_occurrence } => writeln!(
+                    &mut notes,
+                    "    {index}. Removed rule `words` at position {} because {reason}",
+                    removed_occurrence.rule_position()
+                ),
+                RemovedPlaceholder { removed_occurrence } => writeln!(
+                    &mut notes,
+                    "    {index}. Removed relevancy rule `{}` at position {} because the query is a placeholder search (`q`: \"\")",
+                    removed_occurrence.rule_name(&self.source_criteria, &self.source_sort),
+                    removed_occurrence.rule_position()
+                ),
+                TruncatedVector { vector_rule, truncated_from } => writeln!(
+                    &mut notes,
+                    "    {index}. Truncated relevancy rule `{}` at position {} and later rules because the query is a vector search and `vector` was inserted at position {}",
+                    truncated_from.rule_name(&self.source_criteria, &self.source_sort),
+                    truncated_from.rule_position(),
+                    vector_rule.rule_position(),
+                ),
+                RemovedVector { vector_rule, removed_occurrence } => writeln!(
+                    &mut notes,
+                    "    {index}. Removed relevancy rule `{}` at position {} because the query is a vector search and `vector` was already inserted at position {}",
+                    removed_occurrence.rule_name(&self.source_criteria, &self.source_sort),
+                    removed_occurrence.rule_position(),
+                    vector_rule.rule_position(),
+                ),
+                RemovedSort { removed_occurrence } => writeln!(
+                    &mut notes,
+                    "   {index}. Removed rule `sort` at position {} because `query.sort` is empty",
+removed_occurrence.rule_position()
+                ),
+            };
+        }
+        notes
+    }
+}
+
+fn canonicalize_sort(
+    sorted_fields: &mut HashMap<String, RankingRuleSource>,
+    sort_query: &[AscDesc],
+    criterion_index: usize,
+    canonicalization_actions: &mut Vec<CanonicalizationAction>,
+) -> Vec<AscDesc> {
+    let mut geo_sorted = None;
+    let mut canonical_sort = Vec::new();
+    for (sort_index, asc_desc) in sort_query.iter().enumerate() {
+        let source = RankingRuleSource::Sort { criterion_index, sort_index };
+        let asc_desc = asc_desc.clone();
+        match asc_desc.clone() {
+            AscDesc::Asc(Member::Field(s)) | AscDesc::Desc(Member::Field(s)) => {
+                match sorted_fields.entry(s) {
+                    std::collections::hash_map::Entry::Occupied(entry) => canonicalization_actions
+                        .push(CanonicalizationAction::RemovedDuplicate {
+                            earlier_occurrence: *entry.get(),
+                            removed_occurrence: source,
+                        }),
+                    std::collections::hash_map::Entry::Vacant(entry) => {
+                        entry.insert(source);
+                        canonical_sort.push(asc_desc);
+                    }
+                }
+            }
+            AscDesc::Asc(Member::Geo(_)) | AscDesc::Desc(Member::Geo(_)) => match geo_sorted {
+                Some(earlier_sort_index) => {
+                    canonicalization_actions.push(CanonicalizationAction::RemovedDuplicate {
+                        earlier_occurrence: RankingRuleSource::Sort {
+                            criterion_index,
+                            sort_index: earlier_sort_index,
+                        },
+                        removed_occurrence: source,
+                    })
+                }
+                None => {
+                    geo_sorted = Some(sort_index);
+                    canonical_sort.push(asc_desc);
+                }
+            },
+        }
+    }
+    canonical_sort
+}
+
+fn canonicalize_criterion(
+    criterion: Criterion,
+    criterion_index: usize,
+    terms_matching_strategy: TermsMatchingStrategy,
+    words: &mut Option<Option<usize>>,
+    canonicalization_actions: &mut Vec<CanonicalizationAction>,
+    canonical_criteria: &mut Vec<Criterion>,
+    rule: &mut Option<usize>,
+) {
+    *words = match (terms_matching_strategy, words.take()) {
+        (TermsMatchingStrategy::All, words) => words,
+        (_, None) => {
+            // inject words
+            canonicalization_actions.push(CanonicalizationAction::PrependedWords {
+                prepended_index: RankingRuleSource::Criterion(criterion_index),
+            });
+            canonical_criteria.push(Criterion::Words);
+            Some(None)
+        }
+        (_, words) => words,
+    };
+    if let Some(previous_index) = *rule {
+        canonicalization_actions.push(CanonicalizationAction::RemovedDuplicate {
+            earlier_occurrence: RankingRuleSource::Criterion(previous_index),
+            removed_occurrence: RankingRuleSource::Criterion(criterion_index),
+        });
+    } else {
+        *rule = Some(criterion_index);
+        canonical_criteria.push(criterion)
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum RankingRuleKind {
+    Relevancy,
+    AscendingSort,
+    DescendingSort,
+    AscendingGeoSort,
+    DescendingGeoSort,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub struct RankingRule {
+    source: RankingRuleSource,
+    kind: RankingRuleKind,
+}
+
+#[derive(Debug, Clone, Copy)]
+pub enum RankingRuleSource {
+    Criterion(usize),
+    CoalescedCriteria(usize, usize),
+    Sort { criterion_index: usize, sort_index: usize },
+}
+
+impl RankingRuleSource {
+    fn rule_name(&self, criteria: &[Criterion], sort: &Option<Vec<AscDesc>>) -> String {
+        match self {
+            RankingRuleSource::Criterion(criterion_index) => criteria
+                .get(*criterion_index)
+                .map(|c| c.to_string())
+                .unwrap_or_else(|| "unknown".into()),
+            RankingRuleSource::CoalescedCriteria(begin, end) => {
+                let rules: Vec<_> = criteria
+                    .get(*begin..=*end)
+                    .iter()
+                    .flat_map(|c| c.iter())
+                    .map(|c| c.to_string())
+                    .collect();
+                rules.join(", ")
+            }
+            RankingRuleSource::Sort { criterion_index: _, sort_index } => {
+                match sort.as_deref().and_then(|sort| sort.get(*sort_index)) {
+                    Some(sort) => match sort {
+                        AscDesc::Asc(Member::Field(field_name)) => format!("{field_name}:asc"),
+                        AscDesc::Desc(Member::Field(field_name)) => {
+                            format!("{field_name}:desc")
+                        }
+                        AscDesc::Asc(Member::Geo(_)) => "_geo(..):asc".to_string(),
+                        AscDesc::Desc(Member::Geo(_)) => "_geo(..):desc".to_string(),
+                    },
+                    None => "unknown".into(),
+                }
+            }
+        }
+    }
+
+    fn rule_position(&self) -> String {
+        match self {
+            RankingRuleSource::Criterion(criterion_index) => {
+                format!("#{criterion_index} in ranking rules")
+            }
+            RankingRuleSource::CoalescedCriteria(begin, end) => {
+                format!("#{begin} to #{end} in ranking rules")
+            }
+            RankingRuleSource::Sort { criterion_index, sort_index } => format!(
+                "#{sort_index} in `query.sort` (as `sort` is #{criterion_index} in ranking rules)"
+            ),
+        }
+    }
+}
+
+impl RankingRule {
+    fn from_criterion<'a>(
+        criterion_index: usize,
+        criterion: &'a Criterion,
+        sort: &'a Option<Vec<AscDesc>>,
+    ) -> impl Iterator<Item = Self> + 'a {
+        let kind = match criterion {
+            Criterion::Words
+            | Criterion::Typo
+            | Criterion::Proximity
+            | Criterion::Attribute
+            | Criterion::Exactness => RankingRuleKind::Relevancy,
+            Criterion::Asc(s) if s == "_geo" => RankingRuleKind::AscendingGeoSort,
+
+            Criterion::Asc(_) => RankingRuleKind::AscendingSort,
+            Criterion::Desc(s) if s == "_geo" => RankingRuleKind::DescendingGeoSort,
+
+            Criterion::Desc(_) => RankingRuleKind::DescendingSort,
+            Criterion::Sort => {
+                return either::Right(sort.iter().flatten().enumerate().map(
+                    move |(rule_index, asc_desc)| {
+                        Self::from_asc_desc(asc_desc, criterion_index, rule_index)
+                    },
+                ))
+            }
+        };
+
+        either::Left(std::iter::once(Self {
+            source: RankingRuleSource::Criterion(criterion_index),
+            kind,
+        }))
+    }
+
+    fn from_asc_desc(asc_desc: &AscDesc, sort_index: usize, rule_index_in_sort: usize) -> Self {
+        let kind = match asc_desc {
+            AscDesc::Asc(Member::Field(_)) => RankingRuleKind::AscendingSort,
+            AscDesc::Desc(Member::Field(_)) => RankingRuleKind::DescendingSort,
+            AscDesc::Asc(Member::Geo(_)) => RankingRuleKind::AscendingGeoSort,
+            AscDesc::Desc(Member::Geo(_)) => RankingRuleKind::DescendingGeoSort,
+        };
+        Self {
+            source: RankingRuleSource::Sort {
+                criterion_index: sort_index,
+                sort_index: rule_index_in_sort,
+            },
+            kind,
+        }
+    }
+
+    fn as_string(
+        &self,
+        canonical_criteria: &[Criterion],
+        canonical_sort: &Option<Vec<AscDesc>>,
+        query_index: usize,
+        index_uid: &str,
+    ) -> String {
+        let kind = match self.kind {
+            RankingRuleKind::Relevancy => "relevancy",
+            RankingRuleKind::AscendingSort => "ascending sort",
+            RankingRuleKind::DescendingSort => "descending sort",
+            RankingRuleKind::AscendingGeoSort => "ascending geo sort",
+            RankingRuleKind::DescendingGeoSort => "descending geo sort",
+        };
+        let rules = self.fetch_from_source(canonical_criteria, canonical_sort);
+
+        let source = match self.source {
+            RankingRuleSource::Criterion(criterion_index) => format!("`queries[{query_index}]`, `{index_uid}.rankingRules[{criterion_index}]`"),
+            RankingRuleSource::CoalescedCriteria(begin, end) => format!("`queries[{query_index}]`, `{index_uid}.rankingRules[{begin}..={end}]`"),
+            RankingRuleSource::Sort { criterion_index, sort_index } => format!("`queries[{query_index}].sort[{sort_index}]`, `{index_uid}.rankingRules[{criterion_index}]`"),
+        };
+
+        format!("{source}: {kind} {rules}")
+    }
+
+    fn fetch_from_source(
+        &self,
+        canonical_criteria: &[Criterion],
+        canonical_sort: &Option<Vec<AscDesc>>,
+    ) -> String {
+        let rule_name = match self.source {
+            RankingRuleSource::Criterion(index) => {
+                canonical_criteria.get(index).map(|criterion| criterion.to_string())
+            }
+            RankingRuleSource::CoalescedCriteria(begin, end) => {
+                let rules: Vec<String> = canonical_criteria
+                    .get(begin..=end)
+                    .into_iter()
+                    .flat_map(|criteria| criteria.iter())
+                    .map(|criterion| criterion.to_string())
+                    .collect();
+
+                (!rules.is_empty()).then_some(rules.join(", "))
+            }
+            RankingRuleSource::Sort { criterion_index: _, sort_index } => canonical_sort
+                .as_deref()
+                .and_then(|canonical_sort| canonical_sort.get(sort_index))
+                .and_then(|asc_desc: &AscDesc| match asc_desc {
+                    AscDesc::Asc(Member::Field(s)) | AscDesc::Desc(Member::Field(s)) => {
+                        Some(format!("on field `{s}`"))
+                    }
+                    _ => None,
+                }),
+        };
+
+        let rule_name = rule_name.unwrap_or_else(|| "default".into());
+
+        format!("rule(s) {rule_name}")
+    }
+}

meilisearch/tests/auth/authorization.rs

@@ -78,7 +78,7 @@ pub static ALL_ACTIONS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
 });
 
 static INVALID_RESPONSE: Lazy<Value> = Lazy::new(|| {
-    json!({"message": "The provided API key is invalid.",
+    json!({"message": null,
         "code": "invalid_api_key",
         "type": "auth",
         "link": "https://docs.meilisearch.com/errors#invalid_api_key"
@@ -119,7 +119,8 @@ async fn error_access_expired_key() {
     thread::sleep(time::Duration::new(1, 0));
 
     for (method, route) in AUTHORIZATIONS.keys() {
-        let (response, code) = server.dummy_request(method, route).await;
+        let (mut response, code) = server.dummy_request(method, route).await;
+        response["message"] = serde_json::json!(null);
 
         assert_eq!(response, INVALID_RESPONSE.clone(), "on route: {:?} - {:?}", method, route);
         assert_eq!(403, code, "{:?}", &response);
@@ -149,7 +150,8 @@ async fn error_access_unauthorized_index() {
         // filter `products` index routes
         .filter(|(_, route)| route.starts_with("/indexes/products"))
     {
-        let (response, code) = server.dummy_request(method, route).await;
+        let (mut response, code) = server.dummy_request(method, route).await;
+        response["message"] = serde_json::json!(null);
 
         assert_eq!(response, INVALID_RESPONSE.clone(), "on route: {:?} - {:?}", method, route);
         assert_eq!(403, code, "{:?}", &response);
@@ -176,7 +178,8 @@ async fn error_access_unauthorized_action() {
 
         let key = response["key"].as_str().unwrap();
         server.use_api_key(key);
-        let (response, code) = server.dummy_request(method, route).await;
+        let (mut response, code) = server.dummy_request(method, route).await;
+        response["message"] = serde_json::json!(null);
 
         assert_eq!(response, INVALID_RESPONSE.clone(), "on route: {:?} - {:?}", method, route);
         assert_eq!(403, code, "{:?}", &response);
@@ -280,7 +283,7 @@ async fn access_authorized_no_index_restriction() {
                 route,
                 action
             );
-            assert_ne!(code, 403);
+            assert_ne!(code, 403, "on route: {:?} - {:?} with action: {:?}", method, route, action);
         }
     }
 }

meilisearch/tests/auth/errors.rs

@@ -1,7 +1,10 @@
+use actix_web::http::StatusCode;
+use actix_web::test;
+use jsonwebtoken::{EncodingKey, Header};
 use meili_snap::*;
 use uuid::Uuid;
 
-use crate::common::Server;
+use crate::common::{Server, Value};
 use crate::json;
 
 #[actix_rt::test]
@@ -436,3 +439,262 @@ async fn patch_api_keys_unknown_field() {
     }
     "###);
 }
+
+async fn send_request_with_custom_auth(
+    app: impl actix_web::dev::Service<
+        actix_http::Request,
+        Response = actix_web::dev::ServiceResponse<impl actix_web::body::MessageBody>,
+        Error = actix_web::Error,
+    >,
+    url: &str,
+    auth: &str,
+) -> (Value, StatusCode) {
+    let req = test::TestRequest::get().uri(url).insert_header(("Authorization", auth)).to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+
+    (response, status_code)
+}
+
+#[actix_rt::test]
+async fn invalid_auth_format() {
+    let server = Server::new_auth().await;
+    let app = server.init_web_app().await;
+
+    let req = test::TestRequest::get().uri("/indexes/dog/documents").to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    snapshot!(status_code, @"401 Unauthorized");
+    snapshot!(response, @r###"
+    {
+      "message": "The Authorization header is missing. It must use the bearer authorization method.",
+      "code": "missing_authorization_header",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#missing_authorization_header"
+    }
+    "###);
+
+    let req = test::TestRequest::get().uri("/indexes/dog/documents").to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    snapshot!(status_code, @"401 Unauthorized");
+    snapshot!(response, @r###"
+    {
+      "message": "The Authorization header is missing. It must use the bearer authorization method.",
+      "code": "missing_authorization_header",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#missing_authorization_header"
+    }
+    "###);
+
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/documents", "Bearer").await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "The provided API key is invalid.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+}
+
+#[actix_rt::test]
+async fn invalid_api_key() {
+    let server = Server::new_auth().await;
+    let app = server.init_web_app().await;
+
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/search", "Bearer kefir").await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "The provided API key is invalid.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+
+    let uuid = Uuid::nil();
+    let key = json!({ "actions": ["search"], "indexes": ["dog"], "expiresAt": null, "uid": uuid.to_string() });
+    let req = test::TestRequest::post()
+        .uri("/keys")
+        .insert_header(("Authorization", "Bearer MASTER_KEY"))
+        .set_json(&key)
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    snapshot!(json_string!(response, { ".createdAt" => "[date]",  ".updatedAt" => "[date]" }), @r###"
+    {
+      "name": null,
+      "description": null,
+      "key": "aeb94973e0b6e912d94165430bbe87dee91a7c4f891ce19050c3910ec96977e9",
+      "uid": "00000000-0000-0000-0000-000000000000",
+      "actions": [
+        "search"
+      ],
+      "indexes": [
+        "dog"
+      ],
+      "expiresAt": null,
+      "createdAt": "[date]",
+      "updatedAt": "[date]"
+    }
+    "###);
+    let key = response["key"].as_str().unwrap();
+
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/doggo/search", &format!("Bearer {key}"))
+            .await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "The API key cannot acces the index `doggo`, authorized indexes are [\"dog\"].",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+}
+
+#[actix_rt::test]
+async fn invalid_tenant_token() {
+    let server = Server::new_auth().await;
+    let app = server.init_web_app().await;
+
+    // The tenant token won't be recognized at all if we're not on a search route
+    let claims = json!({ "tamo": "kefir" });
+    let jwt = jsonwebtoken::encode(&Header::default(), &claims, &EncodingKey::from_secret(b"tamo"))
+        .unwrap();
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/documents", &format!("Bearer {jwt}"))
+            .await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "The provided API key is invalid.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+
+    let claims = json!({ "tamo": "kefir" });
+    let jwt = jsonwebtoken::encode(&Header::default(), &claims, &EncodingKey::from_secret(b"tamo"))
+        .unwrap();
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/search", &format!("Bearer {jwt}")).await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "Could not decode tenant token, JSON error: missing field `searchRules` at line 1 column 16.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+
+    // The error messages are not ideal but that's expected since we cannot _yet_ use deserr
+    let claims = json!({ "searchRules": "kefir" });
+    let jwt = jsonwebtoken::encode(&Header::default(), &claims, &EncodingKey::from_secret(b"tamo"))
+        .unwrap();
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/search", &format!("Bearer {jwt}")).await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "Could not decode tenant token, JSON error: data did not match any variant of untagged enum SearchRules at line 1 column 23.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+
+    let uuid = Uuid::nil();
+    let claims = json!({ "searchRules": ["kefir"], "apiKeyUid": uuid.to_string() });
+    let jwt = jsonwebtoken::encode(&Header::default(), &claims, &EncodingKey::from_secret(b"tamo"))
+        .unwrap();
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/search", &format!("Bearer {jwt}")).await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "Could not decode tenant token, InvalidSignature.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+
+    // ~~ For the next tests we first need a valid API key
+    let key = json!({ "actions": ["search"], "indexes": ["dog"], "expiresAt": null, "uid": uuid.to_string() });
+    let req = test::TestRequest::post()
+        .uri("/keys")
+        .insert_header(("Authorization", "Bearer MASTER_KEY"))
+        .set_json(&key)
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    snapshot!(json_string!(response, { ".createdAt" => "[date]",  ".updatedAt" => "[date]" }), @r###"
+    {
+      "name": null,
+      "description": null,
+      "key": "aeb94973e0b6e912d94165430bbe87dee91a7c4f891ce19050c3910ec96977e9",
+      "uid": "00000000-0000-0000-0000-000000000000",
+      "actions": [
+        "search"
+      ],
+      "indexes": [
+        "dog"
+      ],
+      "expiresAt": null,
+      "createdAt": "[date]",
+      "updatedAt": "[date]"
+    }
+    "###);
+    let key = response["key"].as_str().unwrap();
+
+    let claims = json!({ "searchRules": ["doggo", "catto"], "apiKeyUid": uuid.to_string() });
+    let jwt = jsonwebtoken::encode(
+        &Header::default(),
+        &claims,
+        &EncodingKey::from_secret(key.as_bytes()),
+    )
+    .unwrap();
+    // Try to access an index that is not authorized by the tenant token
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/search", &format!("Bearer {jwt}")).await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "The provided tenant token cannot acces the index `dog`, allowed indexes are [\"catto\", \"doggo\"].",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+
+    // Try to access an index that *is* authorized by the tenant token but not by the api key used to generate the tt
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/doggo/search", &format!("Bearer {jwt}"))
+            .await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "The API key used to generate this tenant token cannot acces the index `doggo`.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+}

meilisearch/tests/auth/tenant_token.rs

@@ -53,7 +53,8 @@ static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
 });
 
 static INVALID_RESPONSE: Lazy<Value> = Lazy::new(|| {
-    json!({"message": "The provided API key is invalid.",
+    json!({
+        "message": null,
         "code": "invalid_api_key",
         "type": "auth",
         "link": "https://docs.meilisearch.com/errors#invalid_api_key"
@@ -191,7 +192,9 @@ macro_rules! compute_forbidden_search {
                 server.use_api_key(&web_token);
                 let index = server.index("sales");
                 index
-                    .search(json!({}), |response, code| {
+                    .search(json!({}), |mut response, code| {
+                        // We don't assert anything on the message since it may change between cases
+                        response["message"] = serde_json::json!(null);
                         assert_eq!(
                             response,
                             INVALID_RESPONSE.clone(),
@@ -495,7 +498,8 @@ async fn error_access_forbidden_routes() {
 
     for ((method, route), actions) in AUTHORIZATIONS.iter() {
         if !actions.contains("search") {
-            let (response, code) = server.dummy_request(method, route).await;
+            let (mut response, code) = server.dummy_request(method, route).await;
+            response["message"] = serde_json::json!(null);
             assert_eq!(response, INVALID_RESPONSE.clone());
             assert_eq!(code, 403);
         }
@@ -529,14 +533,16 @@ async fn error_access_expired_parent_key() {
     server.use_api_key(&web_token);
 
     // test search request while parent_key is not expired
-    let (response, code) = server.dummy_request("POST", "/indexes/products/search").await;
+    let (mut response, code) = server.dummy_request("POST", "/indexes/products/search").await;
+    response["message"] = serde_json::json!(null);
     assert_ne!(response, INVALID_RESPONSE.clone());
     assert_ne!(code, 403);
 
     // wait until the key is expired.
     thread::sleep(time::Duration::new(1, 0));
 
-    let (response, code) = server.dummy_request("POST", "/indexes/products/search").await;
+    let (mut response, code) = server.dummy_request("POST", "/indexes/products/search").await;
+    response["message"] = serde_json::json!(null);
     assert_eq!(response, INVALID_RESPONSE.clone());
     assert_eq!(code, 403);
 }
@@ -585,7 +591,8 @@ async fn error_access_modified_token() {
     .join(".");
 
     server.use_api_key(&altered_token);
-    let (response, code) = server.dummy_request("POST", "/indexes/products/search").await;
+    let (mut response, code) = server.dummy_request("POST", "/indexes/products/search").await;
+    response["message"] = serde_json::json!(null);
     assert_eq!(response, INVALID_RESPONSE.clone());
     assert_eq!(code, 403);
 }

meilisearch/tests/auth/tenant_token_multi_search.rs

@@ -109,9 +109,11 @@ static NESTED_DOCUMENTS: Lazy<Value> = Lazy::new(|| {
 
 fn invalid_response(query_index: Option<usize>) -> Value {
     let message = if let Some(query_index) = query_index {
-        format!("Inside `.queries[{query_index}]`: The provided API key is invalid.")
+        json!(format!("Inside `.queries[{query_index}]`: The provided API key is invalid."))
     } else {
-        "The provided API key is invalid.".to_string()
+        // if it's anything else we simply return null and will tests all the
+        // error messages somewhere else
+        json!(null)
     };
     json!({"message": message,
         "code": "invalid_api_key",
@@ -308,6 +310,23 @@ macro_rules! compute_authorized_single_search {
                     tenant_token,
                     key_content
                 );
+
+                // federated
+                let (response, code) = server.multi_search(json!({"federation": {}, "queries" : [{"indexUid": "sales", "filter": $filter}]})).await;
+                assert_eq!(
+                    200, code,
+                    "{} using tenant_token: {:?} generated with parent_key: {:?}",
+                    response, tenant_token, key_content
+                );
+                assert_eq!(
+                    // same count as the search is federated over a single query
+                    $expected_count,
+                    response["hits"].as_array().unwrap().len(),
+                    "{} using tenant_token: {:?} generated with parent_key: {:?}",
+                    response,
+                    tenant_token,
+                    key_content
+                );
             }
         }
     };
@@ -373,6 +392,25 @@ macro_rules! compute_authorized_multiple_search {
                     tenant_token,
                     key_content
                 );
+
+                let (response, code) = server.multi_search(json!({"federation": {}, "queries" : [
+                    {"indexUid": "sales", "filter": $filter1},
+                    {"indexUid": "products", "filter": $filter2},
+                ]})).await;
+                assert_eq!(
+                    code, 200,
+                    "{} using tenant_token: {:?} generated with parent_key: {:?}",
+                    response, tenant_token, key_content
+                );
+                assert_eq!(
+                    response["hits"].as_array().unwrap().len(),
+                    // sum of counts as the search is federated across to queries in different indexes
+                    $expected_count1 + $expected_count2,
+                    "{} using tenant_token: {:?} generated with parent_key: {:?}",
+                    response,
+                    tenant_token,
+                    key_content
+                );
             }
         }
     };
@@ -414,7 +452,28 @@ macro_rules! compute_forbidden_single_search {
             for (tenant_token, failed_query_index) in $tenant_tokens.iter().zip(failed_query_indexes.into_iter()) {
                 let web_token = generate_tenant_token(&uid, &key, tenant_token.clone());
                 server.use_api_key(&web_token);
-                let (response, code) = server.multi_search(json!({"queries" : [{"indexUid": "sales"}]})).await;
+                let (mut response, code) = server.multi_search(json!({"queries" : [{"indexUid": "sales"}]})).await;
+                if failed_query_index.is_none() && !response["message"].is_null() {
+                    response["message"] = serde_json::json!(null);
+                }
+                assert_eq!(
+                    response,
+                    invalid_response(failed_query_index),
+                    "{} using tenant_token: {:?} generated with parent_key: {:?}",
+                    response,
+                    tenant_token,
+                    key_content
+                );
+                assert_eq!(
+                    code, 403,
+                    "{} using tenant_token: {:?} generated with parent_key: {:?}",
+                    response, tenant_token, key_content
+                );
+
+                let (mut response, code) = server.multi_search(json!({"federation": {}, "queries" : [{"indexUid": "sales"}]})).await;
+                if failed_query_index.is_none() && !response["message"].is_null() {
+                    response["message"] = serde_json::json!(null);
+                }
                 assert_eq!(
                     response,
                     invalid_response(failed_query_index),
@@ -469,10 +528,34 @@ macro_rules! compute_forbidden_multiple_search {
             for (tenant_token, failed_query_index) in $tenant_tokens.iter().zip(failed_query_indexes.into_iter()) {
                 let web_token = generate_tenant_token(&uid, &key, tenant_token.clone());
                 server.use_api_key(&web_token);
-                let (response, code) = server.multi_search(json!({"queries" : [
+                let (mut response, code) = server.multi_search(json!({"queries" : [
                     {"indexUid": "sales"},
                     {"indexUid": "products"},
                 ]})).await;
+                if failed_query_index.is_none() && !response["message"].is_null() {
+                    response["message"] = serde_json::json!(null);
+                }
+                assert_eq!(
+                    response,
+                    invalid_response(failed_query_index),
+                    "{} using tenant_token: {:?} generated with parent_key: {:?}",
+                    response,
+                    tenant_token,
+                    key_content
+                );
+                assert_eq!(
+                    code, 403,
+                    "{} using tenant_token: {:?} generated with parent_key: {:?}",
+                    response, tenant_token, key_content
+                );
+
+                let (mut response, code) = server.multi_search(json!({"federation": {}, "queries" : [
+                    {"indexUid": "sales"},
+                    {"indexUid": "products"},
+                ]})).await;
+                if failed_query_index.is_none() && !response["message"].is_null() {
+                    response["message"] = serde_json::json!(null);
+                }
                 assert_eq!(
                     response,
                     invalid_response(failed_query_index),
@@ -1073,18 +1156,20 @@ async fn error_access_expired_parent_key() {
     server.use_api_key(&web_token);
 
     // test search request while parent_key is not expired
-    let (response, code) = server
+    let (mut response, code) = server
         .multi_search(json!({"queries" : [{"indexUid": "sales"}, {"indexUid": "products"}]}))
         .await;
+    response["message"] = serde_json::json!(null);
     assert_ne!(response, invalid_response(None));
     assert_ne!(code, 403);
 
     // wait until the key is expired.
     thread::sleep(time::Duration::new(1, 0));
 
-    let (response, code) = server
+    let (mut response, code) = server
         .multi_search(json!({"queries" : [{"indexUid": "sales"}, {"indexUid": "products"}]}))
         .await;
+    response["message"] = serde_json::json!(null);
     assert_eq!(response, invalid_response(None));
     assert_eq!(code, 403);
 }
@@ -1134,8 +1219,9 @@ async fn error_access_modified_token() {
     .join(".");
 
     server.use_api_key(&altered_token);
-    let (response, code) =
+    let (mut response, code) =
         server.multi_search(json!({"queries" : [{"indexUid": "products"}]})).await;
+    response["message"] = serde_json::json!(null);
     assert_eq!(response, invalid_response(None));
     assert_eq!(code, 403);
 }

meilisearch/tests/common/index.rs

@@ -182,14 +182,10 @@ impl Index<'_> {
         self.service.get(url).await
     }
 
-    pub async fn get_document(
-        &self,
-        id: u64,
-        options: Option<GetDocumentOptions>,
-    ) -> (Value, StatusCode) {
+    pub async fn get_document(&self, id: u64, options: Option<Value>) -> (Value, StatusCode) {
         let mut url = format!("/indexes/{}/documents/{}", urlencode(self.uid.as_ref()), id);
-        if let Some(fields) = options.and_then(|o| o.fields) {
-            let _ = write!(url, "?fields={}", fields.join(","));
+        if let Some(options) = options {
+            write!(url, "{}", yaup::to_string(&options).unwrap()).unwrap();
         }
         self.service.get(url).await
     }
@@ -205,18 +201,11 @@ impl Index<'_> {
     }
 
     pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
-        let mut url = format!("/indexes/{}/documents?", urlencode(self.uid.as_ref()));
-        if let Some(limit) = options.limit {
-            let _ = write!(url, "limit={}&", limit);
-        }
-
-        if let Some(offset) = options.offset {
-            let _ = write!(url, "offset={}&", offset);
-        }
-
-        if let Some(attributes_to_retrieve) = options.attributes_to_retrieve {
-            let _ = write!(url, "fields={}&", attributes_to_retrieve.join(","));
-        }
+        let url = format!(
+            "/indexes/{}/documents{}",
+            urlencode(self.uid.as_ref()),
+            yaup::to_string(&options).unwrap()
+        );
 
         self.service.get(url).await
     }
@@ -376,7 +365,7 @@ impl Index<'_> {
     }
 
     pub async fn search_get(&self, query: &str) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/search?{}", urlencode(self.uid.as_ref()), query);
+        let url = format!("/indexes/{}/search{}", urlencode(self.uid.as_ref()), query);
         self.service.get(url).await
     }
 
@@ -413,7 +402,7 @@ impl Index<'_> {
     }
 
     pub async fn similar_get(&self, query: &str) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/similar?{}", urlencode(self.uid.as_ref()), query);
+        let url = format!("/indexes/{}/similar{}", urlencode(self.uid.as_ref()), query);
         self.service.get(url).await
     }
 
@@ -435,13 +424,14 @@ impl Index<'_> {
     }
 }
 
-pub struct GetDocumentOptions {
-    pub fields: Option<Vec<&'static str>>,
-}
-
-#[derive(Debug, Default)]
+#[derive(Debug, Default, serde::Serialize)]
+#[serde(rename_all = "camelCase")]
 pub struct GetAllDocumentsOptions {
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub limit: Option<usize>,
+    #[serde(skip_serializing_if = "Option::is_none")]
     pub offset: Option<usize>,
-    pub attributes_to_retrieve: Option<Vec<&'static str>>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub fields: Option<Vec<&'static str>>,
+    pub retrieve_vectors: bool,
 }

meilisearch/tests/common/mod.rs

@@ -6,7 +6,7 @@ pub mod service;
 use std::fmt::{self, Display};
 
 #[allow(unused)]
-pub use index::{GetAllDocumentsOptions, GetDocumentOptions};
+pub use index::GetAllDocumentsOptions;
 use meili_snap::json_string;
 use serde::{Deserialize, Serialize};
 #[allow(unused)]
@@ -42,6 +42,12 @@ impl std::ops::Deref for Value {
     }
 }
 
+impl std::ops::DerefMut for Value {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.0
+    }
+}
+
 impl PartialEq<serde_json::Value> for Value {
     fn eq(&self, other: &serde_json::Value) -> bool {
         &self.0 == other
@@ -65,7 +71,7 @@ impl Display for Value {
         write!(
             f,
             "{}",
-            json_string!(self, { ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" })
+            json_string!(self, { ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]", ".processingTimeMs" => "[duration]" })
         )
     }
 }

meilisearch/tests/common/server.rs

@@ -6,7 +6,7 @@ use std::time::Duration;
 use actix_http::body::MessageBody;
 use actix_web::dev::ServiceResponse;
 use actix_web::http::StatusCode;
-use byte_unit::{Byte, ByteUnit};
+use byte_unit::{Byte, Unit};
 use clap::Parser;
 use meilisearch::option::{IndexerOpts, MaxMemory, Opt};
 use meilisearch::{analytics, create_app, setup_meilisearch, SubscriberForSecondLayer};
@@ -231,9 +231,9 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
         env: "development".to_owned(),
         #[cfg(feature = "analytics")]
         no_analytics: true,
-        max_index_size: Byte::from_unit(100.0, ByteUnit::MiB).unwrap(),
-        max_task_db_size: Byte::from_unit(1.0, ByteUnit::GiB).unwrap(),
-        http_payload_size_limit: Byte::from_unit(10.0, ByteUnit::MiB).unwrap(),
+        max_index_size: Byte::from_u64_with_unit(100, Unit::MiB).unwrap(),
+        max_task_db_size: Byte::from_u64_with_unit(1, Unit::GiB).unwrap(),
+        http_payload_size_limit: Byte::from_u64_with_unit(10, Unit::MiB).unwrap(),
         snapshot_dir: ".".into(),
         indexer_options: IndexerOpts {
             // memory has to be unlimited because several meilisearch are running in test context.

meilisearch/tests/documents/add_documents.rs

@@ -183,6 +183,58 @@ async fn add_single_document_gzip_encoded() {
     }
     "###);
 }
+#[actix_rt::test]
+async fn add_single_document_gzip_encoded_with_incomplete_error() {
+    let document = json!("kefir");
+
+    // this is a what is expected and should work
+    let server = Server::new().await;
+    let app = server.init_web_app().await;
+    // post
+    let document = serde_json::to_string(&document).unwrap();
+    let req = test::TestRequest::post()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "application/json"))
+        .insert_header(("content-encoding", "gzip"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    snapshot!(status_code, @"400 Bad Request");
+    snapshot!(json_string!(response),
+        @r###"
+    {
+      "message": "The provided payload is incomplete and cannot be parsed",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+
+    // put
+    let req = test::TestRequest::put()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "application/json"))
+        .insert_header(("content-encoding", "gzip"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    snapshot!(status_code, @"400 Bad Request");
+    snapshot!(json_string!(response),
+        @r###"
+    {
+      "message": "The provided payload is incomplete and cannot be parsed",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+}
 
 /// Here we try document request with every encoding
 #[actix_rt::test]
@@ -1040,6 +1092,52 @@ async fn document_addition_with_primary_key() {
     "###);
 }
 
+#[actix_rt::test]
+async fn document_addition_with_huge_int_primary_key() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = json!([
+        {
+            "primary": 14630868576586246730u64,
+            "content": "foo",
+        }
+    ]);
+    let (response, code) = index.add_documents(documents, Some("primary")).await;
+    snapshot!(code, @"202 Accepted");
+
+    let response = index.wait_task(response.uid()).await;
+    snapshot!(response,
+        @r###"
+    {
+      "uid": 0,
+      "indexUid": "test",
+      "status": "succeeded",
+      "type": "documentAdditionOrUpdate",
+      "canceledBy": null,
+      "details": {
+        "receivedDocuments": 1,
+        "indexedDocuments": 1
+      },
+      "error": null,
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+
+    let (response, code) = index.get_document(14630868576586246730u64, None).await;
+    snapshot!(code, @"200 OK");
+    snapshot!(json_string!(response),
+        @r###"
+    {
+      "primary": 14630868576586246730,
+      "content": "foo"
+    }
+    "###);
+}
+
 #[actix_rt::test]
 async fn replace_document() {
     let server = Server::new().await;
@@ -2176,7 +2274,7 @@ async fn error_add_documents_payload_size() {
     snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]" }),
         @r###"
     {
-      "message": "The provided payload reached the size limit. The maximum accepted payload size is 10.00 MiB.",
+      "message": "The provided payload reached the size limit. The maximum accepted payload size is 10 MiB.",
       "code": "payload_too_large",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#payload_too_large"

meilisearch/tests/documents/errors.rs

@@ -719,7 +719,7 @@ async fn fetch_document_by_filter() {
 
     let (response, code) = index.get_document_by_filter(json!(null)).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Invalid value type: expected an object, but found null",
       "code": "bad_request",
@@ -730,7 +730,7 @@ async fn fetch_document_by_filter() {
 
     let (response, code) = index.get_document_by_filter(json!({ "offset": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Invalid value type at `.offset`: expected a positive integer, but found a string: `\"doggo\"`",
       "code": "invalid_document_offset",
@@ -741,7 +741,7 @@ async fn fetch_document_by_filter() {
 
     let (response, code) = index.get_document_by_filter(json!({ "limit": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Invalid value type at `.limit`: expected a positive integer, but found a string: `\"doggo\"`",
       "code": "invalid_document_limit",
@@ -752,7 +752,7 @@ async fn fetch_document_by_filter() {
 
     let (response, code) = index.get_document_by_filter(json!({ "fields": "doggo" })).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Invalid value type at `.fields`: expected an array, but found a string: `\"doggo\"`",
       "code": "invalid_document_fields",
@@ -763,7 +763,7 @@ async fn fetch_document_by_filter() {
 
     let (response, code) = index.get_document_by_filter(json!({ "filter": true })).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Invalid syntax for the filter parameter: `expected String, Array, found: true`.",
       "code": "invalid_document_filter",
@@ -774,7 +774,7 @@ async fn fetch_document_by_filter() {
 
     let (response, code) = index.get_document_by_filter(json!({ "filter": "cool doggo" })).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `cool doggo`.\n1:11 cool doggo",
       "code": "invalid_document_filter",
@@ -786,7 +786,7 @@ async fn fetch_document_by_filter() {
     let (response, code) =
         index.get_document_by_filter(json!({ "filter": "doggo = bernese" })).await;
     snapshot!(code, @"400 Bad Request");
-    snapshot!(json_string!(response), @r###"
+    snapshot!(response, @r###"
     {
       "message": "Attribute `doggo` is not filterable. Available filterable attributes are: `color`.\n1:6 doggo = bernese",
       "code": "invalid_document_filter",
@@ -795,3 +795,70 @@ async fn fetch_document_by_filter() {
     }
     "###);
 }
+
+#[actix_rt::test]
+async fn retrieve_vectors() {
+    let server = Server::new().await;
+    let index = server.index("doggo");
+
+    // GET ALL DOCUMENTS BY QUERY
+    let (response, _code) = index.get_all_documents_raw("?retrieveVectors=tamo").await;
+    snapshot!(response, @r###"
+    {
+      "message": "Invalid value in parameter `retrieveVectors`: could not parse `tamo` as a boolean, expected either `true` or `false`",
+      "code": "invalid_document_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_document_retrieve_vectors"
+    }
+    "###);
+    let (response, _code) = index.get_all_documents_raw("?retrieveVectors=true").await;
+    snapshot!(response, @r###"
+    {
+      "message": "Passing `retrieveVectors` as a parameter requires enabling the `vector store` experimental feature. See https://github.com/meilisearch/product/discussions/677",
+      "code": "feature_not_enabled",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
+    }
+    "###);
+
+    // FETCH ALL DOCUMENTS BY POST
+    let (response, _code) =
+        index.get_document_by_filter(json!({ "retrieveVectors": "tamo" })).await;
+    snapshot!(response, @r###"
+    {
+      "message": "Invalid value type at `.retrieveVectors`: expected a boolean, but found a string: `\"tamo\"`",
+      "code": "invalid_document_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_document_retrieve_vectors"
+    }
+    "###);
+    let (response, _code) = index.get_document_by_filter(json!({ "retrieveVectors": true })).await;
+    snapshot!(response, @r###"
+    {
+      "message": "Passing `retrieveVectors` as a parameter requires enabling the `vector store` experimental feature. See https://github.com/meilisearch/product/discussions/677",
+      "code": "feature_not_enabled",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
+    }
+    "###);
+
+    // GET A SINGLE DOCUMENT
+    let (response, _code) = index.get_document(0, Some(json!({"retrieveVectors": "tamo"}))).await;
+    snapshot!(response, @r###"
+    {
+      "message": "Invalid value in parameter `retrieveVectors`: could not parse `tamo` as a boolean, expected either `true` or `false`",
+      "code": "invalid_document_retrieve_vectors",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_document_retrieve_vectors"
+    }
+    "###);
+    let (response, _code) = index.get_document(0, Some(json!({"retrieveVectors": true}))).await;
+    snapshot!(response, @r###"
+    {
+      "message": "Passing `retrieveVectors` as a parameter requires enabling the `vector store` experimental feature. See https://github.com/meilisearch/product/discussions/677",
+      "code": "feature_not_enabled",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
+    }
+    "###);
+}

meilisearch/tests/documents/get_documents.rs

@@ -1,10 +1,10 @@
+use actix_web::http::header::ACCEPT_ENCODING;
 use actix_web::test;
-use http::header::ACCEPT_ENCODING;
 use meili_snap::*;
 use urlencoding::encode as urlencode;
 
 use crate::common::encoder::Encoder;
-use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server, Value};
+use crate::common::{GetAllDocumentsOptions, Server, Value};
 use crate::json;
 
 // TODO: partial test since we are testing error, amd error is not yet fully implemented in
@@ -59,8 +59,7 @@ async fn get_document() {
         })
     );
 
-    let (response, code) =
-        index.get_document(0, Some(GetDocumentOptions { fields: Some(vec!["id"]) })).await;
+    let (response, code) = index.get_document(0, Some(json!({ "fields": ["id"] }))).await;
     assert_eq!(code, 200);
     assert_eq!(
         response,
@@ -69,9 +68,8 @@ async fn get_document() {
         })
     );
 
-    let (response, code) = index
-        .get_document(0, Some(GetDocumentOptions { fields: Some(vec!["nested.content"]) }))
-        .await;
+    let (response, code) =
+        index.get_document(0, Some(json!({ "fields": ["nested.content"] }))).await;
     assert_eq!(code, 200);
     assert_eq!(
         response,
@@ -211,7 +209,7 @@ async fn test_get_all_documents_attributes_to_retrieve() {
 
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
-            attributes_to_retrieve: Some(vec!["name"]),
+            fields: Some(vec!["name"]),
             ..Default::default()
         })
         .await;
@@ -225,9 +223,19 @@ async fn test_get_all_documents_attributes_to_retrieve() {
     assert_eq!(response["limit"], json!(20));
     assert_eq!(response["total"], json!(77));
 
+    let (response, code) = index.get_all_documents_raw("?fields=").await;
+    assert_eq!(code, 200);
+    assert_eq!(response["results"].as_array().unwrap().len(), 20);
+    for results in response["results"].as_array().unwrap() {
+        assert_eq!(results.as_object().unwrap().keys().count(), 0);
+    }
+    assert_eq!(response["offset"], json!(0));
+    assert_eq!(response["limit"], json!(20));
+    assert_eq!(response["total"], json!(77));
+
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
-            attributes_to_retrieve: Some(vec![]),
+            fields: Some(vec!["wrong"]),
             ..Default::default()
         })
         .await;
@@ -242,22 +250,7 @@ async fn test_get_all_documents_attributes_to_retrieve() {
 
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
-            attributes_to_retrieve: Some(vec!["wrong"]),
-            ..Default::default()
-        })
-        .await;
-    assert_eq!(code, 200);
-    assert_eq!(response["results"].as_array().unwrap().len(), 20);
-    for results in response["results"].as_array().unwrap() {
-        assert_eq!(results.as_object().unwrap().keys().count(), 0);
-    }
-    assert_eq!(response["offset"], json!(0));
-    assert_eq!(response["limit"], json!(20));
-    assert_eq!(response["total"], json!(77));
-
-    let (response, code) = index
-        .get_all_documents(GetAllDocumentsOptions {
-            attributes_to_retrieve: Some(vec!["name", "tags"]),
+            fields: Some(vec!["name", "tags"]),
             ..Default::default()
         })
         .await;
@@ -270,10 +263,7 @@ async fn test_get_all_documents_attributes_to_retrieve() {
     }
 
     let (response, code) = index
-        .get_all_documents(GetAllDocumentsOptions {
-            attributes_to_retrieve: Some(vec!["*"]),
-            ..Default::default()
-        })
+        .get_all_documents(GetAllDocumentsOptions { fields: Some(vec!["*"]), ..Default::default() })
         .await;
     assert_eq!(code, 200);
     assert_eq!(response["results"].as_array().unwrap().len(), 20);
@@ -283,7 +273,7 @@ async fn test_get_all_documents_attributes_to_retrieve() {
 
     let (response, code) = index
         .get_all_documents(GetAllDocumentsOptions {
-            attributes_to_retrieve: Some(vec!["*", "wrong"]),
+            fields: Some(vec!["*", "wrong"]),
             ..Default::default()
         })
         .await;
@@ -316,12 +306,10 @@ async fn get_document_s_nested_attributes_to_retrieve() {
     assert_eq!(code, 202);
     index.wait_task(1).await;
 
-    let (response, code) =
-        index.get_document(0, Some(GetDocumentOptions { fields: Some(vec!["content"]) })).await;
+    let (response, code) = index.get_document(0, Some(json!({ "fields": ["content"] }))).await;
     assert_eq!(code, 200);
     assert_eq!(response, json!({}));
-    let (response, code) =
-        index.get_document(1, Some(GetDocumentOptions { fields: Some(vec!["content"]) })).await;
+    let (response, code) = index.get_document(1, Some(json!({ "fields": ["content"] }))).await;
     assert_eq!(code, 200);
     assert_eq!(
         response,
@@ -333,9 +321,7 @@ async fn get_document_s_nested_attributes_to_retrieve() {
         })
     );
 
-    let (response, code) = index
-        .get_document(0, Some(GetDocumentOptions { fields: Some(vec!["content.truc"]) }))
-        .await;
+    let (response, code) = index.get_document(0, Some(json!({ "fields": ["content.truc"] }))).await;
     assert_eq!(code, 200);
     assert_eq!(
         response,
@@ -343,9 +329,7 @@ async fn get_document_s_nested_attributes_to_retrieve() {
             "content.truc": "foobar",
         })
     );
-    let (response, code) = index
-        .get_document(1, Some(GetDocumentOptions { fields: Some(vec!["content.truc"]) }))
-        .await;
+    let (response, code) = index.get_document(1, Some(json!({ "fields": ["content.truc"] }))).await;
     assert_eq!(code, 200);
     assert_eq!(
         response,
@@ -540,3 +524,208 @@ async fn get_document_by_filter() {
     }
     "###);
 }
+
+#[actix_rt::test]
+async fn get_document_with_vectors() {
+    let server = Server::new().await;
+    let index = server.index("doggo");
+    let (value, code) = server.set_features(json!({"vectorStore": true})).await;
+    snapshot!(code, @"200 OK");
+    snapshot!(value, @r###"
+    {
+      "vectorStore": true,
+      "metrics": false,
+      "logsRoute": false,
+      "editDocumentsByFunction": false
+    }
+    "###);
+
+    let (response, code) = index
+        .update_settings(json!({
+          "embedders": {
+              "manual": {
+                  "source": "userProvided",
+                  "dimensions": 3,
+              }
+          },
+        }))
+        .await;
+    snapshot!(code, @"202 Accepted");
+    server.wait_task(response.uid()).await;
+
+    let documents = json!([
+      {"id": 0, "name": "kefir", "_vectors": { "manual": [0, 0, 0] }},
+      {"id": 1, "name": "echo", "_vectors": { "manual": null }},
+    ]);
+    let (value, code) = index.add_documents(documents, None).await;
+    snapshot!(code, @"202 Accepted");
+    index.wait_task(value.uid()).await;
+
+    // by default you shouldn't see the `_vectors` object
+    let (documents, _code) = index.get_all_documents(Default::default()).await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "id": 0,
+          "name": "kefir"
+        },
+        {
+          "id": 1,
+          "name": "echo"
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 2
+    }
+    "###);
+    let (documents, _code) = index.get_document(0, None).await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "id": 0,
+      "name": "kefir"
+    }
+    "###);
+
+    // if we try to retrieve the vectors with the `fields` parameter they
+    // still shouldn't be displayed
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            fields: Some(vec!["name", "_vectors"]),
+            ..Default::default()
+        })
+        .await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "name": "kefir"
+        },
+        {
+          "name": "echo"
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 2
+    }
+    "###);
+    let (documents, _code) =
+        index.get_document(0, Some(json!({"fields": ["name", "_vectors"]}))).await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "name": "kefir"
+    }
+    "###);
+
+    // If we specify the retrieve vectors boolean and nothing else we should get the vectors
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions { retrieve_vectors: true, ..Default::default() })
+        .await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "id": 0,
+          "name": "kefir",
+          "_vectors": {
+            "manual": {
+              "embeddings": [
+                [
+                  0.0,
+                  0.0,
+                  0.0
+                ]
+              ],
+              "regenerate": false
+            }
+          }
+        },
+        {
+          "id": 1,
+          "name": "echo",
+          "_vectors": {}
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 2
+    }
+    "###);
+    let (documents, _code) = index.get_document(0, Some(json!({"retrieveVectors": true}))).await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "id": 0,
+      "name": "kefir",
+      "_vectors": {
+        "manual": {
+          "embeddings": [
+            [
+              0.0,
+              0.0,
+              0.0
+            ]
+          ],
+          "regenerate": false
+        }
+      }
+    }
+    "###);
+
+    // If we specify the retrieve vectors boolean and exclude vectors form the `fields` we should still get the vectors
+    let (documents, _code) = index
+        .get_all_documents(GetAllDocumentsOptions {
+            retrieve_vectors: true,
+            fields: Some(vec!["name"]),
+            ..Default::default()
+        })
+        .await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "results": [
+        {
+          "name": "kefir",
+          "_vectors": {
+            "manual": {
+              "embeddings": [
+                [
+                  0.0,
+                  0.0,
+                  0.0
+                ]
+              ],
+              "regenerate": false
+            }
+          }
+        },
+        {
+          "name": "echo",
+          "_vectors": {}
+        }
+      ],
+      "offset": 0,
+      "limit": 20,
+      "total": 2
+    }
+    "###);
+    let (documents, _code) =
+        index.get_document(0, Some(json!({"retrieveVectors": true, "fields": ["name"]}))).await;
+    snapshot!(json_string!(documents), @r###"
+    {
+      "name": "kefir",
+      "_vectors": {
+        "manual": {
+          "embeddings": [
+            [
+              0.0,
+              0.0,
+              0.0
+            ]
+          ],
+          "regenerate": false
+        }
+      }
+    }
+    "###);
+}

meilisearch/tests/dumps/mod.rs

@@ -1859,7 +1859,8 @@ async fn import_dump_v6_containing_experimental_features() {
     {
       "vectorStore": false,
       "metrics": false,
-      "logsRoute": false
+      "logsRoute": false,
+      "editDocumentsByFunction": false
     }
     "###);
 
@@ -1938,3 +1939,212 @@ async fn import_dump_v6_containing_experimental_features() {
         })
         .await;
 }
+
+// In this test we must generate the dump ourselves to ensure the
+// `user provided` vectors are well set
+#[actix_rt::test]
+#[cfg_attr(target_os = "windows", ignore)]
+async fn generate_and_import_dump_containing_vectors() {
+    let temp = tempfile::tempdir().unwrap();
+    let mut opt = default_settings(temp.path());
+    let server = Server::new_with_options(opt.clone()).await.unwrap();
+    let (code, _) = server.set_features(json!({"vectorStore": true})).await;
+    snapshot!(code, @r###"
+    {
+      "vectorStore": true,
+      "metrics": false,
+      "logsRoute": false,
+      "editDocumentsByFunction": false
+    }
+    "###);
+    let index = server.index("pets");
+    let (response, code) = index
+        .update_settings(json!(
+        {
+            "embedders": {
+                "doggo_embedder": {
+                    "source": "huggingFace",
+                    "model": "sentence-transformers/all-MiniLM-L6-v2",
+                    "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+                    "documentTemplate": "{{doc.doggo}}",
+                }
+            }
+        }
+        ))
+        .await;
+    snapshot!(code, @"202 Accepted");
+    let response = index.wait_task(response.uid()).await;
+    snapshot!(response);
+    let (response, code) = index
+        .add_documents(
+            json!([
+                {"id": 0, "doggo": "kefir", "_vectors": { "doggo_embedder": vec![0; 384] }},
+                {"id": 1, "doggo": "echo", "_vectors": { "doggo_embedder": { "regenerate": false, "embeddings": vec![1; 384] }}},
+                {"id": 2, "doggo": "intel", "_vectors": { "doggo_embedder": { "regenerate": true, "embeddings": vec![2; 384] }}},
+                {"id": 3, "doggo": "bill", "_vectors": { "doggo_embedder": { "regenerate": true }}},
+                {"id": 4, "doggo": "max" },
+            ]),
+            None,
+        )
+        .await;
+    snapshot!(code, @"202 Accepted");
+    let response = index.wait_task(response.uid()).await;
+    snapshot!(response);
+
+    let (response, code) = server.create_dump().await;
+    snapshot!(code, @"202 Accepted");
+    let response = index.wait_task(response.uid()).await;
+    snapshot!(response["status"], @r###""succeeded""###);
+
+    // ========= We made a dump, now we should clear the DB and try to import our dump
+    drop(server);
+    tokio::fs::remove_dir_all(&opt.db_path).await.unwrap();
+    let dump_name = format!("{}.dump", response["details"]["dumpUid"].as_str().unwrap());
+    let dump_path = opt.dump_dir.join(dump_name);
+    assert!(dump_path.exists(), "path: `{}`", dump_path.display());
+
+    opt.import_dump = Some(dump_path);
+    // NOTE: We shouldn't have to change the database path but I lost one hour
+    // because of a « bad path » error and that fixed it.
+    opt.db_path = temp.path().join("data.ms");
+
+    let mut server = Server::new_auth_with_options(opt, temp).await;
+    server.use_api_key("MASTER_KEY");
+
+    let (indexes, code) = server.list_indexes(None, None).await;
+    assert_eq!(code, 200, "{indexes}");
+
+    snapshot!(indexes["results"].as_array().unwrap().len(), @"1");
+    snapshot!(indexes["results"][0]["uid"], @r###""pets""###);
+    snapshot!(indexes["results"][0]["primaryKey"], @r###""id""###);
+
+    let (response, code) = server.get_features().await;
+    meili_snap::snapshot!(code, @"200 OK");
+    meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
+    {
+      "vectorStore": true,
+      "metrics": false,
+      "logsRoute": false,
+      "editDocumentsByFunction": false
+    }
+    "###);
+
+    let index = server.index("pets");
+
+    let (response, code) = index.settings().await;
+    meili_snap::snapshot!(code, @"200 OK");
+    meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
+    {
+      "displayedAttributes": [
+        "*"
+      ],
+      "searchableAttributes": [
+        "*"
+      ],
+      "filterableAttributes": [],
+      "sortableAttributes": [],
+      "rankingRules": [
+        "words",
+        "typo",
+        "proximity",
+        "attribute",
+        "sort",
+        "exactness"
+      ],
+      "stopWords": [],
+      "nonSeparatorTokens": [],
+      "separatorTokens": [],
+      "dictionary": [],
+      "synonyms": {},
+      "distinctAttribute": null,
+      "proximityPrecision": "byWord",
+      "typoTolerance": {
+        "enabled": true,
+        "minWordSizeForTypos": {
+          "oneTypo": 5,
+          "twoTypos": 9
+        },
+        "disableOnWords": [],
+        "disableOnAttributes": []
+      },
+      "faceting": {
+        "maxValuesPerFacet": 100,
+        "sortFacetValuesBy": {
+          "*": "alpha"
+        }
+      },
+      "pagination": {
+        "maxTotalHits": 1000
+      },
+      "embedders": {
+        "doggo_embedder": {
+          "source": "huggingFace",
+          "model": "sentence-transformers/all-MiniLM-L6-v2",
+          "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+          "documentTemplate": "{{doc.doggo}}"
+        }
+      },
+      "searchCutoffMs": null
+    }
+    "###);
+
+    index
+        .search(json!({"retrieveVectors": true}), |response, code| {
+            snapshot!(code, @"200 OK");
+            snapshot!(json_string!(response["hits"], { "[]._vectors.doggo_embedder.embeddings" => "[vector]" }), @r###"
+            [
+              {
+                "id": 0,
+                "doggo": "kefir",
+                "_vectors": {
+                  "doggo_embedder": {
+                    "embeddings": "[vector]",
+                    "regenerate": false
+                  }
+                }
+              },
+              {
+                "id": 1,
+                "doggo": "echo",
+                "_vectors": {
+                  "doggo_embedder": {
+                    "embeddings": "[vector]",
+                    "regenerate": false
+                  }
+                }
+              },
+              {
+                "id": 2,
+                "doggo": "intel",
+                "_vectors": {
+                  "doggo_embedder": {
+                    "embeddings": "[vector]",
+                    "regenerate": true
+                  }
+                }
+              },
+              {
+                "id": 3,
+                "doggo": "bill",
+                "_vectors": {
+                  "doggo_embedder": {
+                    "embeddings": "[vector]",
+                    "regenerate": true
+                  }
+                }
+              },
+              {
+                "id": 4,
+                "doggo": "max",
+                "_vectors": {
+                  "doggo_embedder": {
+                    "embeddings": "[vector]",
+                    "regenerate": true
+                  }
+                }
+              }
+            ]
+            "###);
+        })
+        .await;
+}

meilisearch/tests/dumps/snapshots/mod.rs/generate_and_import_dump_containing_vectors/1.snap

@@ -0,0 +1,25 @@
+---
+source: meilisearch/tests/dumps/mod.rs
+---
+{
+  "uid": 0,
+  "indexUid": "pets",
+  "status": "succeeded",
+  "type": "settingsUpdate",
+  "canceledBy": null,
+  "details": {
+    "embedders": {
+      "doggo_embedder": {
+        "source": "huggingFace",
+        "model": "sentence-transformers/all-MiniLM-L6-v2",
+        "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+        "documentTemplate": "{{doc.doggo}}"
+      }
+    }
+  },
+  "error": null,
+  "duration": "[duration]",
+  "enqueuedAt": "[date]",
+  "startedAt": "[date]",
+  "finishedAt": "[date]"
+}

meilisearch/tests/dumps/snapshots/mod.rs/generate_and_import_dump_containing_vectors/2.snap

@@ -0,0 +1,19 @@
+---
+source: meilisearch/tests/dumps/mod.rs
+---
+{
+  "uid": 1,
+  "indexUid": "pets",
+  "status": "succeeded",
+  "type": "documentAdditionOrUpdate",
+  "canceledBy": null,
+  "details": {
+    "receivedDocuments": 5,
+    "indexedDocuments": 5
+  },
+  "error": null,
+  "duration": "[duration]",
+  "enqueuedAt": "[date]",
+  "startedAt": "[date]",
+  "finishedAt": "[date]"
+}