Use an experimental feature to avoid copying 64k in memory

4786: Update dependencies r=Kerollmops a=irevoire

# Pull Request

## Related issue
Fixes #4753

## What does this PR do?
- Update all dependencies except rustls
- [x] Release charabia
- [x] Update charabia
- [x] Double check that the docker build works after updating charabia



Co-authored-by: Tamo <tamo@meilisearch.com>
Co-authored-by: Clément Renault <clement@meilisearch.com>

.github/workflows/bench-manual.yml

@@ -18,11 +18,9 @@ jobs:
         timeout-minutes: 180 # 3h
         steps:
             - uses: actions/checkout@v3
-            - uses: actions-rs/toolchain@v1
+            - uses: helix-editor/rust-toolchain@v1
               with:
                 profile: minimal
-                toolchain: stable
-                override: true
 
             - name: Run benchmarks - workload ${WORKLOAD_NAME} - branch ${{ github.ref }} - commit ${{ github.sha }}
               run: |

.github/workflows/bench-pr.yml

@@ -35,12 +35,17 @@ jobs:
             fetch-depth: 0 # fetch full history to be able to get main commit sha
             ref: ${{ steps.comment-branch.outputs.head_ref }}
 
-        - uses: actions-rs/toolchain@v1
+        - uses: helix-editor/rust-toolchain@v1
           with:
             profile: minimal
-            toolchain: stable
-            override: true
 
         - name: Run benchmarks on PR ${{ github.event.issue.id }}
           run: |
-            cargo xtask bench --api-key "${{ secrets.BENCHMARK_API_KEY }}" --dashboard-url "${{ vars.BENCHMARK_DASHBOARD_URL }}" --reason "[Comment](${{ github.event.comment.html_url }}) on [#${{ github.event.issue.number }}](${{ github.event.issue.html_url }})" -- ${{ steps.command.outputs.command-arguments }}
+            cargo xtask bench --api-key "${{ secrets.BENCHMARK_API_KEY }}" \
+               --dashboard-url "${{ vars.BENCHMARK_DASHBOARD_URL }}" \
+               --reason "[Comment](${{ github.event.comment.html_url }}) on [#${{ github.event.issue.number }}](${{ github.event.issue.html_url }})" \
+               -- ${{ steps.command.outputs.command-arguments }} > benchlinks.txt
+
+        - name: Send comment in PR
+          run: |
+            gh pr comment ${{github.event.issue.number}} --body-file benchlinks.txt

.github/workflows/bench-push-indexing.yml

@@ -12,11 +12,9 @@ jobs:
         timeout-minutes: 180 # 3h
         steps:
           - uses: actions/checkout@v3
-          - uses: actions-rs/toolchain@v1
+          - uses: helix-editor/rust-toolchain@v1
             with:
               profile: minimal
-              toolchain: stable
-              override: true
 
           # Run benchmarks
           - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch main - Commit ${{ github.sha }}

.github/workflows/benchmarks-manual.yml

@@ -18,11 +18,9 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/benchmarks-pr.yml

@@ -13,11 +13,9 @@ jobs:
     runs-on: benchmarks
     timeout-minutes: 4320 # 72h
     steps:
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       - name: Check for Command
         id: command

.github/workflows/benchmarks-push-indexing.yml

@@ -16,11 +16,9 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/benchmarks-push-search-geo.yml

@@ -15,11 +15,9 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/benchmarks-push-search-songs.yml

@@ -15,11 +15,9 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/benchmarks-push-search-wiki.yml

@@ -15,11 +15,9 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/flaky-tests.yml

@@ -1,4 +1,6 @@
 name: Look for flaky tests
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 on:
   workflow_dispatch:
   schedule:
@@ -16,10 +18,7 @@ jobs:
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
+    - uses: helix-editor/rust-toolchain@v1
     - name: Install cargo-flaky
       run: cargo install cargo-flaky
     - name: Run cargo flaky in the dumps

.github/workflows/fuzzer-indexing.yml

@@ -1,5 +1,6 @@
 name: Run the indexing fuzzer
-
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 on:
   push:
     branches:
@@ -12,11 +13,9 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
 
       # Run benchmarks
       - name: Run the fuzzer

.github/workflows/publish-apt-brew-pkg.yml

@@ -15,6 +15,8 @@ jobs:
 
   debian:
     name: Publish debian packagge
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     needs: check-version
     container:
@@ -25,10 +27,7 @@ jobs:
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
+    - uses: helix-editor/rust-toolchain@v1
     - name: Install cargo-deb
       run: cargo install cargo-deb
     - uses: actions/checkout@v3

.github/workflows/publish-binaries.yml

@@ -35,6 +35,8 @@ jobs:
   publish-linux:
     name: Publish binary for Linux
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     needs: check-version
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27
@@ -45,10 +47,7 @@ jobs:
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
+    - uses: helix-editor/rust-toolchain@v1
     - name: Build
       run: cargo build --release --locked
     # No need to upload binaries for dry run (cron)
@@ -78,10 +77,7 @@ jobs:
             asset_name: meilisearch-windows-amd64.exe
     steps:
     - uses: actions/checkout@v3
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
+    - uses: helix-editor/rust-toolchain@v1
     - name: Build
       run: cargo build --release --locked
     # No need to upload binaries for dry run (cron)
@@ -107,12 +103,10 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
       - name: Installing Rust toolchain
-        uses: actions-rs/toolchain@v1
+        uses: helix-editor/rust-toolchain@v1
         with:
-          toolchain: stable
           profile: minimal
           target: ${{ matrix.target }}
-          override: true
       - name: Cargo build
         uses: actions-rs/cargo@v1
         with:
@@ -132,6 +126,8 @@ jobs:
     name: Publish binary for aarch64
     runs-on: ubuntu-latest
     needs: check-version
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27
       image: ubuntu:18.04
@@ -154,12 +150,10 @@ jobs:
           add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
           apt-get update -y && apt-get install -y docker-ce
       - name: Installing Rust toolchain
-        uses: actions-rs/toolchain@v1
+        uses: helix-editor/rust-toolchain@v1
         with:
-          toolchain: stable
           profile: minimal
           target: ${{ matrix.target }}
-          override: true
       - name: Configure target aarch64 GNU
         ## Environment variable is not passed using env:
         ## LD gold won't work with MUSL

.github/workflows/publish-docker-images.yml

@@ -80,10 +80,11 @@ jobs:
             type=ref,event=tag
             type=raw,value=nightly,enable=${{ github.event_name != 'push' }}
             type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
+            type=semver,pattern=v{{major}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
             type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v6
         with:
           push: true
           platforms: linux/amd64,linux/arm64

.github/workflows/test-suite.yml

@@ -21,6 +21,8 @@ jobs:
   test-linux:
     name: Tests on ubuntu-18.04
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
       image: ubuntu:18.04
@@ -31,10 +33,7 @@ jobs:
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
       - name: Setup test with Rust stable
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+        uses: helix-editor/rust-toolchain@v1
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
       - name: Run cargo check without any default features
@@ -59,10 +58,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: helix-editor/rust-toolchain@v1
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -77,6 +73,8 @@ jobs:
   test-all-features:
     name: Tests almost all features
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
       image: ubuntu:18.04
@@ -87,10 +85,7 @@ jobs:
         run: |
           apt-get update
           apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: helix-editor/rust-toolchain@v1
       - name: Run cargo build with almost all features
         run: |
           cargo build --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda)"
@@ -100,6 +95,8 @@ jobs:
 
   test-disabled-tokenization:
     name: Test disabled tokenization
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       image: ubuntu:18.04
@@ -110,13 +107,10 @@ jobs:
         run: |
           apt-get update
           apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: helix-editor/rust-toolchain@v1
       - name: Run cargo tree without default features and check lindera is not present
         run: |
-          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -vqz lindera; then
+          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -qz lindera; then
             echo "lindera has been found in the sources and it shouldn't"
             exit 1
           fi
@@ -127,6 +121,8 @@ jobs:
   # We run tests in debug also, to make sure that the debug_assertions are hit
   test-debug:
     name: Run tests in debug
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
@@ -137,10 +133,7 @@ jobs:
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: helix-editor/rust-toolchain@v1
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
       - name: Run tests in debug
@@ -154,11 +147,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.75.0
-          override: true
           components: clippy
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
@@ -173,10 +164,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
+          toolchain: nightly-2024-07-09
           override: true
           components: rustfmt
       - name: Cache dependencies

.github/workflows/update-cargo-toml-version.yml

@@ -18,11 +18,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
-          override: true
       - name: Install sd
         run: cargo install sd
       - name: Update Cargo.toml file

BENCHMARKS.md

@@ -109,6 +109,12 @@ They are JSON files with the following structure (comments are not actually supp
   "run_count": 3,
   // List of arguments to add to the Meilisearch command line.
   "extra_cli_args": ["--max-indexing-threads=1"],
+  // An expression that can be parsed as a comma-separated list of targets and levels
+  // as described in [tracing_subscriber's documentation](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/targets/struct.Targets.html#examples).
+  // The expression is used to filter the spans that are measured for profiling purposes.
+  // Optional, defaults to "indexing::=trace" (for indexing workloads), common other values is
+  // "search::=trace"
+  "target": "indexing::=trace",
   // List of named assets that can be used in the commands.
   "assets": {
     // name of the asset.
@@ -187,8 +193,8 @@ They are JSON files with the following structure (comments are not actually supp
   },
   // Core of the workload.
   // A list of commands to run sequentially.
-  // A command is a request to the Meilisearch instance that is executed while the profiling runs.
-  "commands": [
+  // Optional: A precommand is a request to the Meilisearch instance that is executed before the profiling runs.
+  "precommands": [
     {
       // Meilisearch route to call. `http://localhost:7700/` will be prepended.
       "route": "indexes/movies/settings",
@@ -224,8 +230,11 @@ They are JSON files with the following structure (comments are not actually supp
       // - DontWait: run the next command without waiting the response to this one.
       // - WaitForResponse: run the next command as soon as the response from the server is received.
       // - WaitForTask: run the next command once **all** the Meilisearch tasks created up to now have finished processing.
-      "synchronous": "DontWait"
-    },
+      "synchronous": "WaitForTask"
+    }
+  ],
+  // A command is a request to the Meilisearch instance that is executed while the profiling runs.
+  "commands": [
     {
       "route": "indexes/movies/documents",
       "method": "POST",

CONTRIBUTING.md

@@ -52,6 +52,16 @@ cargo test
 
 This command will be triggered to each PR as a requirement for merging it.
 
+#### Faster build
+
+You can set the `LINDERA_CACHE` environment variable to speed up your successive builds by up to 2 minutes.
+It'll store some built artifacts in the directory of your choice.
+
+We recommend using the standard `$HOME/.cache/lindera` directory:
+```sh
+export LINDERA_CACHE=$HOME/.cache/lindera
+```
+
 #### Snapshot-based tests
 
 We are using [insta](https://insta.rs) to perform snapshot-based testing.
@@ -63,7 +73,7 @@ Furthermore, we provide some macros on top of insta, notably a way to use snapsh
 
 To effectively debug snapshot-based hashes, we recommend you export the `MEILI_TEST_FULL_SNAPS` environment variable so that snapshot are fully created locally:
 
-```
+```sh
 export MEILI_TEST_FULL_SNAPS=true # add this to your .bashrc, .zshrc, ...
 ```
 

Cargo.toml

@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.8.0"
+version = "1.9.0"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",

Dockerfile

@@ -1,7 +1,7 @@
 # Compile
-FROM    rust:1.75.0-alpine3.18 AS compiler
+FROM    rust:1.79.0-alpine3.20 AS compiler
 
-RUN     apk add -q --update-cache --no-cache build-base openssl-dev
+RUN     apk add -q --no-cache build-base openssl-dev
 
 WORKDIR /
 
@@ -20,13 +20,12 @@ RUN     set -eux; \
         cargo build --release -p meilisearch -p meilitool
 
 # Run
-FROM    alpine:3.16
+FROM    alpine:3.20
 
 ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
 ENV     MEILI_SERVER_PROVIDER docker
 
-RUN     apk update --quiet \
-        && apk add -q --no-cache libgcc tini curl
+RUN     apk add -q --no-cache libgcc tini curl
 
 # add meilisearch and meilitool to the `/bin` so you can run it from anywhere
 # and it's easy to find.

README.md

@@ -1,9 +1,6 @@
 <p align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-light-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
-  </a>
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-dark-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
+  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo" target="_blank">
+    <img src="assets/meilisearch-logo-kawaii.png">
   </a>
 </p>
 
@@ -25,7 +22,7 @@
 
 <p align="center">⚡ A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow 🔍</p>
 
-Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow.
+[Meilisearch](https://www.meilisearch.com?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=intro) helps you shape a delightful search experience in a snap, offering features that work out of the box to speed up your workflow.
 
 <p align="center" name="demo">
   <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-light-mode-only" target="_blank">
@@ -36,11 +33,18 @@ Meilisearch helps you shape a delightful search experience in a snap, offering f
   </a>
 </p>
 
-🔥 [**Try it!**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-link) 🔥
+## 🖥 Examples
+
+- [**Movies**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — An application to help you find streaming platforms to watch movies using [hybrid search](https://www.meilisearch.com/solutions/hybrid-search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos).
+- [**Ecommerce**](https://ecommerce.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Ecommerce website using disjunctive [facets](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos), range and rating filtering, and pagination.
+- [**Songs**](https://music.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search through 47 million of songs.
+- [**SaaS**](https://saas.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search for contacts, deals, and companies in this [multi-tenant](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) CRM application.
+
+See the list of all our example apps in our [demos repository](https://github.com/meilisearch/demos).
 
 ## ✨ Features
-
-- **Search-as-you-type:** find search results in less than 50 milliseconds
+- **Hybrid search:** Combine the best of both [semantic](https://www.meilisearch.com/docs/learn/experimental/vector_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) & full-text search to get the most relevant results
+- **Search-as-you-type:** Find & display results in less than 50 milliseconds to provide an intuitive experience
 - **[Typo tolerance](https://www.meilisearch.com/docs/learn/configuration/typo_tolerance?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** get relevant matches even when queries contain typos and misspellings
 - **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your users' search experience with custom filters and build a faceted search interface in a few lines of code
 - **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** sort results based on price, date, or pretty much anything else your users need
@@ -55,15 +59,15 @@ Meilisearch helps you shape a delightful search experience in a snap, offering f
 
 ## 📖 Documentation
 
-You can consult Meilisearch's documentation at [https://www.meilisearch.com/docs](https://www.meilisearch.com/docs/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=docs).
+You can consult Meilisearch's documentation at [meilisearch.com/docs](https://www.meilisearch.com/docs/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=docs).
 
 ## 🚀 Getting started
 
-For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
+For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [documentation](https://www.meilisearch.com/docs?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
 
-## ⚡ Supercharge your Meilisearch experience
+## 🌍 Supercharge your Meilisearch experience
 
-Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). No credit card required.
+Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). Additional features include analytics & monitoring in many regions around the world. No credit card is required.
 
 ## 🧰 SDKs & integration tools
 
@@ -83,15 +87,15 @@ Finally, for more in-depth information, refer to our articles explaining fundame
 
 ## 📊 Telemetry
 
-Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
+Meilisearch collects **anonymized** user data to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
 
-To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
+To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Remember to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
 
 If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) of our documentation.
 
 ## 📫 Get in touch!
 
-Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
+Meilisearch is a search engine created by [Meili]([https://www.welcometothejungle.com/en/companies/meilisearch](https://www.meilisearch.com/careers)), a software development company headquartered in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
 
 🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
 
@@ -105,11 +109,11 @@ Thank you for your support!
 
 ## 👩‍💻 Contributing
 
-Meilisearch is, and will always be, open-source! If you want to contribute to the project, please take a look at [our contribution guidelines](CONTRIBUTING.md).
+Meilisearch is, and will always be, open-source! If you want to contribute to the project, please look at [our contribution guidelines](CONTRIBUTING.md).
 
 ## 📦 Versioning
 
-Meilisearch releases and their associated binaries are available [in this GitHub page](https://github.com/meilisearch/meilisearch/releases).
+Meilisearch releases and their associated binaries are available on the project's [releases page](https://github.com/meilisearch/meilisearch/releases).
 
 The binaries are versioned following [SemVer conventions](https://semver.org/). To know more, read our [versioning policy](https://github.com/meilisearch/engine-team/blob/main/resources/versioning-policy.md).
 

benchmarks/Cargo.toml

@@ -11,24 +11,24 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.79"
+anyhow = "1.0.86"
 csv = "1.3.0"
 milli = { path = "../milli" }
-mimalloc = { version = "0.1.39", default-features = false }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
+mimalloc = { version = "0.1.43", default-features = false }
+serde_json = { version = "1.0.120", features = ["preserve_order"] }
 
 [dev-dependencies]
 criterion = { version = "0.5.1", features = ["html_reports"] }
 rand = "0.8.5"
 rand_chacha = "0.3.1"
-roaring = "0.10.2"
+roaring = "0.10.6"
 
 [build-dependencies]
-anyhow = "1.0.79"
-bytes = "1.5.0"
+anyhow = "1.0.86"
+bytes = "1.6.0"
 convert_case = "0.6.0"
-flate2 = "1.0.28"
-reqwest = { version = "0.11.23", features = ["blocking", "rustls-tls"], default-features = false }
+flate2 = "1.0.30"
+reqwest = { version = "0.12.5", features = ["blocking", "rustls-tls"], default-features = false }
 
 [features]
 default = ["milli/all-tokenizations"]

build-info/Cargo.toml

@@ -11,8 +11,8 @@ license.workspace = true
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-time = { version = "0.3.34", features = ["parsing"] }
+time = { version = "0.3.36", features = ["parsing"] }
 
 [build-dependencies]
-anyhow = "1.0.80"
-vergen-git2 = "1.0.0-beta.2"
+anyhow = "1.0.86"
+vergen-git2 = "1.0.0"

dump/Cargo.toml

@@ -11,22 +11,21 @@ readme.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.79"
-flate2 = "1.0.28"
-http = "0.2.11"
-meilisearch-auth = { path = "../meilisearch-auth" }
+anyhow = "1.0.86"
+flate2 = "1.0.30"
+http = "1.1.0"
 meilisearch-types = { path = "../meilisearch-types" }
 once_cell = "1.19.0"
-regex = "1.10.2"
-roaring = { version = "0.10.2", features = ["serde"] }
-serde = { version = "1.0.195", features = ["derive"] }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
-tar = "0.4.40"
-tempfile = "3.9.0"
-thiserror = "1.0.56"
-time = { version = "0.3.31", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+regex = "1.10.5"
+roaring = { version = "0.10.6", features = ["serde"] }
+serde = { version = "1.0.204", features = ["derive"] }
+serde_json = { version = "1.0.120", features = ["preserve_order"] }
+tar = "0.4.41"
+tempfile = "3.10.1"
+thiserror = "1.0.61"
+time = { version = "0.3.36", features = ["serde-well-known", "formatting", "parsing", "macros"] }
 tracing = "0.1.40"
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
+uuid = { version = "1.10.0", features = ["serde", "v4"] }
 
 [dev-dependencies]
 big_s = "1.0.2"

dump/src/reader/compat/v2_to_v3.rs

@@ -425,7 +425,7 @@ pub(crate) mod test {
         let mut dump = v2::V2Reader::open(dir).unwrap().to_v3();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/compat/v3_to_v4.rs

@@ -358,7 +358,7 @@ pub(crate) mod test {
         let mut dump = v3::V3Reader::open(dir).unwrap().to_v4();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/compat/v4_to_v5.rs

@@ -394,8 +394,8 @@ pub(crate) mod test {
         let mut dump = v4::V4Reader::open(dir).unwrap().to_v5();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/compat/v5_to_v6.rs

@@ -442,8 +442,8 @@ pub(crate) mod test {
         let mut dump = v5::V5Reader::open(dir).unwrap().to_v6();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/mod.rs

@@ -197,13 +197,147 @@ pub(crate) mod test {
     use super::*;
     use crate::reader::v6::RuntimeTogglableFeatures;
 
+    #[test]
+    fn import_dump_v6_with_vectors() {
+        // dump containing two indexes
+        //
+        // "vector", configured with an embedder
+        // contains:
+        // - one document with an overriden vector,
+        // - one document with a natural vector
+        // - one document with a _vectors map containing one additional embedder name and a natural vector
+        // - one document with a _vectors map containing one additional embedder name and an overriden vector
+        //
+        // "novector", no embedder
+        // contains:
+        // - a document without vector
+        // - a document with a random _vectors field
+        let dump = File::open("tests/assets/v6-with-vectors.dump").unwrap();
+        let mut dump = DumpReader::open(dump).unwrap();
+
+        // top level infos
+        insta::assert_snapshot!(dump.date().unwrap(), @"2024-05-16 15:51:34.151044 +00:00:00");
+        insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
+
+        // tasks
+        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"278f63325ef06ca04d01df98d8207b94");
+        assert_eq!(update_files.len(), 10);
+        assert!(update_files[0].is_none()); // the dump creation
+        assert!(update_files[1].is_none());
+        assert!(update_files[2].is_none());
+        assert!(update_files[3].is_none());
+        assert!(update_files[4].is_none());
+        assert!(update_files[5].is_none());
+        assert!(update_files[6].is_none());
+        assert!(update_files[7].is_none());
+        assert!(update_files[8].is_none());
+        assert!(update_files[9].is_none());
+
+        // indexes
+        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        // the index are not ordered in any way by default
+        indexes.sort_by_key(|index| index.metadata().uid.to_string());
+
+        let mut vector_index = indexes.pop().unwrap();
+        let mut novector_index = indexes.pop().unwrap();
+        assert!(indexes.is_empty());
+
+        // vector
+
+        insta::assert_json_snapshot!(vector_index.metadata(), @r###"
+        {
+          "uid": "vector",
+          "primaryKey": "id",
+          "createdAt": "2024-05-16T15:33:17.240962Z",
+          "updatedAt": "2024-05-16T15:40:55.723052Z"
+        }
+        "###);
+
+        {
+            let documents: Result<Vec<_>> = vector_index.documents().unwrap().collect();
+            let mut documents = documents.unwrap();
+            assert_eq!(documents.len(), 4);
+
+            documents.sort_by_key(|doc| doc.get("id").unwrap().to_string());
+
+            {
+                let document = documents.pop().unwrap();
+                insta::assert_json_snapshot!(document);
+            }
+
+            {
+                let document = documents.pop().unwrap();
+                insta::assert_json_snapshot!(document);
+            }
+
+            {
+                let document = documents.pop().unwrap();
+                insta::assert_json_snapshot!(document);
+            }
+
+            {
+                let document = documents.pop().unwrap();
+                insta::assert_json_snapshot!(document);
+            }
+        }
+
+        // novector
+
+        insta::assert_json_snapshot!(novector_index.metadata(), @r###"
+        {
+          "uid": "novector",
+          "primaryKey": "id",
+          "createdAt": "2024-05-16T15:33:03.568055Z",
+          "updatedAt": "2024-05-16T15:33:07.530217Z"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(novector_index.settings().unwrap().embedders, @"null");
+
+        {
+            let documents: Result<Vec<_>> = novector_index.documents().unwrap().collect();
+            let mut documents = documents.unwrap();
+            assert_eq!(documents.len(), 2);
+
+            documents.sort_by_key(|doc| doc.get("id").unwrap().to_string());
+
+            {
+                let document = documents.pop().unwrap();
+                insta::assert_json_snapshot!(document, @r###"
+                {
+                  "id": "e1",
+                  "other": "random1",
+                  "_vectors": "toto"
+                }
+                "###);
+            }
+
+            {
+                let document = documents.pop().unwrap();
+                insta::assert_json_snapshot!(document, @r###"
+                {
+                  "id": "e0",
+                  "other": "random0"
+                }
+                "###);
+            }
+        }
+
+        assert_eq!(
+            dump.features().unwrap().unwrap(),
+            RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
+        );
+    }
+
     #[test]
     fn import_dump_v6_experimental() {
         let dump = File::open("tests/assets/v6-with-experimental.dump").unwrap();
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-07-06 7:10:27.21958 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2023-07-06 7:10:27.21958 +00:00:00");
         insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
 
         // tasks
@@ -249,8 +383,8 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
@@ -329,8 +463,8 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
@@ -406,7 +540,7 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
         assert_eq!(dump.instance_uid().unwrap(), None);
 
         // tasks
@@ -499,7 +633,7 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
         assert_eq!(dump.instance_uid().unwrap(), None);
 
         // tasks
@@ -592,7 +726,7 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
         assert_eq!(dump.instance_uid().unwrap(), None);
 
         // tasks

dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-5.snap

@@ -0,0 +1,783 @@
+---
+source: dump/src/reader/mod.rs
+expression: document
+---
+{
+  "id": "e3",
+  "desc": "overriden vector + map",
+  "_vectors": {
+    "default": [
+      0.2,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1
+    ],
+    "toto": [
+      0.1
+    ]
+  }
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-6.snap

@@ -0,0 +1,786 @@
+---
+source: dump/src/reader/mod.rs
+expression: document
+---
+{
+  "id": "e2",
+  "desc": "natural vector + map",
+  "_vectors": {
+    "toto": [],
+    "default": {
+      "embeddings": [
+        [
+          -0.05189208313822746,
+          -0.9273212552070618,
+          0.1443813145160675,
+          0.0932632014155388,
+          0.2665371894836426,
+          0.36266782879829407,
+          0.6402910947799683,
+          0.32014018297195435,
+          0.030915971845388412,
+          -0.9312191605567932,
+          -0.3718109726905823,
+          -0.2700554132461548,
+          -1.1014580726623535,
+          0.9154956936836244,
+          -0.3406888246536255,
+          1.0077725648880005,
+          0.6577560901641846,
+          -0.3955195546150207,
+          -0.4148270785808563,
+          0.1855088472366333,
+          0.5062315464019775,
+          -0.3632686734199524,
+          -0.2277890294790268,
+          0.2560805082321167,
+          -0.3853609561920166,
+          -0.1604762226343155,
+          -0.13947471976280212,
+          -0.20147813856601715,
+          -0.4466346800327301,
+          -0.3761846721172333,
+          0.1443382054567337,
+          0.18205296993255615,
+          0.49359792470932007,
+          -0.22538000345230105,
+          -0.4996317625045776,
+          -0.22734887897968292,
+          -0.6034309267997742,
+          -0.7857939600944519,
+          -0.34923747181892395,
+          -0.3466345965862274,
+          0.21176661550998688,
+          -0.5101462006568909,
+          -0.3403083384037018,
+          0.000315118464641273,
+          0.236465722322464,
+          -0.10246097296476364,
+          -1.3013339042663574,
+          0.3419138789176941,
+          -0.32963496446609497,
+          -0.0901619717478752,
+          -0.5426247119903564,
+          0.22656650841236117,
+          -0.44758284091949463,
+          0.14151698350906372,
+          -0.1089438870549202,
+          0.5500766634941101,
+          -0.670711100101471,
+          -0.6227269768714905,
+          0.3894464075565338,
+          -0.27609574794769287,
+          0.7028202414512634,
+          -0.19697771966457367,
+          0.328511506319046,
+          0.5063360929489136,
+          0.4065195322036743,
+          0.2614171802997589,
+          -0.30274391174316406,
+          1.0393824577331543,
+          -0.7742937207221985,
+          -0.7874112129211426,
+          -0.6749666929244995,
+          0.5190866589546204,
+          0.004123548045754433,
+          -0.28312963247299194,
+          -0.038731709122657776,
+          -1.0142987966537476,
+          -0.09519586712121964,
+          0.8755272626876831,
+          0.4876938760280609,
+          0.7811151742935181,
+          0.85174959897995,
+          0.11826585978269576,
+          0.5373436808586121,
+          0.3649002015590668,
+          0.19064077734947205,
+          -0.00287026260048151,
+          -0.7305403351783752,
+          -0.015206154435873032,
+          -0.7899249196052551,
+          0.19407285749912265,
+          0.08596625179052353,
+          -0.28976231813430786,
+          -0.1525907665491104,
+          0.3798313438892365,
+          0.050306469202041626,
+          -0.5697937607765198,
+          0.4219021201133728,
+          0.276252806186676,
+          0.1559903472661972,
+          0.10030482709407806,
+          -0.4043720066547394,
+          -0.1969818025827408,
+          0.5739826560020447,
+          0.2116064727306366,
+          -1.4620544910430908,
+          -0.7802462577819824,
+          -0.24739810824394223,
+          -0.09791352599859238,
+          -0.4413802027702331,
+          0.21549351513385773,
+          -0.9520436525344848,
+          -0.08762510865926743,
+          0.08154498040676117,
+          -0.6154940724372864,
+          -1.01079523563385,
+          0.885427713394165,
+          0.6967288851737976,
+          0.27186504006385803,
+          -0.43194177746772766,
+          -0.11248451471328735,
+          0.7576630711555481,
+          0.4998855590820313,
+          0.0264343973249197,
+          0.9872855544090272,
+          0.5634694695472717,
+          0.053698331117630005,
+          0.19410227239131927,
+          0.3570743501186371,
+          -0.23670297861099243,
+          -0.9114483594894408,
+          0.07884842902421951,
+          0.7318344116210938,
+          0.44630110263824463,
+          0.08745364099740982,
+          -0.347101628780365,
+          -0.4314247667789459,
+          -0.5060274004936218,
+          0.003706763498485088,
+          0.44320008158683777,
+          -0.00788921769708395,
+          -0.1368623524904251,
+          -0.17391923069953918,
+          0.14473655819892883,
+          0.10927865654230118,
+          0.6974599361419678,
+          0.005052129738032818,
+          -0.016953065991401672,
+          -0.1256176233291626,
+          -0.036742497235536575,
+          0.5591985583305359,
+          -0.37619709968566895,
+          0.22429119050502777,
+          0.5403043031692505,
+          -0.8603790998458862,
+          -0.3456307053565979,
+          0.9292937517166138,
+          0.5074859261512756,
+          0.6310645937919617,
+          -0.3091641068458557,
+          0.46902573108673096,
+          0.7891915440559387,
+          0.4499550759792328,
+          0.2744995653629303,
+          0.2712305784225464,
+          -0.04349074140191078,
+          -0.3638863265514374,
+          0.7839881777763367,
+          0.7352104783058167,
+          -0.19457511603832245,
+          -0.5957832932472229,
+          -0.43704694509506226,
+          -1.084769368171692,
+          0.4904985725879669,
+          0.5385226011276245,
+          0.1891629993915558,
+          0.12338479608297348,
+          0.8315675258636475,
+          -0.07830192148685455,
+          1.0916285514831543,
+          -0.28066861629486084,
+          -1.3585069179534912,
+          0.5203898549079895,
+          0.08678033947944641,
+          -0.2566044330596924,
+          0.09484415501356123,
+          -0.0180208683013916,
+          1.0264745950698853,
+          -0.023572135716676712,
+          0.5864979028701782,
+          0.7625196576118469,
+          -0.2543414533138275,
+          -0.8877770900726318,
+          0.7611982822418213,
+          -0.06220436468720436,
+          0.937336564064026,
+          0.2704363465309143,
+          -0.37733694911003113,
+          0.5076137781143188,
+          -0.30641937255859375,
+          0.6252772808074951,
+          -0.0823579877614975,
+          -0.03736555948853493,
+          0.4131673276424408,
+          -0.6514252424240112,
+          0.12918265163898468,
+          -0.4483584463596344,
+          0.6750786304473877,
+          -0.37008383870124817,
+          -0.02324833907186985,
+          0.38027650117874146,
+          -0.26374951004981995,
+          0.4346931278705597,
+          0.42882832884788513,
+          -0.48798441886901855,
+          1.1882442235946655,
+          0.5132288336753845,
+          0.5284568667411804,
+          -0.03538886830210686,
+          0.29620853066444397,
+          -1.0683696269989014,
+          0.25936177372932434,
+          0.10404160618782043,
+          -0.25796034932136536,
+          0.027896970510482788,
+          -0.09225251525640488,
+          1.4811025857925415,
+          0.641173779964447,
+          -0.13838383555412292,
+          -0.3437179923057556,
+          0.5667019486427307,
+          -0.5400741696357727,
+          0.31090837717056274,
+          0.6470608115196228,
+          -0.3747067153453827,
+          -0.7364534735679626,
+          -0.07431528717279434,
+          0.5173454880714417,
+          -0.6578747034072876,
+          0.7107478976249695,
+          -0.7918999791145325,
+          -0.0648345872759819,
+          0.609937846660614,
+          -0.7329513430595398,
+          0.9741371870040894,
+          0.17912346124649048,
+          -0.02658769302070141,
+          0.5162150859832764,
+          -0.3978803157806397,
+          -0.7833885550498962,
+          -0.6497276425361633,
+          -0.3898126780986786,
+          -0.0952848568558693,
+          0.2663288116455078,
+          -0.1604052186012268,
+          0.373076468706131,
+          -0.8357769250869751,
+          -0.05217683315277099,
+          -0.2680160701274872,
+          0.8389158248901367,
+          0.6833611130714417,
+          -0.6712407469749451,
+          0.7406917214393616,
+          -0.44522786140441895,
+          -0.34645363688468933,
+          -0.27384576201438904,
+          -0.9878405928611756,
+          -0.8166060447692871,
+          0.06268279999494553,
+          0.38567957282066345,
+          -0.3274703919887543,
+          0.5296315550804138,
+          -0.11810623109340668,
+          0.23029841482639313,
+          0.08616159111261368,
+          -0.2195747196674347,
+          0.09430307894945145,
+          0.4057176411151886,
+          0.4892159104347229,
+          -0.1636916548013687,
+          -0.6071445345878601,
+          0.41256585717201233,
+          0.622254490852356,
+          -0.41223976016044617,
+          -0.6686707139015198,
+          -0.7474371790885925,
+          -0.8509522080421448,
+          -0.16754287481307983,
+          -0.9078601002693176,
+          -0.29653599858283997,
+          -0.5020652413368225,
+          0.4692700505256653,
+          0.01281109917908907,
+          -0.16071580350399017,
+          0.03388889133930206,
+          -0.020511148497462273,
+          0.5027827024459839,
+          -0.20729811489582065,
+          0.48107290267944336,
+          0.33669769763946533,
+          -0.5275911688804626,
+          0.48271527886390686,
+          0.2738940715789795,
+          -0.033152539283037186,
+          -0.13629786670207977,
+          -0.05965912342071533,
+          -0.26200807094573975,
+          0.04002794995903969,
+          -0.34095603227615356,
+          -3.986898899078369,
+          -0.46819332242012024,
+          -0.422744482755661,
+          -0.169097900390625,
+          0.6008929014205933,
+          0.058016058057546616,
+          -0.11401277780532836,
+          -0.3077819049358368,
+          -0.09595538675785063,
+          0.6723822355270386,
+          0.19367831945419312,
+          0.28304359316825867,
+          0.1609862744808197,
+          0.7567598819732666,
+          0.6889985799789429,
+          0.06907720118761063,
+          -0.04188092052936554,
+          -0.7434936165809631,
+          0.13321782648563385,
+          0.8456063270568848,
+          -0.10364038497209548,
+          -0.45084846019744873,
+          -0.4758241474628449,
+          0.43882066011428833,
+          -0.6432598829269409,
+          0.7217311859130859,
+          -0.24189773201942444,
+          0.12737572193145752,
+          -1.1008601188659668,
+          -0.3305315673351288,
+          0.14614742994308472,
+          -0.7819333076477051,
+          0.5287120342254639,
+          -0.055538054555654526,
+          0.1877404749393463,
+          -0.6907662153244019,
+          0.5616975426673889,
+          -0.4611121714115143,
+          -0.26109233498573303,
+          -0.12898315489292145,
+          -0.3724522292613983,
+          -0.7191406488418579,
+          -0.4425233602523804,
+          -0.644108235836029,
+          0.8424481153488159,
+          0.17532426118850708,
+          -0.5121750235557556,
+          -0.6467239260673523,
+          -0.0008507720194756985,
+          0.7866212129592896,
+          -0.02644744887948036,
+          -0.005045140627771616,
+          0.015782782807946205,
+          0.16334445774555206,
+          -0.1913367658853531,
+          -0.13697923719882965,
+          -0.6684983372688293,
+          0.18346354365348816,
+          -0.341105580329895,
+          0.5427411198616028,
+          0.3779832422733307,
+          -0.6778115034103394,
+          -0.2931850254535675,
+          -0.8805161714553833,
+          -0.4212774932384491,
+          -0.5368952751159668,
+          -1.3937891721725464,
+          -1.225494146347046,
+          0.4276703894138336,
+          1.1205668449401855,
+          -0.6005299687385559,
+          0.15732505917549133,
+          -0.3914784789085388,
+          -1.357046604156494,
+          -0.4707142114639282,
+          -0.1497287154197693,
+          -0.25035548210144043,
+          -0.34328439831733704,
+          0.39083412289619446,
+          0.1623048633337021,
+          -0.9275814294815063,
+          -0.6430015563964844,
+          0.2973862886428833,
+          0.5580436587333679,
+          -0.6232585310935974,
+          -0.6611042022705078,
+          0.4015969038009643,
+          -1.0232892036437988,
+          -0.2585645020008087,
+          -0.5431421399116516,
+          0.5021264553070068,
+          -0.48601630330085754,
+          -0.010242084041237833,
+          0.5862035155296326,
+          0.7316920161247253,
+          0.4036808013916016,
+          0.4269520044326782,
+          -0.705938458442688,
+          0.7747307419776917,
+          0.10164368897676468,
+          0.7887958884239197,
+          -0.9612497091293336,
+          0.12755516171455383,
+          0.06812842190265656,
+          -0.022603651508688927,
+          0.14722754061222076,
+          -0.5588505268096924,
+          -0.20689940452575684,
+          0.3557641804218292,
+          -0.6812759637832642,
+          0.2860803008079529,
+          -0.38954633474349976,
+          0.1759403496980667,
+          -0.5678874850273132,
+          -0.1692986786365509,
+          -0.14578519761562347,
+          0.5711379051208496,
+          1.0208125114440918,
+          0.7759483456611633,
+          -0.372348427772522,
+          -0.5460885763168335,
+          0.7190321683883667,
+          -0.6914990544319153,
+          0.13365162909030914,
+          -0.4854792356491089,
+          0.4054908752441406,
+          0.4502798914909363,
+          -0.3041122555732727,
+          -0.06726965308189392,
+          -0.05570871382951737,
+          -0.0455719493329525,
+          0.4785125255584717,
+          0.8867972493171692,
+          0.4107886850833893,
+          0.6121342182159424,
+          -0.20477132499217987,
+          -0.5598517656326294,
+          -0.6443566679954529,
+          -0.5905212759971619,
+          -0.5571200251579285,
+          0.17573799192905426,
+          -0.28621870279312134,
+          0.1685224026441574,
+          0.09719007462263109,
+          -0.04223639518022537,
+          -0.28623101115226746,
+          -0.1449810117483139,
+          -0.3789580464363098,
+          -0.5227636098861694,
+          -0.049728814512491226,
+          0.7849089503288269,
+          0.16792525351047516,
+          0.9849340915679932,
+          -0.6559549570083618,
+          0.35723909735679626,
+          -0.6822739243507385,
+          1.2873116731643677,
+          0.19993330538272855,
+          0.03512010723352432,
+          -0.6972134113311768,
+          0.18453484773635864,
+          -0.2437680810689926,
+          0.2156416028738022,
+          0.5230382680892944,
+          0.22020135819911957,
+          0.8314080238342285,
+          0.15627102553844452,
+          -0.7330264449119568,
+          0.3888184726238251,
+          -0.22034703195095065,
+          0.5457669496536255,
+          -0.48084837198257446,
+          -0.45576658844947815,
+          -0.09287727624177931,
+          -0.06968110054731369,
+          0.35125672817230225,
+          -0.4278119504451752,
+          0.2038476765155792,
+          0.11392722278833388,
+          0.9433983564376832,
+          -0.4097744226455689,
+          0.035297419875860214,
+          -0.4274404048919678,
+          -0.25100165605545044,
+          1.0943366289138794,
+          -0.07634022831916809,
+          -0.2925529479980469,
+          -0.7512530088424683,
+          0.2649727463722229,
+          -0.4078235328197479,
+          -0.3372223973274231,
+          0.05190162733197212,
+          0.005654910113662481,
+          -0.0001571219472680241,
+          -0.35445958375930786,
+          -0.7837416529655457,
+          0.1500556766986847,
+          0.4383024573326111,
+          0.6099548935890198,
+          0.05951934307813645,
+          -0.21325334906578064,
+          0.0199207104742527,
+          -0.22704418003559113,
+          -0.6481077671051025,
+          0.37442275881767273,
+          -1.015955924987793,
+          0.38637226819992065,
+          -0.06489371508359909,
+          -0.494120329618454,
+          0.3469836115837097,
+          0.15402406454086304,
+          -0.7660972476005554,
+          -0.7053225040435791,
+          -0.25964751839637756,
+          0.014004424214363098,
+          -0.2860170006752014,
+          -0.17565494775772095,
+          -0.45117494463920593,
+          -0.0031954257283359766,
+          0.09676837921142578,
+          -0.514464259147644,
+          0.41698193550109863,
+          -0.21642713248729703,
+          -0.5398141145706177,
+          -0.3647628426551819,
+          0.37005379796028137,
+          0.239425927400589,
+          -0.08833975344896317,
+          0.934946596622467,
+          -0.48340797424316406,
+          0.6241437792778015,
+          -0.7253676652908325,
+          -0.04303571209311485,
+          1.1125205755233765,
+          -0.15692919492721558,
+          -0.2914651036262512,
+          -0.5117168426513672,
+          0.21365483105182648,
+          0.4924402534961701,
+          0.5269662141799927,
+          0.0352792888879776,
+          -0.149167999625206,
+          -0.6019760370254517,
+          0.08245442807674408,
+          0.4900692105293274,
+          0.518824577331543,
+          -0.00005570516441366635,
+          -0.553304135799408,
+          0.22217543423175812,
+          0.5047767758369446,
+          0.135724738240242,
+          1.1511540412902832,
+          -0.3541218340396881,
+          -0.9712511897087096,
+          0.8353699445724487,
+          -0.39227569103240967,
+          -0.9117669463157654,
+          -0.26349931955337524,
+          0.05597023293375969,
+          0.20695461332798004,
+          0.3178807199001312,
+          1.0663238763809204,
+          0.5062212347984314,
+          0.7288597822189331,
+          0.09899299591779707,
+          0.553720235824585,
+          0.675009548664093,
+          -0.20067055523395536,
+          0.3138423264026642,
+          -0.6886593103408813,
+          -0.2910398542881012,
+          -1.3186300992965698,
+          -0.4684459865093231,
+          -0.095743365585804,
+          -0.1257995069026947,
+          -0.4858281314373016,
+          -0.4935407340526581,
+          -0.3266896903514862,
+          -0.3928797245025635,
+          -0.40803104639053345,
+          -0.9975396394729614,
+          0.4229583740234375,
+          0.37309643626213074,
+          0.4431034922599793,
+          0.30364808440208435,
+          -0.3765178918838501,
+          0.5616499185562134,
+          0.16904796659946442,
+          -0.7343707084655762,
+          0.2560209631919861,
+          0.6166825294494629,
+          0.3200829327106476,
+          -0.4483652710914612,
+          0.16224201023578644,
+          -0.31495288014411926,
+          -0.42713335156440735,
+          0.7270734906196594,
+          0.7049484848976135,
+          -0.0571461021900177,
+          0.04477125033736229,
+          -0.6647796034812927,
+          1.183672308921814,
+          0.36199676990509033,
+          0.046881116926670074,
+          0.4515796303749085,
+          0.9278061985969543,
+          0.31471705436706543,
+          -0.7073333859443665,
+          -0.3443860113620758,
+          0.5440067052841187,
+          -0.15020819008350372,
+          -0.541202962398529,
+          0.5203295946121216,
+          1.2192286252975464,
+          -0.9983593225479126,
+          -0.18758884072303772,
+          0.2758221924304962,
+          -0.6511523723602295,
+          -0.1584404855966568,
+          -0.236241415143013,
+          0.2692437767982483,
+          -0.4941152036190033,
+          0.4987454116344452,
+          -0.3331359028816223,
+          0.3163745701313019,
+          0.745529294013977,
+          -0.2905873656272888,
+          0.13602906465530396,
+          0.4679684340953827,
+          1.0555986166000366,
+          1.075700044631958,
+          0.5368486046791077,
+          -0.5118206739425659,
+          0.8668332099914551,
+          -0.5726966857910156,
+          -0.7811751961708069,
+          0.1938626915216446,
+          -0.1929349899291992,
+          0.1757766306400299,
+          0.6384295225143433,
+          0.26462844014167786,
+          0.9542630314826964,
+          0.19313029944896695,
+          1.264248013496399,
+          -0.6304428577423096,
+          0.0487106591463089,
+          -0.16211535036563873,
+          -0.7894763350486755,
+          0.3582514822483063,
+          -0.04153040423989296,
+          0.635784387588501,
+          0.6554391980171204,
+          -0.47010496258735657,
+          -0.8302040696144104,
+          -0.1350124627351761,
+          0.2568812072277069,
+          0.13614831864833832,
+          -0.2563649117946625,
+          -1.0434694290161133,
+          0.3232482671737671,
+          0.47882452607154846,
+          0.4298652410507202,
+          1.0563770532608032,
+          -0.28917592763900757,
+          -0.8533256649971008,
+          0.10648339986801147,
+          0.6376127004623413,
+          -0.20832888782024384,
+          0.2370245456695557,
+          0.0018312990432605147,
+          -0.2034837007522583,
+          0.01051164511591196,
+          -1.105310082435608,
+          0.29724350571632385,
+          0.15604574978351593,
+          0.1973688006401062,
+          0.44394731521606445,
+          0.3974513411521912,
+          -0.13625948131084442,
+          0.9571986198425292,
+          0.2257384955883026,
+          0.2323588728904724,
+          -0.5583669543266296,
+          -0.7854922413825989,
+          0.1647188365459442,
+          -1.6098142862319946,
+          0.318587988615036,
+          -0.13399995863437653,
+          -0.2172701060771942,
+          -0.767514705657959,
+          -0.5813586711883545,
+          -0.3195130527019501,
+          -0.04894036799669266,
+          0.2929930090904236,
+          -0.8213384747505188,
+          0.07181350141763687,
+          0.7469993829727173,
+          0.6407455801963806,
+          0.16365697979927063,
+          0.7870153188705444,
+          0.6524736881256104,
+          0.6399973630905151,
+          -0.04992736503481865,
+          -0.03959266096353531,
+          -0.2512352466583252,
+          0.8448855876922607,
+          -0.1422702670097351,
+          0.1216789186000824,
+          -1.2647287845611572,
+          0.5931149125099182,
+          0.7186052203178406,
+          -0.06118432432413101,
+          -1.1942816972732544,
+          -0.17677085101604462,
+          0.31543800234794617,
+          -0.32252824306488037,
+          0.8255583047866821,
+          -0.14529970288276672,
+          -0.2695446312427521,
+          -0.33378756046295166,
+          -0.1653425395488739,
+          0.1454019844532013,
+          -0.3920115828514099,
+          0.912214994430542,
+          -0.7279734015464783,
+          0.7374742031097412,
+          0.933980405330658,
+          0.13429680466651917,
+          -0.514870285987854,
+          0.3989711999893189,
+          -0.11613689363002776,
+          0.4022413492202759,
+          -0.9990655779838562,
+          -0.33749932050704956,
+          -0.4334589838981629,
+          -1.376373291015625,
+          -0.2993924915790558,
+          -0.09454808384180068,
+          -0.01314175222069025,
+          -0.001090060803107917,
+          0.2137461006641388,
+          0.2938512861728668,
+          0.17508235573768616,
+          0.8260607123374939,
+          -0.7218498587608337,
+          0.2414487451314926,
+          -0.47296759486198425,
+          -0.3002610504627228,
+          -1.238540768623352,
+          0.08663805574178696,
+          0.6805586218833923,
+          0.5909030437469482,
+          -0.42807504534721375,
+          -0.22887496650218964,
+          0.47537800669670105,
+          -1.0474627017974854,
+          0.6338009238243103,
+          0.06548397243022919,
+          0.4971011281013489,
+          1.3484878540039063
+        ]
+      ],
+      "regenerate": true
+    }
+  }
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-7.snap

@@ -0,0 +1,785 @@
+---
+source: dump/src/reader/mod.rs
+expression: document
+---
+{
+  "id": "e1",
+  "desc": "natural vector",
+  "_vectors": {
+    "default": {
+      "embeddings": [
+        [
+          -0.2979458272457123,
+          -0.5288640856742859,
+          -0.019957859069108963,
+          -0.18495318293571472,
+          0.7429973483085632,
+          0.5238497257232666,
+          0.432366281747818,
+          0.32744166254997253,
+          0.0020762972999364138,
+          -0.9507834911346436,
+          -0.35097137093544006,
+          0.08469701558351517,
+          -1.4176613092422483,
+          0.4647577106952667,
+          -0.69340580701828,
+          1.0372896194458008,
+          0.3716741800308227,
+          0.06031008064746857,
+          -0.6152024269104004,
+          0.007914665155112743,
+          0.7954924702644348,
+          -0.20773003995418549,
+          0.09376765787601472,
+          0.04508133605122566,
+          -0.2084471583366394,
+          -0.1518009901046753,
+          0.018195509910583496,
+          -0.07044368237257004,
+          -0.18119366466999057,
+          -0.4480230510234833,
+          0.3822529911994934,
+          0.1911812424659729,
+          0.4674372375011444,
+          0.06963984668254852,
+          -0.09341949224472046,
+          0.005675444379448891,
+          -0.6774799227714539,
+          -0.7066726684570313,
+          -0.39256376028060913,
+          0.04005039855837822,
+          0.2084812968969345,
+          -0.7872875928878784,
+          -0.8205880522727966,
+          0.2919981777667999,
+          -0.06004738807678223,
+          -0.4907574355602264,
+          -1.5937862396240234,
+          0.24249385297298431,
+          -0.14709846675395966,
+          -0.11860740929841997,
+          -0.8299489617347717,
+          0.472964346408844,
+          -0.497518390417099,
+          -0.22205302119255063,
+          -0.4196169078350067,
+          0.32697558403015137,
+          -0.360930860042572,
+          -0.9789686799049376,
+          0.1887447088956833,
+          -0.403737336397171,
+          0.18524253368377688,
+          0.3768732249736786,
+          0.3666233420372009,
+          0.3511938452720642,
+          0.6985810995101929,
+          0.41721710562705994,
+          0.09754953533411026,
+          0.6204307079315186,
+          -1.0762996673583984,
+          -0.06263761967420578,
+          -0.7376511693000793,
+          0.6849768161773682,
+          -0.1745152473449707,
+          -0.40449759364128113,
+          0.20757411420345304,
+          -0.8424443006515503,
+          0.330015629529953,
+          0.3489064872264862,
+          1.0954371690750122,
+          0.8487558960914612,
+          1.1076823472976685,
+          0.61430823802948,
+          0.4155903458595276,
+          0.4111340939998626,
+          0.05753209814429283,
+          -0.06429877132177353,
+          -0.765606164932251,
+          -0.41703930497169495,
+          -0.508820652961731,
+          0.19859947264194489,
+          -0.16607828438282013,
+          -0.28112146258354187,
+          0.11032675206661224,
+          0.38809511065483093,
+          -0.36498191952705383,
+          -0.48671194911003113,
+          0.6755134463310242,
+          0.03958442434668541,
+          0.4478721618652344,
+          -0.10335399955511092,
+          -0.9546685814857484,
+          -0.6087718605995178,
+          0.17498846352100372,
+          0.08320838958024979,
+          -1.4478336572647097,
+          -0.605027437210083,
+          -0.5867993235588074,
+          -0.14711688458919525,
+          -0.5447602272033691,
+          -0.026259321719408035,
+          -0.6997418403625488,
+          -0.07349082082509995,
+          0.10638900846242905,
+          -0.7133527398109436,
+          -0.9396815299987792,
+          1.087092399597168,
+          1.1885089874267578,
+          0.4011896848678589,
+          -0.4089202582836151,
+          -0.10938972979784012,
+          0.6726722121238708,
+          0.24576938152313232,
+          -0.24247920513153076,
+          1.1499971151351929,
+          0.47813335061073303,
+          -0.05331678315997124,
+          0.32338133454322815,
+          0.4870913326740265,
+          -0.23144258558750153,
+          -1.2023426294326782,
+          0.2349330335855484,
+          1.080536961555481,
+          0.29334118962287903,
+          0.391574501991272,
+          -0.15818795561790466,
+          -0.2948290705680847,
+          -0.024689948186278343,
+          0.06602869182825089,
+          0.5937030911445618,
+          -0.047901444137096405,
+          -0.512734591960907,
+          -0.35780075192451477,
+          0.28751692175865173,
+          0.4298716187477112,
+          0.9242428541183472,
+          -0.17208744585514069,
+          0.11515070497989656,
+          -0.0335976779460907,
+          -0.3422986567020416,
+          0.5344581604003906,
+          0.19895796477794647,
+          0.33001241087913513,
+          0.6390730142593384,
+          -0.6074934005737305,
+          -0.2553696632385254,
+          0.9644920229911804,
+          0.2699219584465027,
+          0.6403993368148804,
+          -0.6380003690719604,
+          -0.027310986071825027,
+          0.638815701007843,
+          0.27719101309776306,
+          -0.13553589582443237,
+          0.750195324420929,
+          0.1224869191646576,
+          -0.20613941550254825,
+          0.8444448709487915,
+          0.16200250387191772,
+          -0.24750925600528717,
+          -0.739950954914093,
+          -0.28443849086761475,
+          -1.176282525062561,
+          0.516107976436615,
+          0.3774825632572174,
+          0.10906043648719788,
+          0.07962015271186829,
+          0.7384604215621948,
+          -0.051241904497146606,
+          1.1730090379714966,
+          -0.4828610122203827,
+          -1.404372215270996,
+          0.8811132311820984,
+          -0.3839482367038727,
+          0.022516896948218346,
+          -0.0491158664226532,
+          -0.43027013540267944,
+          1.2049334049224854,
+          -0.27309560775756836,
+          0.6883630752563477,
+          0.8264574408531189,
+          -0.5020735263824463,
+          -0.4874092042446137,
+          0.6007202863693237,
+          -0.4965405762195587,
+          1.1302915811538696,
+          0.032572727650403976,
+          -0.3731859028339386,
+          0.658271849155426,
+          -0.9023059010505676,
+          0.7400162220001221,
+          0.014550759457051754,
+          -0.19699542224407196,
+          0.2319706380367279,
+          -0.789058268070221,
+          -0.14905710518360138,
+          -0.5826214551925659,
+          0.207652747631073,
+          -0.4507439732551574,
+          -0.3163885474205017,
+          0.3604124188423157,
+          -0.45119962096214294,
+          0.3428427278995514,
+          0.3005594313144684,
+          -0.36026081442832947,
+          1.1014249324798584,
+          0.40884315967559814,
+          0.34991952776908875,
+          -0.1806638240814209,
+          0.27440476417541504,
+          -0.7118373513221741,
+          0.4645499587059021,
+          0.214790478348732,
+          -0.2343102991580963,
+          0.10500429570674896,
+          -0.28034430742263794,
+          1.2267805337905884,
+          1.0561333894729614,
+          -0.497364342212677,
+          -0.6143305897712708,
+          0.24963727593421936,
+          -0.33136463165283203,
+          -0.01473914459347725,
+          0.495918869972229,
+          -0.6985538005828857,
+          -1.0033197402954102,
+          0.35937801003456116,
+          0.6325868368148804,
+          -0.6808838844299316,
+          1.0354058742523191,
+          -0.7214401960372925,
+          -0.33318862318992615,
+          0.874398410320282,
+          -0.6594992280006409,
+          0.6830640435218811,
+          -0.18534131348133087,
+          0.024834271520376205,
+          0.19901277124881744,
+          -0.5992477536201477,
+          -1.2126628160476685,
+          -0.9245557188987732,
+          -0.3898217976093292,
+          -0.1286519467830658,
+          0.4217943847179413,
+          -0.1143646091222763,
+          0.5630772709846497,
+          -0.5240639448165894,
+          0.21152715384960177,
+          -0.3792001008987427,
+          0.8266305327415466,
+          1.170984387397766,
+          -0.8072142004966736,
+          0.11382893472909927,
+          -0.17953898012638092,
+          -0.1789460331201553,
+          -0.15078622102737427,
+          -1.2082908153533936,
+          -0.7812382578849792,
+          -0.10903695970773696,
+          0.7303897142410278,
+          -0.39054441452026367,
+          0.19511254131793976,
+          -0.09121843427419662,
+          0.22400228679180145,
+          0.30143046379089355,
+          0.1141919493675232,
+          0.48112115263938904,
+          0.7307931780815125,
+          0.09701362252235413,
+          -0.2795647978782654,
+          -0.3997688889503479,
+          0.5540812611579895,
+          0.564578115940094,
+          -0.40065160393714905,
+          -0.3629159033298493,
+          -0.3789091110229492,
+          -0.7298538088798523,
+          -0.6996853351593018,
+          -0.4477842152118683,
+          -0.289089560508728,
+          -0.6430277824401855,
+          0.2344944179058075,
+          0.3742927014827728,
+          -0.5079357028007507,
+          0.28841453790664673,
+          0.06515737622976303,
+          0.707315981388092,
+          0.09498685598373412,
+          0.8365515470504761,
+          0.10002726316452026,
+          -0.7695478200912476,
+          0.6264724135398865,
+          0.7562043070793152,
+          -0.23112858831882477,
+          -0.2871039807796478,
+          -0.25010058283805847,
+          0.2783474028110504,
+          -0.03224996477365494,
+          -0.9119359850883484,
+          -3.6940200328826904,
+          -0.5099936127662659,
+          -0.1604711413383484,
+          0.17453284561634064,
+          0.41759559512138367,
+          0.1419190913438797,
+          -0.11362407356500626,
+          -0.33312007784843445,
+          0.11511333286762238,
+          0.4667884409427643,
+          -0.0031647447030991316,
+          0.15879854559898376,
+          0.3042248487472534,
+          0.5404849052429199,
+          0.8515422344207764,
+          0.06286454200744629,
+          0.43790125846862793,
+          -0.8682025074958801,
+          -0.06363756954669952,
+          0.5547921657562256,
+          -0.01483887154608965,
+          -0.07361344993114471,
+          -0.929947018623352,
+          0.3502565622329712,
+          -0.5080993175506592,
+          1.0380364656448364,
+          -0.2017953395843506,
+          0.21319580078125,
+          -1.0763001441955566,
+          -0.556368887424469,
+          0.1949922740459442,
+          -0.6445739269256592,
+          0.6791343688964844,
+          0.21188358962535855,
+          0.3736183941364288,
+          -0.21800459921360016,
+          0.7597446441650391,
+          -0.3732394874095917,
+          -0.4710160195827484,
+          0.025146087631583217,
+          0.05341297015547752,
+          -0.9522109627723694,
+          -0.6000866889953613,
+          -0.08469046652317047,
+          0.5966026186943054,
+          0.3444081246852875,
+          -0.461188405752182,
+          -0.5279349088668823,
+          0.10296865552663804,
+          0.5175143480300903,
+          -0.20671147108078003,
+          0.13392412662506104,
+          0.4812754988670349,
+          0.2993808686733246,
+          -0.3005635440349579,
+          0.5141698122024536,
+          -0.6239235401153564,
+          0.2877119481563568,
+          -0.4452739953994751,
+          0.5621107816696167,
+          0.5047508478164673,
+          -0.4226335883140564,
+          -0.18578553199768064,
+          -1.1967322826385498,
+          0.28178197145462036,
+          -0.8692031502723694,
+          -1.1812998056411743,
+          -1.4526212215423584,
+          0.4645712077617645,
+          0.9327932000160216,
+          -0.6560136675834656,
+          0.461549699306488,
+          -0.5621527433395386,
+          -1.328449010848999,
+          -0.08676894754171371,
+          0.00021918353741057217,
+          -0.18864136934280396,
+          0.1259666532278061,
+          0.18240638077259064,
+          -0.14919660985469818,
+          -0.8965857625007629,
+          -0.7539900541305542,
+          0.013973715715110302,
+          0.504276692867279,
+          -0.704748272895813,
+          -0.6428424119949341,
+          0.6303996443748474,
+          -0.5404738187789917,
+          -0.31176653504371643,
+          -0.21262824535369873,
+          0.18736739456653595,
+          -0.7998970746994019,
+          0.039946746081113815,
+          0.7390344738960266,
+          0.4283199906349182,
+          0.3795057237148285,
+          0.07204607129096985,
+          -0.9230587482452391,
+          0.9440426230430604,
+          0.26272690296173096,
+          0.5598306655883789,
+          -1.0520871877670288,
+          -0.2677186131477356,
+          -0.1888762265443802,
+          0.30426350235939026,
+          0.4746131896972656,
+          -0.5746733546257019,
+          -0.4197768568992615,
+          0.8565112948417664,
+          -0.6767723560333252,
+          0.23448683321475983,
+          -0.2010004222393036,
+          0.4112907350063324,
+          -0.6497949957847595,
+          -0.418667733669281,
+          -0.4950824975967407,
+          0.44438859820365906,
+          1.026281714439392,
+          0.482397586107254,
+          -0.26220494508743286,
+          -0.3640787005424499,
+          0.5907743573188782,
+          -0.8771642446517944,
+          0.09708411991596222,
+          -0.3671700060367584,
+          0.4331349730491638,
+          0.619417667388916,
+          -0.2684665620326996,
+          -0.5123821496963501,
+          -0.1502324342727661,
+          -0.012190685607492924,
+          0.3580845892429352,
+          0.8617186546325684,
+          0.3493645489215851,
+          1.0270192623138428,
+          0.18297909200191495,
+          -0.5881339311599731,
+          -0.1733516901731491,
+          -0.5040576457977295,
+          -0.340370237827301,
+          -0.26767754554748535,
+          -0.28570041060447693,
+          -0.032928116619586945,
+          0.6029254794120789,
+          0.17397655546665192,
+          0.09346921741962431,
+          0.27815181016921997,
+          -0.46699589490890503,
+          -0.8148876428604126,
+          -0.3964351713657379,
+          0.3812595009803772,
+          0.13547226786613464,
+          0.7126688361167908,
+          -0.3473474085330963,
+          -0.06573959439992905,
+          -0.6483767032623291,
+          1.4808889627456665,
+          0.30924928188323975,
+          -0.5085946917533875,
+          -0.8613000512123108,
+          0.3048902451992035,
+          -0.4241599142551422,
+          0.15909206867218018,
+          0.5764641761779785,
+          -0.07879110425710678,
+          1.015336513519287,
+          0.07599356025457382,
+          -0.7025855779647827,
+          0.30047643184661865,
+          -0.35094937682151794,
+          0.2522146999835968,
+          -0.2338722199201584,
+          -0.8326804637908936,
+          -0.13695412874221802,
+          -0.03452421352267265,
+          0.47974953055381775,
+          -0.18385636806488037,
+          0.32438594102859497,
+          0.1797013282775879,
+          0.787494957447052,
+          -0.12579888105392456,
+          -0.07507286965847015,
+          -0.4389670491218567,
+          0.2720070779323578,
+          0.8138866424560547,
+          0.01974171027541161,
+          -0.3057698905467987,
+          -0.6709924936294556,
+          0.0885881632566452,
+          -0.2862754464149475,
+          0.03475658595561981,
+          -0.1285519152879715,
+          0.3838353455066681,
+          -0.2944154739379883,
+          -0.4204859137535095,
+          -0.4416137933731079,
+          0.13426260650157928,
+          0.36733248829841614,
+          0.573428750038147,
+          -0.14928072690963745,
+          -0.026076916605234143,
+          0.33286052942276,
+          -0.5340145826339722,
+          -0.17279052734375,
+          -0.01154550164937973,
+          -0.6620771884918213,
+          0.18390542268753052,
+          -0.08265615254640579,
+          -0.2489682286977768,
+          0.2429984211921692,
+          -0.044153645634651184,
+          -0.986578404903412,
+          -0.33574509620666504,
+          -0.5387663841247559,
+          0.19767941534519196,
+          0.12540718913078308,
+          -0.3403128981590271,
+          -0.4154576361179352,
+          0.17275673151016235,
+          0.09407442808151244,
+          -0.5414086580276489,
+          0.4393929839134216,
+          0.1725579798221588,
+          -0.4998118281364441,
+          -0.6926208138465881,
+          0.16552448272705078,
+          0.6659538149833679,
+          -0.10949844866991044,
+          0.986426830291748,
+          0.01748848147690296,
+          0.4003709554672241,
+          -0.5430638194084167,
+          0.35347291827201843,
+          0.6887399554252625,
+          0.08274628221988678,
+          0.13407137989997864,
+          -0.591465950012207,
+          0.3446292281150818,
+          0.6069018244743347,
+          0.1935492902994156,
+          -0.0989871397614479,
+          0.07008486241102219,
+          -0.8503749370574951,
+          -0.09507356584072112,
+          0.6259510517120361,
+          0.13934025168418884,
+          0.06392545253038406,
+          -0.4112265408039093,
+          -0.08475656062364578,
+          0.4974113404750824,
+          -0.30606114864349365,
+          1.111435890197754,
+          -0.018766529858112335,
+          -0.8422622680664063,
+          0.4325508773326874,
+          -0.2832120656967163,
+          -0.4859798848628998,
+          -0.41498348116874695,
+          0.015977520495653152,
+          0.5292825698852539,
+          0.4538311660289765,
+          1.1328668594360352,
+          0.22632671892642975,
+          0.7918671369552612,
+          0.33401933312416077,
+          0.7306135296821594,
+          0.3548600673675537,
+          0.12506209313869476,
+          0.8573207855224609,
+          -0.5818327069282532,
+          -0.6953738927841187,
+          -1.6171947717666626,
+          -0.1699674427509308,
+          0.6318262815475464,
+          -0.05671752244234085,
+          -0.28145185112953186,
+          -0.3976689279079437,
+          -0.2041076272726059,
+          -0.5495951175689697,
+          -0.5152917504310608,
+          -0.9309796094894408,
+          0.101932130753994,
+          0.1367802917957306,
+          0.1490798443555832,
+          0.5304336547851563,
+          -0.5082434415817261,
+          0.06688683480024338,
+          0.14657628536224365,
+          -0.782435953617096,
+          0.2962816655635834,
+          0.6965363621711731,
+          0.8496337532997131,
+          -0.3042965829372406,
+          0.04343798756599426,
+          0.0330701619386673,
+          -0.5662598013877869,
+          1.1086925268173218,
+          0.756072998046875,
+          -0.204134538769722,
+          0.2404300570487976,
+          -0.47848284244537354,
+          1.3659011125564575,
+          0.5645433068275452,
+          -0.15836156904697418,
+          0.43395575881004333,
+          0.5944653749465942,
+          1.0043466091156006,
+          -0.49446743726730347,
+          -0.5954391360282898,
+          0.5341240763664246,
+          0.020598189905285835,
+          -0.4036853015422821,
+          0.4473709762096405,
+          1.1998231410980225,
+          -0.9317775368690492,
+          -0.23321466147899628,
+          0.2052552700042725,
+          -0.7423108816146851,
+          -0.19917210936546328,
+          -0.1722569614648819,
+          -0.034072667360305786,
+          -0.00671181408688426,
+          0.46396249532699585,
+          -0.1372445821762085,
+          0.053376372903585434,
+          0.7392690777778625,
+          -0.38447609543800354,
+          0.07497968524694443,
+          0.5197252631187439,
+          1.3746477365493774,
+          0.9060075879096984,
+          0.20000585913658145,
+          -0.4053704142570496,
+          0.7497360110282898,
+          -0.34087055921554565,
+          -1.101803183555603,
+          0.273650586605072,
+          -0.5125769376754761,
+          0.22472351789474487,
+          0.480757474899292,
+          -0.19845178723335263,
+          0.8857700824737549,
+          0.30752456188201904,
+          1.1109285354614258,
+          -0.6768012642860413,
+          0.524367094039917,
+          -0.22495046257972717,
+          -0.4224412739276886,
+          0.40753406286239624,
+          -0.23133376240730288,
+          0.3297771215438843,
+          0.4905449151992798,
+          -0.6813114285469055,
+          -0.7543983459472656,
+          -0.5599071383476257,
+          0.14351597428321838,
+          -0.029278717935085297,
+          -0.3970443606376648,
+          -0.303079217672348,
+          0.24161772429943085,
+          0.008353390730917454,
+          -0.0062365154735744,
+          1.0824860334396362,
+          -0.3704061508178711,
+          -1.0337258577346802,
+          0.04638749733567238,
+          1.163011074066162,
+          -0.31737643480300903,
+          0.013986887410283089,
+          0.19223114848136905,
+          -0.2260770797729492,
+          -0.210910826921463,
+          -1.0191949605941772,
+          0.22356095910072327,
+          0.09353553503751756,
+          0.18096882104873657,
+          0.14867214858531952,
+          0.43408671021461487,
+          -0.33312076330184937,
+          0.8173948526382446,
+          0.6428242921829224,
+          0.20215003192424777,
+          -0.6634518504142761,
+          -0.4132290482521057,
+          0.29815030097961426,
+          -1.579406976699829,
+          -0.0981958732008934,
+          -0.03941014781594277,
+          0.1709178239107132,
+          -0.5481140613555908,
+          -0.5338194966316223,
+          -0.3528362512588501,
+          -0.11561278253793716,
+          -0.21793591976165771,
+          -1.1570470333099363,
+          0.2157980799674988,
+          0.42083489894866943,
+          0.9639263153076172,
+          0.09747201204299928,
+          0.15671424567699432,
+          0.4034591615200043,
+          0.6728067994117737,
+          -0.5216875672340393,
+          0.09657668322324751,
+          -0.2416689097881317,
+          0.747975766658783,
+          0.1021689772605896,
+          0.11652665585279463,
+          -1.0484966039657593,
+          0.8489304780960083,
+          0.7169828414916992,
+          -0.09012343734502792,
+          -1.3173753023147583,
+          0.057890523225069046,
+          -0.006231260951608419,
+          -0.1018214002251625,
+          0.936040461063385,
+          -0.0502331368625164,
+          -0.4284322261810303,
+          -0.38209280371665955,
+          -0.22668412327766416,
+          0.0782942995429039,
+          -0.4881664514541626,
+          0.9268959760665894,
+          0.001867273123934865,
+          0.42261114716529846,
+          0.8283362984657288,
+          0.4256294071674347,
+          -0.7965338826179504,
+          0.4840078353881836,
+          -0.19861412048339844,
+          0.33977967500686646,
+          -0.4604192078113556,
+          -0.3107339143753052,
+          -0.2839638590812683,
+          -1.5734281539916992,
+          0.005220232997089624,
+          0.09239906817674635,
+          -0.7828494906425476,
+          -0.1397123783826828,
+          0.2576255202293396,
+          0.21372435986995697,
+          -0.23169949650764465,
+          0.4016408920288086,
+          -0.462497353553772,
+          -0.2186472862958908,
+          -0.5617868900299072,
+          -0.3649831712245941,
+          -1.1585862636566162,
+          -0.08222806453704834,
+          0.931126832962036,
+          0.4327389597892761,
+          -0.46451422572135925,
+          -0.5430706143379211,
+          -0.27434298396110535,
+          -0.9479129314422609,
+          0.1845661848783493,
+          0.3972720205783844,
+          0.4883299469947815,
+          1.04031240940094
+        ]
+      ],
+      "regenerate": true
+    }
+  }
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-8.snap

@@ -0,0 +1,780 @@
+---
+source: dump/src/reader/mod.rs
+expression: document
+---
+{
+  "id": "e0",
+  "desc": "overriden vector",
+  "_vectors": {
+    "default": [
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1
+    ]
+  }
+}

dump/src/reader/v2/mod.rs

@@ -252,7 +252,7 @@ pub(crate) mod test {
         let mut dump = V2Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
@@ -349,7 +349,7 @@ pub(crate) mod test {
         let mut dump = V2Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/v3/mod.rs

@@ -267,7 +267,7 @@ pub(crate) mod test {
         let mut dump = V3Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/v3/settings.rs

@@ -152,6 +152,7 @@ impl Settings<Unchecked> {
 }
 
 #[derive(Debug, Clone, Deserialize)]
+#[allow(dead_code)] // otherwise rustc complains that the fields go unused
 #[cfg_attr(test, derive(serde::Serialize))]
 #[serde(deny_unknown_fields)]
 #[serde(rename_all = "camelCase")]

dump/src/reader/v4/mod.rs

@@ -262,8 +262,8 @@ pub(crate) mod test {
         let mut dump = V4Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/v4/settings.rs

@@ -182,6 +182,7 @@ impl Settings<Unchecked> {
     }
 }
 
+#[allow(dead_code)] // otherwise rustc complains that the fields go unused
 #[derive(Debug, Clone, Deserialize)]
 #[cfg_attr(test, derive(serde::Serialize))]
 #[serde(deny_unknown_fields)]

dump/src/reader/v5/mod.rs

@@ -299,8 +299,8 @@ pub(crate) mod test {
         let mut dump = V5Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
+        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/v5/tasks.rs

@@ -200,6 +200,7 @@ impl std::ops::Deref for IndexUid {
     }
 }
 
+#[allow(dead_code)] // otherwise rustc complains that the fields go unused
 #[derive(Debug)]
 #[cfg_attr(test, derive(serde::Serialize))]
 #[cfg_attr(test, serde(rename_all = "camelCase"))]

dump/src/writer.rs

@@ -281,7 +281,7 @@ pub(crate) mod test {
         let dump_path = dump.path();
 
         // ==== checking global file hierarchy (we want to be sure there isn't too many files or too few)
-        insta::assert_display_snapshot!(create_directory_hierarchy(dump_path), @r###"
+        insta::assert_snapshot!(create_directory_hierarchy(dump_path), @r###"
         .
         ├---- indexes/
         │    └---- doggos/

file-store/Cargo.toml

@@ -11,10 +11,7 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-tempfile = "3.9.0"
-thiserror = "1.0.56"
+tempfile = "3.10.1"
+thiserror = "1.0.61"
 tracing = "0.1.40"
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
-
-[dev-dependencies]
-faux = "0.1.10"
+uuid = { version = "1.10.0", features = ["serde", "v4"] }

filter-parser/Cargo.toml

@@ -14,7 +14,7 @@ license.workspace = true
 [dependencies]
 nom = "7.1.3"
 nom_locate = "4.2.0"
-unescaper = "0.1.3"
+unescaper = "0.1.5"
 
 [dev-dependencies]
-insta = "1.34.0"
+insta = "1.39.0"

filter-parser/src/lib.rs

@@ -564,121 +564,121 @@ pub mod tests {
 
     #[test]
     fn parse_escaped() {
-        insta::assert_display_snapshot!(p(r"title = 'foo\\'"), @r#"{title} = {foo\}"#);
-        insta::assert_display_snapshot!(p(r"title = 'foo\\\\'"), @r#"{title} = {foo\\}"#);
-        insta::assert_display_snapshot!(p(r"title = 'foo\\\\\\'"), @r#"{title} = {foo\\\}"#);
-        insta::assert_display_snapshot!(p(r"title = 'foo\\\\\\\\'"), @r#"{title} = {foo\\\\}"#);
-        // but it also works with other sequencies
-        insta::assert_display_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}");
+        insta::assert_snapshot!(p(r"title = 'foo\\'"), @r#"{title} = {foo\}"#);
+        insta::assert_snapshot!(p(r"title = 'foo\\\\'"), @r#"{title} = {foo\\}"#);
+        insta::assert_snapshot!(p(r"title = 'foo\\\\\\'"), @r#"{title} = {foo\\\}"#);
+        insta::assert_snapshot!(p(r"title = 'foo\\\\\\\\'"), @r#"{title} = {foo\\\\}"#);
+        // but it also works with other sequences
+        insta::assert_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}");
     }
 
     #[test]
     fn parse() {
         // Test equal
-        insta::assert_display_snapshot!(p("channel = Ponce"), @"{channel} = {Ponce}");
-        insta::assert_display_snapshot!(p("subscribers = 12"), @"{subscribers} = {12}");
-        insta::assert_display_snapshot!(p("channel = 'Mister Mv'"), @"{channel} = {Mister Mv}");
-        insta::assert_display_snapshot!(p("channel = \"Mister Mv\""), @"{channel} = {Mister Mv}");
-        insta::assert_display_snapshot!(p("'dog race' = Borzoi"), @"{dog race} = {Borzoi}");
-        insta::assert_display_snapshot!(p("\"dog race\" = Chusky"), @"{dog race} = {Chusky}");
-        insta::assert_display_snapshot!(p("\"dog race\" = \"Bernese Mountain\""), @"{dog race} = {Bernese Mountain}");
-        insta::assert_display_snapshot!(p("'dog race' = 'Bernese Mountain'"), @"{dog race} = {Bernese Mountain}");
-        insta::assert_display_snapshot!(p("\"dog race\" = 'Bernese Mountain'"), @"{dog race} = {Bernese Mountain}");
+        insta::assert_snapshot!(p("channel = Ponce"), @"{channel} = {Ponce}");
+        insta::assert_snapshot!(p("subscribers = 12"), @"{subscribers} = {12}");
+        insta::assert_snapshot!(p("channel = 'Mister Mv'"), @"{channel} = {Mister Mv}");
+        insta::assert_snapshot!(p("channel = \"Mister Mv\""), @"{channel} = {Mister Mv}");
+        insta::assert_snapshot!(p("'dog race' = Borzoi"), @"{dog race} = {Borzoi}");
+        insta::assert_snapshot!(p("\"dog race\" = Chusky"), @"{dog race} = {Chusky}");
+        insta::assert_snapshot!(p("\"dog race\" = \"Bernese Mountain\""), @"{dog race} = {Bernese Mountain}");
+        insta::assert_snapshot!(p("'dog race' = 'Bernese Mountain'"), @"{dog race} = {Bernese Mountain}");
+        insta::assert_snapshot!(p("\"dog race\" = 'Bernese Mountain'"), @"{dog race} = {Bernese Mountain}");
 
         // Test IN
-        insta::assert_display_snapshot!(p("colour IN[]"), @"{colour} IN[]");
-        insta::assert_display_snapshot!(p("colour IN[green]"), @"{colour} IN[{green}, ]");
-        insta::assert_display_snapshot!(p("colour IN[green,]"), @"{colour} IN[{green}, ]");
-        insta::assert_display_snapshot!(p("colour NOT IN[green,blue]"), @"NOT ({colour} IN[{green}, {blue}, ])");
-        insta::assert_display_snapshot!(p(" colour IN [  green , blue , ]"), @"{colour} IN[{green}, {blue}, ]");
+        insta::assert_snapshot!(p("colour IN[]"), @"{colour} IN[]");
+        insta::assert_snapshot!(p("colour IN[green]"), @"{colour} IN[{green}, ]");
+        insta::assert_snapshot!(p("colour IN[green,]"), @"{colour} IN[{green}, ]");
+        insta::assert_snapshot!(p("colour NOT IN[green,blue]"), @"NOT ({colour} IN[{green}, {blue}, ])");
+        insta::assert_snapshot!(p(" colour IN [  green , blue , ]"), @"{colour} IN[{green}, {blue}, ]");
 
         // Test IN + OR/AND/()
-        insta::assert_display_snapshot!(p(" colour IN [green, blue]  AND color = green "), @"AND[{colour} IN[{green}, {blue}, ], {color} = {green}, ]");
-        insta::assert_display_snapshot!(p("NOT (colour IN [green, blue])  AND color = green "), @"AND[NOT ({colour} IN[{green}, {blue}, ]), {color} = {green}, ]");
-        insta::assert_display_snapshot!(p("x = 1 OR NOT (colour IN [green, blue]  OR color = green) "), @"OR[{x} = {1}, NOT (OR[{colour} IN[{green}, {blue}, ], {color} = {green}, ]), ]");
+        insta::assert_snapshot!(p(" colour IN [green, blue]  AND color = green "), @"AND[{colour} IN[{green}, {blue}, ], {color} = {green}, ]");
+        insta::assert_snapshot!(p("NOT (colour IN [green, blue])  AND color = green "), @"AND[NOT ({colour} IN[{green}, {blue}, ]), {color} = {green}, ]");
+        insta::assert_snapshot!(p("x = 1 OR NOT (colour IN [green, blue]  OR color = green) "), @"OR[{x} = {1}, NOT (OR[{colour} IN[{green}, {blue}, ], {color} = {green}, ]), ]");
 
         // Test whitespace start/end
-        insta::assert_display_snapshot!(p(" colour = green "), @"{colour} = {green}");
-        insta::assert_display_snapshot!(p(" (colour = green OR colour = red) "), @"OR[{colour} = {green}, {colour} = {red}, ]");
-        insta::assert_display_snapshot!(p(" colour IN [green, blue]  AND color = green "), @"AND[{colour} IN[{green}, {blue}, ], {color} = {green}, ]");
-        insta::assert_display_snapshot!(p(" colour NOT  IN [green, blue] "), @"NOT ({colour} IN[{green}, {blue}, ])");
-        insta::assert_display_snapshot!(p(" colour IN [green, blue] "), @"{colour} IN[{green}, {blue}, ]");
+        insta::assert_snapshot!(p(" colour = green "), @"{colour} = {green}");
+        insta::assert_snapshot!(p(" (colour = green OR colour = red) "), @"OR[{colour} = {green}, {colour} = {red}, ]");
+        insta::assert_snapshot!(p(" colour IN [green, blue]  AND color = green "), @"AND[{colour} IN[{green}, {blue}, ], {color} = {green}, ]");
+        insta::assert_snapshot!(p(" colour NOT  IN [green, blue] "), @"NOT ({colour} IN[{green}, {blue}, ])");
+        insta::assert_snapshot!(p(" colour IN [green, blue] "), @"{colour} IN[{green}, {blue}, ]");
 
         // Test conditions
-        insta::assert_display_snapshot!(p("channel != ponce"), @"{channel} != {ponce}");
-        insta::assert_display_snapshot!(p("NOT channel = ponce"), @"NOT ({channel} = {ponce})");
-        insta::assert_display_snapshot!(p("subscribers < 1000"), @"{subscribers} < {1000}");
-        insta::assert_display_snapshot!(p("subscribers > 1000"), @"{subscribers} > {1000}");
-        insta::assert_display_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
-        insta::assert_display_snapshot!(p("subscribers >= 1000"), @"{subscribers} >= {1000}");
-        insta::assert_display_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
-        insta::assert_display_snapshot!(p("subscribers 100 TO 1000"), @"{subscribers} {100} TO {1000}");
+        insta::assert_snapshot!(p("channel != ponce"), @"{channel} != {ponce}");
+        insta::assert_snapshot!(p("NOT channel = ponce"), @"NOT ({channel} = {ponce})");
+        insta::assert_snapshot!(p("subscribers < 1000"), @"{subscribers} < {1000}");
+        insta::assert_snapshot!(p("subscribers > 1000"), @"{subscribers} > {1000}");
+        insta::assert_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
+        insta::assert_snapshot!(p("subscribers >= 1000"), @"{subscribers} >= {1000}");
+        insta::assert_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
+        insta::assert_snapshot!(p("subscribers 100 TO 1000"), @"{subscribers} {100} TO {1000}");
 
         // Test NOT
-        insta::assert_display_snapshot!(p("NOT subscribers < 1000"), @"NOT ({subscribers} < {1000})");
-        insta::assert_display_snapshot!(p("NOT subscribers 100 TO 1000"), @"NOT ({subscribers} {100} TO {1000})");
+        insta::assert_snapshot!(p("NOT subscribers < 1000"), @"NOT ({subscribers} < {1000})");
+        insta::assert_snapshot!(p("NOT subscribers 100 TO 1000"), @"NOT ({subscribers} {100} TO {1000})");
 
         // Test NULL + NOT NULL
-        insta::assert_display_snapshot!(p("subscribers IS NULL"), @"{subscribers} IS NULL");
-        insta::assert_display_snapshot!(p("NOT subscribers IS NULL"), @"NOT ({subscribers} IS NULL)");
-        insta::assert_display_snapshot!(p("subscribers IS NOT NULL"), @"NOT ({subscribers} IS NULL)");
-        insta::assert_display_snapshot!(p("NOT subscribers IS NOT NULL"), @"{subscribers} IS NULL");
-        insta::assert_display_snapshot!(p("subscribers  IS   NOT   NULL"), @"NOT ({subscribers} IS NULL)");
+        insta::assert_snapshot!(p("subscribers IS NULL"), @"{subscribers} IS NULL");
+        insta::assert_snapshot!(p("NOT subscribers IS NULL"), @"NOT ({subscribers} IS NULL)");
+        insta::assert_snapshot!(p("subscribers IS NOT NULL"), @"NOT ({subscribers} IS NULL)");
+        insta::assert_snapshot!(p("NOT subscribers IS NOT NULL"), @"{subscribers} IS NULL");
+        insta::assert_snapshot!(p("subscribers  IS   NOT   NULL"), @"NOT ({subscribers} IS NULL)");
 
         // Test EMPTY + NOT EMPTY
-        insta::assert_display_snapshot!(p("subscribers IS EMPTY"), @"{subscribers} IS EMPTY");
-        insta::assert_display_snapshot!(p("NOT subscribers IS EMPTY"), @"NOT ({subscribers} IS EMPTY)");
-        insta::assert_display_snapshot!(p("subscribers IS NOT EMPTY"), @"NOT ({subscribers} IS EMPTY)");
-        insta::assert_display_snapshot!(p("NOT subscribers IS NOT EMPTY"), @"{subscribers} IS EMPTY");
-        insta::assert_display_snapshot!(p("subscribers  IS   NOT   EMPTY"), @"NOT ({subscribers} IS EMPTY)");
+        insta::assert_snapshot!(p("subscribers IS EMPTY"), @"{subscribers} IS EMPTY");
+        insta::assert_snapshot!(p("NOT subscribers IS EMPTY"), @"NOT ({subscribers} IS EMPTY)");
+        insta::assert_snapshot!(p("subscribers IS NOT EMPTY"), @"NOT ({subscribers} IS EMPTY)");
+        insta::assert_snapshot!(p("NOT subscribers IS NOT EMPTY"), @"{subscribers} IS EMPTY");
+        insta::assert_snapshot!(p("subscribers  IS   NOT   EMPTY"), @"NOT ({subscribers} IS EMPTY)");
 
         // Test EXISTS + NOT EXITS
-        insta::assert_display_snapshot!(p("subscribers EXISTS"), @"{subscribers} EXISTS");
-        insta::assert_display_snapshot!(p("NOT subscribers EXISTS"), @"NOT ({subscribers} EXISTS)");
-        insta::assert_display_snapshot!(p("subscribers NOT EXISTS"), @"NOT ({subscribers} EXISTS)");
-        insta::assert_display_snapshot!(p("NOT subscribers NOT EXISTS"), @"{subscribers} EXISTS");
-        insta::assert_display_snapshot!(p("subscribers NOT   EXISTS"), @"NOT ({subscribers} EXISTS)");
+        insta::assert_snapshot!(p("subscribers EXISTS"), @"{subscribers} EXISTS");
+        insta::assert_snapshot!(p("NOT subscribers EXISTS"), @"NOT ({subscribers} EXISTS)");
+        insta::assert_snapshot!(p("subscribers NOT EXISTS"), @"NOT ({subscribers} EXISTS)");
+        insta::assert_snapshot!(p("NOT subscribers NOT EXISTS"), @"{subscribers} EXISTS");
+        insta::assert_snapshot!(p("subscribers NOT   EXISTS"), @"NOT ({subscribers} EXISTS)");
 
         // Test nested NOT
-        insta::assert_display_snapshot!(p("NOT NOT NOT NOT x = 5"), @"{x} = {5}");
-        insta::assert_display_snapshot!(p("NOT NOT (NOT NOT x = 5)"), @"{x} = {5}");
+        insta::assert_snapshot!(p("NOT NOT NOT NOT x = 5"), @"{x} = {5}");
+        insta::assert_snapshot!(p("NOT NOT (NOT NOT x = 5)"), @"{x} = {5}");
 
         // Test geo radius
-        insta::assert_display_snapshot!(p("_geoRadius(12, 13, 14)"), @"_geoRadius({12}, {13}, {14})");
-        insta::assert_display_snapshot!(p("NOT _geoRadius(12, 13, 14)"), @"NOT (_geoRadius({12}, {13}, {14}))");
-        insta::assert_display_snapshot!(p("_geoRadius(12,13,14)"), @"_geoRadius({12}, {13}, {14})");
+        insta::assert_snapshot!(p("_geoRadius(12, 13, 14)"), @"_geoRadius({12}, {13}, {14})");
+        insta::assert_snapshot!(p("NOT _geoRadius(12, 13, 14)"), @"NOT (_geoRadius({12}, {13}, {14}))");
+        insta::assert_snapshot!(p("_geoRadius(12,13,14)"), @"_geoRadius({12}, {13}, {14})");
 
         // Test geo bounding box
-        insta::assert_display_snapshot!(p("_geoBoundingBox([12, 13], [14, 15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
-        insta::assert_display_snapshot!(p("NOT _geoBoundingBox([12, 13], [14, 15])"), @"NOT (_geoBoundingBox([{12}, {13}], [{14}, {15}]))");
-        insta::assert_display_snapshot!(p("_geoBoundingBox([12,13],[14,15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
+        insta::assert_snapshot!(p("_geoBoundingBox([12, 13], [14, 15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
+        insta::assert_snapshot!(p("NOT _geoBoundingBox([12, 13], [14, 15])"), @"NOT (_geoBoundingBox([{12}, {13}], [{14}, {15}]))");
+        insta::assert_snapshot!(p("_geoBoundingBox([12,13],[14,15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
 
         // Test OR + AND
-        insta::assert_display_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain'"), @"AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
-        insta::assert_display_snapshot!(p("channel = ponce OR 'dog race' != 'bernese mountain'"), @"OR[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
-        insta::assert_display_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000"), @"OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, ]");
-        insta::assert_display_snapshot!(
+        insta::assert_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain'"), @"AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
+        insta::assert_snapshot!(p("channel = ponce OR 'dog race' != 'bernese mountain'"), @"OR[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
+        insta::assert_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000"), @"OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, ]");
+        insta::assert_snapshot!(
         p("channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000 OR colour = red OR colour = blue AND size = 7"),
         @"OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, {colour} = {red}, AND[{colour} = {blue}, {size} = {7}, ], ]"
         );
 
         // Test parentheses
-        insta::assert_display_snapshot!(p("channel = ponce AND ( 'dog race' != 'bernese mountain' OR subscribers > 1000 )"), @"AND[{channel} = {ponce}, OR[{dog race} != {bernese mountain}, {subscribers} > {1000}, ], ]");
-        insta::assert_display_snapshot!(p("(channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000) AND _geoRadius(12, 13, 14)"), @"AND[OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, ], _geoRadius({12}, {13}, {14}), ]");
+        insta::assert_snapshot!(p("channel = ponce AND ( 'dog race' != 'bernese mountain' OR subscribers > 1000 )"), @"AND[{channel} = {ponce}, OR[{dog race} != {bernese mountain}, {subscribers} > {1000}, ], ]");
+        insta::assert_snapshot!(p("(channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000) AND _geoRadius(12, 13, 14)"), @"AND[OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, ], _geoRadius({12}, {13}, {14}), ]");
 
         // Test recursion
         // This is the most that is allowed
-        insta::assert_display_snapshot!(
+        insta::assert_snapshot!(
             p("(((((((((((((((((((((((((((((((((((((((((((((((((x = 1)))))))))))))))))))))))))))))))))))))))))))))))))"),
             @"{x} = {1}"
         );
-        insta::assert_display_snapshot!(
+        insta::assert_snapshot!(
             p("NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT x = 1"),
             @"NOT ({x} = {1})"
         );
 
         // Confusing keywords
-        insta::assert_display_snapshot!(p(r#"NOT "OR" EXISTS AND "EXISTS" NOT EXISTS"#), @"AND[NOT ({OR} EXISTS), NOT ({EXISTS} EXISTS), ]");
+        insta::assert_snapshot!(p(r#"NOT "OR" EXISTS AND "EXISTS" NOT EXISTS"#), @"AND[NOT ({OR} EXISTS), NOT ({EXISTS} EXISTS), ]");
     }
 
     #[test]
@@ -689,182 +689,182 @@ pub mod tests {
             Fc::parse(s).unwrap_err().to_string()
         }
 
-        insta::assert_display_snapshot!(p("channel = Ponce = 12"), @r###"
+        insta::assert_snapshot!(p("channel = Ponce = 12"), @r###"
         Found unexpected characters at the end of the filter: `= 12`. You probably forgot an `OR` or an `AND` rule.
         17:21 channel = Ponce = 12
         "###);
 
-        insta::assert_display_snapshot!(p("channel =    "), @r###"
+        insta::assert_snapshot!(p("channel =    "), @r###"
         Was expecting a value but instead got nothing.
         14:14 channel =
         "###);
 
-        insta::assert_display_snapshot!(p("channel = 🐻"), @r###"
+        insta::assert_snapshot!(p("channel = 🐻"), @r###"
         Was expecting a value but instead got `🐻`.
         11:12 channel = 🐻
         "###);
 
-        insta::assert_display_snapshot!(p("channel = 🐻 AND followers < 100"), @r###"
+        insta::assert_snapshot!(p("channel = 🐻 AND followers < 100"), @r###"
         Was expecting a value but instead got `🐻`.
         11:12 channel = 🐻 AND followers < 100
         "###);
 
-        insta::assert_display_snapshot!(p("'OR'"), @r###"
+        insta::assert_snapshot!(p("'OR'"), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `\'OR\'`.
         1:5 'OR'
         "###);
 
-        insta::assert_display_snapshot!(p("OR"), @r###"
+        insta::assert_snapshot!(p("OR"), @r###"
         Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes.
         1:3 OR
         "###);
 
-        insta::assert_display_snapshot!(p("channel Ponce"), @r###"
+        insta::assert_snapshot!(p("channel Ponce"), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `channel Ponce`.
         1:14 channel Ponce
         "###);
 
-        insta::assert_display_snapshot!(p("channel = Ponce OR"), @r###"
+        insta::assert_snapshot!(p("channel = Ponce OR"), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.
         19:19 channel = Ponce OR
         "###);
 
-        insta::assert_display_snapshot!(p("_geoRadius"), @r###"
+        insta::assert_snapshot!(p("_geoRadius"), @r###"
         The `_geoRadius` filter expects three arguments: `_geoRadius(latitude, longitude, radius)`.
         1:11 _geoRadius
         "###);
 
-        insta::assert_display_snapshot!(p("_geoRadius = 12"), @r###"
+        insta::assert_snapshot!(p("_geoRadius = 12"), @r###"
         The `_geoRadius` filter expects three arguments: `_geoRadius(latitude, longitude, radius)`.
         1:16 _geoRadius = 12
         "###);
 
-        insta::assert_display_snapshot!(p("_geoBoundingBox"), @r###"
+        insta::assert_snapshot!(p("_geoBoundingBox"), @r###"
         The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
         1:16 _geoBoundingBox
         "###);
 
-        insta::assert_display_snapshot!(p("_geoBoundingBox = 12"), @r###"
+        insta::assert_snapshot!(p("_geoBoundingBox = 12"), @r###"
         The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
         1:21 _geoBoundingBox = 12
         "###);
 
-        insta::assert_display_snapshot!(p("_geoBoundingBox(1.0, 1.0)"), @r###"
+        insta::assert_snapshot!(p("_geoBoundingBox(1.0, 1.0)"), @r###"
         The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
         1:26 _geoBoundingBox(1.0, 1.0)
         "###);
 
-        insta::assert_display_snapshot!(p("_geoPoint(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("_geoPoint(12, 13, 14)"), @r###"
         `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
         1:22 _geoPoint(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("position <= _geoPoint(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("position <= _geoPoint(12, 13, 14)"), @r###"
         `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
         13:34 position <= _geoPoint(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("_geoDistance(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("_geoDistance(12, 13, 14)"), @r###"
         `_geoDistance` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
         1:25 _geoDistance(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("position <= _geoDistance(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("position <= _geoDistance(12, 13, 14)"), @r###"
         `_geoDistance` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
         13:37 position <= _geoDistance(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("_geo(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("_geo(12, 13, 14)"), @r###"
         `_geo` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
         1:17 _geo(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("position <= _geo(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("position <= _geo(12, 13, 14)"), @r###"
         `_geo` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
         13:29 position <= _geo(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("position <= _geoRadius(12, 13, 14)"), @r###"
+        insta::assert_snapshot!(p("position <= _geoRadius(12, 13, 14)"), @r###"
         The `_geoRadius` filter is an operation and can't be used as a value.
         13:35 position <= _geoRadius(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("channel = 'ponce"), @r###"
+        insta::assert_snapshot!(p("channel = 'ponce"), @r###"
         Expression `\'ponce` is missing the following closing delimiter: `'`.
         11:17 channel = 'ponce
         "###);
 
-        insta::assert_display_snapshot!(p("channel = \"ponce"), @r###"
+        insta::assert_snapshot!(p("channel = \"ponce"), @r###"
         Expression `\"ponce` is missing the following closing delimiter: `"`.
         11:17 channel = "ponce
         "###);
 
-        insta::assert_display_snapshot!(p("channel = mv OR (followers >= 1000"), @r###"
+        insta::assert_snapshot!(p("channel = mv OR (followers >= 1000"), @r###"
         Expression `(followers >= 1000` is missing the following closing delimiter: `)`.
         17:35 channel = mv OR (followers >= 1000
         "###);
 
-        insta::assert_display_snapshot!(p("channel = mv OR followers >= 1000)"), @r###"
+        insta::assert_snapshot!(p("channel = mv OR followers >= 1000)"), @r###"
         Found unexpected characters at the end of the filter: `)`. You probably forgot an `OR` or an `AND` rule.
         34:35 channel = mv OR followers >= 1000)
         "###);
 
-        insta::assert_display_snapshot!(p("colour NOT EXIST"), @r###"
+        insta::assert_snapshot!(p("colour NOT EXIST"), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `colour NOT EXIST`.
         1:17 colour NOT EXIST
         "###);
 
-        insta::assert_display_snapshot!(p("subscribers 100 TO1000"), @r###"
+        insta::assert_snapshot!(p("subscribers 100 TO1000"), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `subscribers 100 TO1000`.
         1:23 subscribers 100 TO1000
         "###);
 
-        insta::assert_display_snapshot!(p("channel = ponce ORdog != 'bernese mountain'"), @r###"
+        insta::assert_snapshot!(p("channel = ponce ORdog != 'bernese mountain'"), @r###"
         Found unexpected characters at the end of the filter: `ORdog != \'bernese mountain\'`. You probably forgot an `OR` or an `AND` rule.
         17:44 channel = ponce ORdog != 'bernese mountain'
         "###);
 
-        insta::assert_display_snapshot!(p("colour IN blue, green]"), @r###"
+        insta::assert_snapshot!(p("colour IN blue, green]"), @r###"
         Expected `[` after `IN` keyword.
         11:23 colour IN blue, green]
         "###);
 
-        insta::assert_display_snapshot!(p("colour IN [blue, green, 'blue' > 2]"), @r###"
+        insta::assert_snapshot!(p("colour IN [blue, green, 'blue' > 2]"), @r###"
         Expected only comma-separated field names inside `IN[..]` but instead found `> 2]`.
         32:36 colour IN [blue, green, 'blue' > 2]
         "###);
 
-        insta::assert_display_snapshot!(p("colour IN [blue, green, AND]"), @r###"
+        insta::assert_snapshot!(p("colour IN [blue, green, AND]"), @r###"
         Expected only comma-separated field names inside `IN[..]` but instead found `AND]`.
         25:29 colour IN [blue, green, AND]
         "###);
 
-        insta::assert_display_snapshot!(p("colour IN [blue, green"), @r###"
+        insta::assert_snapshot!(p("colour IN [blue, green"), @r###"
         Expected matching `]` after the list of field names given to `IN[`
         23:23 colour IN [blue, green
         "###);
 
-        insta::assert_display_snapshot!(p("colour IN ['blue, green"), @r###"
+        insta::assert_snapshot!(p("colour IN ['blue, green"), @r###"
         Expression `\'blue, green` is missing the following closing delimiter: `'`.
         12:24 colour IN ['blue, green
         "###);
 
-        insta::assert_display_snapshot!(p("x = EXISTS"), @r###"
+        insta::assert_snapshot!(p("x = EXISTS"), @r###"
         Was expecting a value but instead got `EXISTS`, which is a reserved keyword. To use `EXISTS` as a field name or a value, surround it by quotes.
         5:11 x = EXISTS
         "###);
 
-        insta::assert_display_snapshot!(p("AND = 8"), @r###"
+        insta::assert_snapshot!(p("AND = 8"), @r###"
         Was expecting a value but instead got `AND`, which is a reserved keyword. To use `AND` as a field name or a value, surround it by quotes.
         1:4 AND = 8
         "###);
 
-        insta::assert_display_snapshot!(p("((((((((((((((((((((((((((((((((((((((((((((((((((x = 1))))))))))))))))))))))))))))))))))))))))))))))))))"), @r###"
+        insta::assert_snapshot!(p("((((((((((((((((((((((((((((((((((((((((((((((((((x = 1))))))))))))))))))))))))))))))))))))))))))))))))))"), @r###"
         The filter exceeded the maximum depth limit. Try rewriting the filter so that it contains fewer nested conditions.
         51:106 ((((((((((((((((((((((((((((((((((((((((((((((((((x = 1))))))))))))))))))))))))))))))))))))))))))))))))))
         "###);
 
-        insta::assert_display_snapshot!(
+        insta::assert_snapshot!(
             p("NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT x = 1"),
             @r###"
         The filter exceeded the maximum depth limit. Try rewriting the filter so that it contains fewer nested conditions.
@@ -872,40 +872,40 @@ pub mod tests {
         "###
         );
 
-        insta::assert_display_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###"
+        insta::assert_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###"
         Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes.
         5:7 NOT OR EXISTS AND EXISTS NOT EXISTS
         "###);
 
-        insta::assert_display_snapshot!(p(r#"value NULL"#), @r###"
+        insta::assert_snapshot!(p(r#"value NULL"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value NULL`.
         1:11 value NULL
         "###);
-        insta::assert_display_snapshot!(p(r#"value NOT NULL"#), @r###"
+        insta::assert_snapshot!(p(r#"value NOT NULL"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value NOT NULL`.
         1:15 value NOT NULL
         "###);
-        insta::assert_display_snapshot!(p(r#"value EMPTY"#), @r###"
+        insta::assert_snapshot!(p(r#"value EMPTY"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value EMPTY`.
         1:12 value EMPTY
         "###);
-        insta::assert_display_snapshot!(p(r#"value NOT EMPTY"#), @r###"
+        insta::assert_snapshot!(p(r#"value NOT EMPTY"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value NOT EMPTY`.
         1:16 value NOT EMPTY
         "###);
-        insta::assert_display_snapshot!(p(r#"value IS"#), @r###"
+        insta::assert_snapshot!(p(r#"value IS"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS`.
         1:9 value IS
         "###);
-        insta::assert_display_snapshot!(p(r#"value IS NOT"#), @r###"
+        insta::assert_snapshot!(p(r#"value IS NOT"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT`.
         1:13 value IS NOT
         "###);
-        insta::assert_display_snapshot!(p(r#"value IS EXISTS"#), @r###"
+        insta::assert_snapshot!(p(r#"value IS EXISTS"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS EXISTS`.
         1:16 value IS EXISTS
         "###);
-        insta::assert_display_snapshot!(p(r#"value IS NOT EXISTS"#), @r###"
+        insta::assert_snapshot!(p(r#"value IS NOT EXISTS"#), @r###"
         Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT EXISTS`.
         1:20 value IS NOT EXISTS
         "###);

fuzzers/Cargo.toml

@@ -12,9 +12,9 @@ license.workspace = true
 
 [dependencies]
 arbitrary = { version = "1.3.2", features = ["derive"] }
-clap = { version = "4.4.17", features = ["derive"] }
-fastrand = "2.0.1"
+clap = { version = "4.5.9", features = ["derive"] }
+fastrand = "2.1.0"
 milli = { path = "../milli" }
-serde = { version = "1.0.195", features = ["derive"] }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
-tempfile = "3.9.0"
+serde = { version = "1.0.204", features = ["derive"] }
+serde_json = { version = "1.0.120", features = ["preserve_order"] }
+tempfile = "3.10.1"

fuzzers/src/bin/fuzz-indexing.rs

@@ -110,7 +110,7 @@ fn main() {
 
                             // after executing a batch we check if the database is corrupted
                             let res = index.search(&wtxn).execute().unwrap();
-                            index.documents(&wtxn, res.documents_ids).unwrap();
+                            index.compressed_documents(&wtxn, res.documents_ids).unwrap();
                             progression.fetch_add(1, Ordering::Relaxed);
                         }
                         wtxn.abort();

index-scheduler/Cargo.toml

@@ -11,37 +11,38 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.79"
+anyhow = "1.0.86"
 bincode = "1.3.3"
 csv = "1.3.0"
-derive_builder = "0.12.0"
+derive_builder = "0.20.0"
 dump = { path = "../dump" }
-enum-iterator = "1.5.0"
+enum-iterator = "2.1.0"
 file-store = { path = "../file-store" }
-flate2 = "1.0.28"
+flate2 = "1.0.30"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-page_size = "0.5.0"
-puffin = { version = "0.16.0", features = ["serialization"] }
-rayon = "1.8.1"
-roaring = { version = "0.10.2", features = ["serde"] }
-serde = { version = "1.0.195", features = ["derive"] }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
+page_size = "0.6.0"
+rayon = "1.10.0"
+roaring = { version = "0.10.6", features = ["serde"] }
+serde = { version = "1.0.204", features = ["derive"] }
+serde_json = { version = "1.0.120", features = ["preserve_order"] }
 synchronoise = "1.0.1"
-tempfile = "3.9.0"
-thiserror = "1.0.56"
-time = { version = "0.3.31", features = [
+tempfile = "3.10.1"
+thiserror = "1.0.61"
+time = { version = "0.3.36", features = [
     "serde-well-known",
     "formatting",
     "parsing",
     "macros",
 ] }
 tracing = "0.1.40"
-ureq = "2.9.1"
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
+ureq = "2.10.0"
+uuid = { version = "1.10.0", features = ["serde", "v4"] }
 
 [dev-dependencies]
+arroy = "0.4.0"
 big_s = "1.0.2"
 crossbeam = "0.8.4"
-insta = { version = "1.34.0", features = ["json", "redactions"] }
+insta = { version = "1.39.0", features = ["json", "redactions"] }
+maplit = "1.0.2"
 meili-snap = { path = "../meili-snap" }

index-scheduler/src/batch.rs

@@ -13,7 +13,7 @@ We can combine the two tasks in a single batch:
 1. import documents X and Y
 
 Processing this batch is functionally equivalent to processing the two
-tasks individally, but should be much faster since we are only performing
+tasks individually, but should be much faster since we are only performing
 one indexing operation.
 */
 
@@ -31,6 +31,9 @@ use meilisearch_types::milli::heed::CompactionOption;
 use meilisearch_types::milli::update::{
     IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Settings as MilliSettings,
 };
+use meilisearch_types::milli::vector::parsed_vectors::{
+    ExplicitVectors, VectorOrArrayOfVectors, RESERVED_VECTORS_FIELD_NAME,
+};
 use meilisearch_types::milli::{self, Filter};
 use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked};
 use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status, Task};
@@ -526,8 +529,6 @@ impl IndexScheduler {
         #[cfg(test)]
         self.maybe_fail(crate::tests::FailureLocation::InsideCreateBatch)?;
 
-        puffin::profile_function!();
-
         let enqueued = &self.get_status(rtxn, Status::Enqueued)?;
         let to_cancel = self.get_kind(rtxn, Kind::TaskCancelation)? & enqueued;
 
@@ -636,8 +637,6 @@ impl IndexScheduler {
             self.breakpoint(crate::Breakpoint::InsideProcessBatch);
         }
 
-        puffin::profile_function!(batch.to_string());
-
         match batch {
             Batch::TaskCancelation { mut task, previous_started_at, previous_processing_tasks } => {
                 // 1. Retrieve the tasks that matched the query at enqueue-time.
@@ -785,10 +784,12 @@ impl IndexScheduler {
                 let dst = temp_snapshot_dir.path().join("auth");
                 fs::create_dir_all(&dst)?;
                 // TODO We can't use the open_auth_store_env function here but we should
-                let auth = milli::heed::EnvOpenOptions::new()
-                    .map_size(1024 * 1024 * 1024) // 1 GiB
-                    .max_dbs(2)
-                    .open(&self.auth_path)?;
+                let auth = unsafe {
+                    milli::heed::EnvOpenOptions::new()
+                        .map_size(1024 * 1024 * 1024) // 1 GiB
+                        .max_dbs(2)
+                        .open(&self.auth_path)
+                }?;
                 auth.copy_to_file(dst.join("data.mdb"), CompactionOption::Enabled)?;
 
                 // 5. Copy and tarball the flat snapshot
@@ -907,15 +908,74 @@ impl IndexScheduler {
                     let mut index_dumper = dump.create_index(uid, &metadata)?;
 
                     let fields_ids_map = index.fields_ids_map(&rtxn)?;
+                    let dictionary = index.document_decompression_dictionary(&rtxn)?;
                     let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
+                    let embedding_configs = index.embedding_configs(&rtxn)?;
+                    let mut buffer = Vec::new();
 
                     // 3.1. Dump the documents
-                    for ret in index.all_documents(&rtxn)? {
+                    for ret in index.all_compressed_documents(&rtxn)? {
                         if self.must_stop_processing.get() {
                             return Err(Error::AbortedTask);
                         }
-                        let (_id, doc) = ret?;
-                        let document = milli::obkv_to_json(&all_fields, &fields_ids_map, doc)?;
+
+                        let (id, compressed) = ret?;
+                        let doc = compressed.decompress_with_optional_dictionary(
+                            &mut buffer,
+                            dictionary.as_ref(),
+                        )?;
+
+                        let mut document = milli::obkv_to_json(&all_fields, &fields_ids_map, doc)?;
+
+                        'inject_vectors: {
+                            let embeddings = index.embeddings(&rtxn, id)?;
+
+                            if embeddings.is_empty() {
+                                break 'inject_vectors;
+                            }
+
+                            let vectors = document
+                                .entry(RESERVED_VECTORS_FIELD_NAME.to_owned())
+                                .or_insert(serde_json::Value::Object(Default::default()));
+
+                            let serde_json::Value::Object(vectors) = vectors else {
+                                return Err(milli::Error::UserError(
+                                    milli::UserError::InvalidVectorsMapType {
+                                        document_id: {
+                                            if let Ok(Some(Ok(index))) = index
+                                                .external_id_of(&rtxn, std::iter::once(id))
+                                                .map(|it| it.into_iter().next())
+                                            {
+                                                index
+                                            } else {
+                                                format!("internal docid={id}")
+                                            }
+                                        },
+                                        value: vectors.clone(),
+                                    },
+                                )
+                                .into());
+                            };
+
+                            for (embedder_name, embeddings) in embeddings {
+                                let user_provided = embedding_configs
+                                    .iter()
+                                    .find(|conf| conf.name == embedder_name)
+                                    .is_some_and(|conf| conf.user_provided.contains(id));
+
+                                let embeddings = ExplicitVectors {
+                                    embeddings: Some(
+                                        VectorOrArrayOfVectors::from_array_of_vectors(embeddings),
+                                    ),
+                                    regenerate: !user_provided,
+                                };
+                                vectors.insert(
+                                    embedder_name,
+                                    serde_json::to_value(embeddings).unwrap(),
+                                );
+                            }
+                        }
+
                         index_dumper.push_document(&document)?;
                     }
 
@@ -1174,8 +1234,6 @@ impl IndexScheduler {
         index: &'i Index,
         operation: IndexOperation,
     ) -> Result<Vec<Task>> {
-        puffin::profile_function!();
-
         match operation {
             IndexOperation::DocumentClear { mut tasks, .. } => {
                 let count = milli::update::ClearDocuments::new(index_wtxn, index).execute()?;

index-scheduler/src/features.rs

@@ -68,19 +68,6 @@ impl RoFeatures {
             .into())
         }
     }
-
-    pub fn check_puffin(&self) -> Result<()> {
-        if self.runtime.export_puffin_reports {
-            Ok(())
-        } else {
-            Err(FeatureNotEnabledError {
-                disabled_action: "Outputting Puffin reports to disk",
-                feature: "export puffin reports",
-                issue_link: "https://github.com/meilisearch/product/discussions/693",
-            }
-            .into())
-        }
-    }
 }
 
 impl FeatureData {

index-scheduler/src/insta_snapshot.rs

@@ -32,7 +32,6 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         features: _,
         max_number_of_tasks: _,
         max_number_of_batched_tasks: _,
-        puffin_frame: _,
         wake_up: _,
         dumps_path: _,
         snapshots_path: _,

index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-15.snap

@@ -0,0 +1,15 @@
+---
+source: index-scheduler/src/lib.rs
+expression: doc
+---
+{
+  "doggo": "Intel",
+  "breed": "beagle",
+  "_vectors": {
+    "noise": [
+      0.1,
+      0.2,
+      0.3
+    ]
+  }
+}

index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-2.snap

@@ -0,0 +1,20 @@
+---
+source: index-scheduler/src/lib.rs
+expression: task.details
+---
+{
+  "embedders": {
+    "A_fakerest": {
+      "source": "rest",
+      "apiKey": "MyXXXX...",
+      "dimensions": 384,
+      "url": "http://localhost:7777"
+    },
+    "B_small_hf": {
+      "source": "huggingFace",
+      "model": "sentence-transformers/all-MiniLM-L6-v2",
+      "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+      "documentTemplate": "{{doc.doggo}} the {{doc.breed}} best doggo"
+    }
+  }
+}

index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-22.snap

@@ -0,0 +1,15 @@
+---
+source: index-scheduler/src/lib.rs
+expression: doc
+---
+{
+  "doggo": "kefir",
+  "breed": "patou",
+  "_vectors": {
+    "noise": [
+      0.1,
+      0.2,
+      0.3
+    ]
+  }
+}

index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-5.snap

@@ -0,0 +1,23 @@
+---
+source: index-scheduler/src/lib.rs
+expression: fakerest_config.embedder_options
+---
+{
+  "Rest": {
+    "api_key": "My super secret",
+    "distribution": null,
+    "dimensions": 384,
+    "url": "http://localhost:7777",
+    "query": null,
+    "input_field": [
+      "input"
+    ],
+    "path_to_embeddings": [
+      "data"
+    ],
+    "embedding_object": [
+      "embedding"
+    ],
+    "input_type": "text"
+  }
+}

index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-8.snap

@@ -0,0 +1,11 @@
+---
+source: index-scheduler/src/lib.rs
+expression: simple_hf_config.embedder_options
+---
+{
+  "HuggingFace": {
+    "model": "sentence-transformers/all-MiniLM-L6-v2",
+    "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+    "distribution": null
+  }
+}

index-scheduler/src/snapshots/index_scheduler__tests__import_vectors.snap

@@ -0,0 +1,20 @@
+---
+source: index-scheduler/src/lib.rs
+expression: task.details
+---
+{
+  "embedders": {
+    "A_fakerest": {
+      "source": "rest",
+      "apiKey": "MyXXXX...",
+      "dimensions": 384,
+      "url": "http://localhost:7777"
+    },
+    "B_small_hf": {
+      "source": "huggingFace",
+      "model": "sentence-transformers/all-MiniLM-L6-v2",
+      "revision": "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+      "documentTemplate": "{{doc.doggo}} the {{doc.breed}} best doggo"
+    }
+  }
+}

index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir succeeds.snap

@@ -0,0 +1,49 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
+2 {uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: None, method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued []
+succeeded [0,1,2,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,2,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,2,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 1, field_distribution: {"_vectors": 1, "breed": 1, "doggo": 1, "id": 1} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+----------------------------------------------------------------------
+### File Store:
+
+----------------------------------------------------------------------
+

index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir.snap

@@ -0,0 +1,48 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
+2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: None, method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued [2,]
+succeeded [0,1,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,2,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,2,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 1, field_distribution: {"_vectors": 1, "breed": 1, "doggo": 1, "id": 1} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+[timestamp] [2,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### File Store:
+00000000-0000-0000-0000-000000000001
+
+----------------------------------------------------------------------
+

index-scheduler/src/snapshots/lib.rs/import_vectors/adding Intel succeeds.snap

@@ -0,0 +1,45 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued []
+succeeded [0,1,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 1, field_distribution: {"_vectors": 1, "breed": 1, "doggo": 1, "id": 1} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### File Store:
+
+----------------------------------------------------------------------
+

index-scheduler/src/snapshots/lib.rs/import_vectors/after adding Intel.snap

@@ -0,0 +1,44 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued [1,]
+succeeded [0,]
+----------------------------------------------------------------------
+### Kind:
+"documentAdditionOrUpdate" [1,]
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,1,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 0, field_distribution: {} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+[timestamp] [1,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### File Store:
+00000000-0000-0000-0000-000000000000
+
+----------------------------------------------------------------------
+

index-scheduler/src/snapshots/lib.rs/import_vectors/after_registering_settings_task_vectors.snap

@@ -0,0 +1,36 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued [0,]
+----------------------------------------------------------------------
+### Kind:
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,]
+----------------------------------------------------------------------
+### Index Mapper:
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Started At:
+----------------------------------------------------------------------
+### Finished At:
+----------------------------------------------------------------------
+### File Store:
+
+----------------------------------------------------------------------
+

index-scheduler/src/snapshots/lib.rs/import_vectors/settings_update_processed_vectors.snap

@@ -0,0 +1,40 @@
+---
+source: index-scheduler/src/lib.rs
+---
+### Autobatching Enabled = true
+### Processing Tasks:
+[]
+----------------------------------------------------------------------
+### All Tasks:
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+----------------------------------------------------------------------
+### Status:
+enqueued []
+succeeded [0,]
+----------------------------------------------------------------------
+### Kind:
+"settingsUpdate" [0,]
+----------------------------------------------------------------------
+### Index Tasks:
+doggos [0,]
+----------------------------------------------------------------------
+### Index Mapper:
+doggos: { number_of_documents: 0, field_distribution: {} }
+
+----------------------------------------------------------------------
+### Canceled By:
+
+----------------------------------------------------------------------
+### Enqueued At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Started At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### Finished At:
+[timestamp] [0,]
+----------------------------------------------------------------------
+### File Store:
+
+----------------------------------------------------------------------
+

index-scheduler/src/utils.rs

@@ -272,9 +272,9 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) {
     }
     for index_uid in index_uids {
         if index_uid == swap.0 {
-            *index_uid = swap.1.to_owned();
+            swap.1.clone_into(index_uid);
         } else if index_uid == swap.1 {
-            *index_uid = swap.0.to_owned();
+            swap.0.clone_into(index_uid);
         }
     }
 }

meili-snap/Cargo.toml

@@ -11,6 +11,6 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-insta = { version = "^1.34.0", features = ["json", "redactions"] }
+insta = { version = "^1.39.0", features = ["json", "redactions"] }
 md5 = "0.7.0"
 once_cell = "1.19"

meilisearch-auth/Cargo.toml

@@ -11,16 +11,16 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-base64 = "0.21.7"
-enum-iterator = "1.5.0"
+base64 = "0.22.1"
+enum-iterator = "2.1.0"
 hmac = "0.12.1"
 maplit = "1.0.2"
 meilisearch-types = { path = "../meilisearch-types" }
 rand = "0.8.5"
-roaring = { version = "0.10.2", features = ["serde"] }
-serde = { version = "1.0.195", features = ["derive"] }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
+roaring = { version = "0.10.6", features = ["serde"] }
+serde = { version = "1.0.204", features = ["derive"] }
+serde_json = { version = "1.0.120", features = ["preserve_order"] }
 sha2 = "0.10.8"
-thiserror = "1.0.56"
-time = { version = "0.3.31", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
+thiserror = "1.0.61"
+time = { version = "0.3.36", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+uuid = { version = "1.10.0", features = ["serde", "v4"] }

meilisearch-auth/src/lib.rs

@@ -188,6 +188,12 @@ impl AuthFilter {
         self.allow_index_creation && self.is_index_authorized(index)
     }
 
+    #[inline]
+    /// Return true if a tenant token was used to generate the search rules.
+    pub fn is_tenant_token(&self) -> bool {
+        self.search_rules.is_some()
+    }
+
     pub fn with_allowed_indexes(allowed_indexes: HashSet<IndexUidPattern>) -> Self {
         Self {
             search_rules: None,
@@ -205,6 +211,7 @@ impl AuthFilter {
                 .unwrap_or(true)
     }
 
+    /// Check if the index is authorized by the API key and the tenant token.
     pub fn is_index_authorized(&self, index: &str) -> bool {
         self.key_authorized_indexes.is_index_authorized(index)
             && self
@@ -214,6 +221,44 @@ impl AuthFilter {
                 .unwrap_or(true)
     }
 
+    /// Only check if the index is authorized by the API key
+    pub fn api_key_is_index_authorized(&self, index: &str) -> bool {
+        self.key_authorized_indexes.is_index_authorized(index)
+    }
+
+    /// Only check if the index is authorized by the tenant token
+    pub fn tenant_token_is_index_authorized(&self, index: &str) -> bool {
+        self.search_rules
+            .as_ref()
+            .map(|search_rules| search_rules.is_index_authorized(index))
+            .unwrap_or(true)
+    }
+
+    /// Return the list of authorized indexes by the tenant token if any
+    pub fn tenant_token_list_index_authorized(&self) -> Vec<String> {
+        match self.search_rules {
+            Some(ref search_rules) => {
+                let mut indexes: Vec<_> = match search_rules {
+                    SearchRules::Set(set) => set.iter().map(|s| s.to_string()).collect(),
+                    SearchRules::Map(map) => map.keys().map(|s| s.to_string()).collect(),
+                };
+                indexes.sort_unstable();
+                indexes
+            }
+            None => Vec::new(),
+        }
+    }
+
+    /// Return the list of authorized indexes by the api key if any
+    pub fn api_key_list_index_authorized(&self) -> Vec<String> {
+        let mut indexes: Vec<_> = match self.key_authorized_indexes {
+            SearchRules::Set(ref set) => set.iter().map(|s| s.to_string()).collect(),
+            SearchRules::Map(ref map) => map.keys().map(|s| s.to_string()).collect(),
+        };
+        indexes.sort_unstable();
+        indexes
+    }
+
     pub fn get_index_search_rules(&self, index: &str) -> Option<IndexSearchRules> {
         if !self.is_index_authorized(index) {
             return None;

meilisearch-auth/src/store.rs

@@ -49,7 +49,7 @@ pub fn open_auth_store_env(path: &Path) -> milli::heed::Result<milli::heed::Env>
     let mut options = EnvOpenOptions::new();
     options.map_size(AUTH_STORE_SIZE); // 1GB
     options.max_dbs(2);
-    options.open(path)
+    unsafe { options.open(path) }
 }
 
 impl HeedAuthStore {

meilisearch-types/Cargo.toml

@@ -11,31 +11,36 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-actix-web = { version = "4.5.1", default-features = false }
-anyhow = "1.0.79"
+actix-web = { version = "4.8.0", default-features = false }
+anyhow = "1.0.86"
 convert_case = "0.6.0"
 csv = "1.3.0"
-deserr = { version = "0.6.1", features = ["actix-web"] }
-either = { version = "1.9.0", features = ["serde"] }
-enum-iterator = "1.5.0"
+deserr = { version = "0.6.2", features = ["actix-web"] }
+either = { version = "1.13.0", features = ["serde"] }
+enum-iterator = "2.1.0"
 file-store = { path = "../file-store" }
-flate2 = "1.0.28"
+flate2 = "1.0.30"
 fst = "0.4.7"
-memmap2 = "0.7.1"
+memmap2 = "0.9.4"
 milli = { path = "../milli" }
-roaring = { version = "0.10.2", features = ["serde"] }
-serde = { version = "1.0.195", features = ["derive"] }
+roaring = { version = "0.10.6", features = ["serde"] }
+serde = { version = "1.0.204", features = ["derive"] }
 serde-cs = "0.2.4"
-serde_json = "1.0.111"
-tar = "0.4.40"
-tempfile = "3.9.0"
-thiserror = "1.0.56"
-time = { version = "0.3.31", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-tokio = "1.35"
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
+serde_json = "1.0.120"
+tar = "0.4.41"
+tempfile = "3.10.1"
+thiserror = "1.0.61"
+time = { version = "0.3.36", features = [
+    "serde-well-known",
+    "formatting",
+    "parsing",
+    "macros",
+] }
+tokio = "1.38"
+uuid = { version = "1.10.0", features = ["serde", "v4"] }
 
 [dev-dependencies]
-insta = "1.34.0"
+insta = "1.39.0"
 meili-snap = { path = "../meili-snap" }
 
 [features]
@@ -49,6 +54,8 @@ chinese-pinyin = ["milli/chinese-pinyin"]
 hebrew = ["milli/hebrew"]
 # japanese specialized tokenization
 japanese = ["milli/japanese"]
+# korean specialized tokenization
+korean = ["milli/korean"]
 # thai specialized tokenization
 thai = ["milli/thai"]
 # allow greek specialized tokenization

meilisearch-types/src/deserr/mod.rs

@@ -26,7 +26,7 @@ pub type DeserrQueryParamError<C = BadRequest> = DeserrError<DeserrQueryParam, C
 
 /// A request deserialization error.
 ///
-/// The first generic paramater is a marker type describing the format of the request: either json (e.g. [`DeserrJson`] or [`DeserrQueryParam`]).
+/// The first generic parameter is a marker type describing the format of the request: either json (e.g. [`DeserrJson`] or [`DeserrQueryParam`]).
 /// The second generic parameter is the default error code for the deserialization error, in case it is not given.
 pub struct DeserrError<Format, C: Default + ErrorCode> {
     pub msg: String,
@@ -189,3 +189,6 @@ merge_with_error_impl_take_error_message!(ParseTaskKindError);
 merge_with_error_impl_take_error_message!(ParseTaskStatusError);
 merge_with_error_impl_take_error_message!(IndexUidFormatError);
 merge_with_error_impl_take_error_message!(InvalidSearchSemanticRatio);
+merge_with_error_impl_take_error_message!(InvalidSearchRankingScoreThreshold);
+merge_with_error_impl_take_error_message!(InvalidSimilarRankingScoreThreshold);
+merge_with_error_impl_take_error_message!(InvalidSimilarId);

meilisearch-types/src/error.rs

@@ -222,6 +222,7 @@ InvalidApiKeyUid                      , InvalidRequest       , BAD_REQUEST ;
 InvalidContentType                    , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
 InvalidDocumentCsvDelimiter           , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentFields                 , InvalidRequest       , BAD_REQUEST ;
+InvalidDocumentRetrieveVectors        , InvalidRequest       , BAD_REQUEST ;
 MissingDocumentFilter                 , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentFilter                 , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentGeoField               , InvalidRequest       , BAD_REQUEST ;
@@ -239,18 +240,27 @@ InvalidIndexUid                       , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToSearchOn     , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToCrop         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToHighlight    , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarAttributesToRetrieve    , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarRetrieveVectors         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToRetrieve     , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchRankingScoreThreshold    , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarRankingScoreThreshold   , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchRetrieveVectors          , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchCropLength               , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchCropMarker               , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchFacets                   , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchSemanticRatio            , InvalidRequest       , BAD_REQUEST ;
 InvalidFacetSearchFacetName           , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarId                      , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchFilter                   , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarFilter                  , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchHighlightPostTag         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchHighlightPreTag          , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchHitsPerPage              , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarLimit                   , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchLimit                    , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchMatchingStrategy         , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarOffset                  , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchOffset                   , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchPage                     , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchQ                        , InvalidRequest       , BAD_REQUEST ;
@@ -259,15 +269,18 @@ InvalidFacetSearchName                , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchVector                   , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchShowMatchesPosition      , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchShowRankingScore         , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarShowRankingScore        , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchShowRankingScoreDetails  , InvalidRequest       , BAD_REQUEST ;
+InvalidSimilarShowRankingScoreDetails , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchSort                     , InvalidRequest       , BAD_REQUEST ;
+InvalidSearchDistinct                 , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsDisplayedAttributes    , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsDistinctAttribute      , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsProximityPrecision     , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsFaceting               , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsFilterableAttributes   , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsPagination             , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsSearchCutoffMs           , InvalidRequest       , BAD_REQUEST ;
+InvalidSettingsSearchCutoffMs         , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsEmbedders              , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsRankingRules           , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSearchableAttributes   , InvalidRequest       , BAD_REQUEST ;
@@ -322,7 +335,8 @@ UnretrievableErrorCode                , InvalidRequest       , BAD_REQUEST ;
 UnsupportedMediaType                  , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
 
 // Experimental features
-VectorEmbeddingError                  , InvalidRequest       , BAD_REQUEST
+VectorEmbeddingError                  , InvalidRequest       , BAD_REQUEST ;
+NotFoundSimilarId                     , InvalidRequest       , BAD_REQUEST
 }
 
 impl ErrorCode for JoinError {
@@ -371,6 +385,7 @@ impl ErrorCode for milli::Error {
                         Code::IndexPrimaryKeyMultipleCandidatesFound
                     }
                     UserError::PrimaryKeyCannotBeChanged(_) => Code::IndexPrimaryKeyAlreadyExists,
+                    UserError::InvalidDistinctAttribute { .. } => Code::InvalidSearchDistinct,
                     UserError::SortRankingRuleMissing => Code::InvalidSearchSort,
                     UserError::InvalidFacetsDistribution { .. } => Code::InvalidSearchFacets,
                     UserError::InvalidSortableAttribute { .. } => Code::InvalidSearchSort,
@@ -383,8 +398,8 @@ impl ErrorCode for milli::Error {
                     UserError::CriterionError(_) => Code::InvalidSettingsRankingRules,
                     UserError::InvalidGeoField { .. } => Code::InvalidDocumentGeoField,
                     UserError::InvalidVectorDimensions { .. } => Code::InvalidVectorDimensions,
-                    UserError::InvalidVectorsMapType { .. } => Code::InvalidVectorsType,
-                    UserError::InvalidVectorsType { .. } => Code::InvalidVectorsType,
+                    UserError::InvalidVectorsMapType { .. }
+                    | UserError::InvalidVectorsEmbedderConf { .. } => Code::InvalidVectorsType,
                     UserError::TooManyVectors(_, _) => Code::TooManyVectors,
                     UserError::SortError(_) => Code::InvalidSearchSort,
                     UserError::InvalidMinTypoWordLenSetting(_, _) => {
@@ -423,7 +438,6 @@ impl ErrorCode for HeedError {
             HeedError::Mdb(_)
             | HeedError::Encoding(_)
             | HeedError::Decoding(_)
-            | HeedError::InvalidDatabaseTyping
             | HeedError::DatabaseClosing
             | HeedError::BadOpenOptions { .. } => Code::Internal,
         }
@@ -488,6 +502,32 @@ impl fmt::Display for deserr_codes::InvalidSearchSemanticRatio {
     }
 }
 
+impl fmt::Display for deserr_codes::InvalidSimilarId {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "the value of `id` is invalid. \
+            A document identifier can be of type integer or string, \
+            only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_)."
+        )
+    }
+}
+
+impl fmt::Display for deserr_codes::InvalidSearchRankingScoreThreshold {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "the value of `rankingScoreThreshold` is invalid, expected a float between `0.0` and `1.0`."
+        )
+    }
+}
+
+impl fmt::Display for deserr_codes::InvalidSimilarRankingScoreThreshold {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        deserr_codes::InvalidSearchRankingScoreThreshold.fmt(f)
+    }
+}
+
 #[macro_export]
 macro_rules! internal_error {
     ($target:ty : $($other:path), *) => {

meilisearch-types/src/features.rs

@@ -6,7 +6,6 @@ pub struct RuntimeTogglableFeatures {
     pub vector_store: bool,
     pub metrics: bool,
     pub logs_route: bool,
-    pub export_puffin_reports: bool,
 }
 
 #[derive(Default, Debug, Clone, Copy)]

meilisearch-types/src/lib.rs

@@ -12,7 +12,7 @@ pub mod star_or;
 pub mod task_view;
 pub mod tasks;
 pub mod versioning;
-pub use milli::{heed, Index};
+pub use milli::{heed, zstd, Index};
 use uuid::Uuid;
 pub use versioning::VERSION_FILE_NAME;
 pub use {milli, serde_cs};

meilisearch-types/src/settings.rs

@@ -8,6 +8,7 @@ use std::str::FromStr;
 
 use deserr::{DeserializeError, Deserr, ErrorKind, MergeWithError, ValuePointerRef};
 use fst::IntoStreamer;
+use milli::index::IndexEmbeddingConfig;
 use milli::proximity::ProximityPrecision;
 use milli::update::Setting;
 use milli::{Criterion, CriterionError, Index, DEFAULT_VALUES_PER_FACET};
@@ -672,7 +673,7 @@ pub fn settings(
     let embedders: BTreeMap<_, _> = index
         .embedding_configs(rtxn)?
         .into_iter()
-        .map(|(name, config)| (name, Setting::Set(config.into())))
+        .map(|IndexEmbeddingConfig { name, config, .. }| (name, Setting::Set(config.into())))
         .collect();
     let embedders = if embedders.is_empty() { Setting::NotSet } else { Setting::Set(embedders) };
 

meilisearch/Cargo.toml

@@ -14,131 +14,123 @@ default-run = "meilisearch"
 
 [dependencies]
 actix-cors = "0.7.0"
-actix-http = { version = "3.6.0", default-features = false, features = [
+actix-http = { version = "3.8.0", default-features = false, features = [
     "compress-brotli",
     "compress-gzip",
     "rustls-0_21",
 ] }
 actix-utils = "3.0.1"
-actix-web = { version = "4.5.1", default-features = false, features = [
+actix-web = { version = "4.8.0", default-features = false, features = [
     "macros",
     "compress-brotli",
     "compress-gzip",
     "cookies",
     "rustls-0_21",
 ] }
-actix-web-static-files = { git = "https://github.com/kilork/actix-web-static-files.git", rev = "2d3b6160", optional = true }
-anyhow = { version = "1.0.79", features = ["backtrace"] }
-async-stream = "0.3.5"
-async-trait = "0.1.77"
-bstr = "1.9.0"
-byte-unit = { version = "4.0.19", default-features = false, features = [
+anyhow = { version = "1.0.86", features = ["backtrace"] }
+async-trait = "0.1.81"
+bstr = "1.9.1"
+byte-unit = { version = "5.1.4", default-features = false, features = [
     "std",
+    "byte",
     "serde",
 ] }
-bytes = "1.5.0"
-clap = { version = "4.4.17", features = ["derive", "env"] }
-crossbeam-channel = "0.5.11"
-deserr = { version = "0.6.1", features = ["actix-web"] }
+bytes = "1.6.0"
+clap = { version = "4.5.9", features = ["derive", "env"] }
+crossbeam-channel = "0.5.13"
+deserr = { version = "0.6.2", features = ["actix-web"] }
 dump = { path = "../dump" }
-either = "1.9.0"
+either = "1.13.0"
 file-store = { path = "../file-store" }
-flate2 = "1.0.28"
+flate2 = "1.0.30"
 fst = "0.4.7"
 futures = "0.3.30"
 futures-util = "0.3.30"
-http = "0.2.11"
 index-scheduler = { path = "../index-scheduler" }
-indexmap = { version = "2.1.0", features = ["serde"] }
-is-terminal = "0.4.10"
-itertools = "0.11.0"
-jsonwebtoken = "9.2.0"
-lazy_static = "1.4.0"
+indexmap = { version = "2.2.6", features = ["serde"] }
+is-terminal = "0.4.12"
+itertools = "0.13.0"
+jsonwebtoken = "9.3.0"
+lazy_static = "1.5.0"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-mimalloc = { version = "0.1.39", default-features = false }
+mimalloc = { version = "0.1.43", default-features = false }
 mime = "0.3.17"
 num_cpus = "1.16.0"
-obkv = "0.2.1"
+obkv = "0.2.2"
 once_cell = "1.19.0"
-ordered-float = "4.2.0"
-parking_lot = "0.12.1"
+ordered-float = "4.2.1"
+parking_lot = "0.12.3"
 permissive-json-pointer = { path = "../permissive-json-pointer" }
-pin-project-lite = "0.2.13"
+pin-project-lite = "0.2.14"
 platform-dirs = "0.3.0"
-prometheus = { version = "0.13.3", features = ["process"] }
-puffin = { version = "0.16.0", features = ["serialization"] }
+prometheus = { version = "0.13.4", features = ["process"] }
 rand = "0.8.5"
-rayon = "1.8.0"
-regex = "1.10.2"
-reqwest = { version = "0.11.23", features = [
+rayon = "1.10.0"
+regex = "1.10.5"
+reqwest = { version = "0.12.5", features = [
     "rustls-tls",
     "json",
 ], default-features = false }
-rustls = "0.21.6"
-rustls-pemfile = "1.0.2"
-segment = { version = "0.2.3", optional = true }
-serde = { version = "1.0.195", features = ["derive"] }
-serde_json = { version = "1.0.111", features = ["preserve_order"] }
+rustls = "0.21.12"
+rustls-pemfile = "1.0.4"
+segment = { version = "0.2.4", optional = true }
+serde = { version = "1.0.204", features = ["derive"] }
+serde_json = { version = "1.0.120", features = ["preserve_order"] }
 sha2 = "0.10.8"
-siphasher = "1.0.0"
+siphasher = "1.0.1"
 slice-group-by = "0.3.1"
-static-files = { version = "0.2.3", optional = true }
-sysinfo = "0.30.5"
-tar = "0.4.40"
-tempfile = "3.9.0"
-thiserror = "1.0.56"
-time = { version = "0.3.31", features = [
+static-files = { version = "0.2.4", optional = true }
+sysinfo = "0.30.13"
+tar = "0.4.41"
+tempfile = "3.10.1"
+thiserror = "1.0.61"
+time = { version = "0.3.36", features = [
     "serde-well-known",
     "formatting",
     "parsing",
     "macros",
 ] }
-tokio = { version = "1.35.1", features = ["full"] }
-tokio-stream = "0.1.14"
-toml = "0.8.8"
-uuid = { version = "1.6.1", features = ["serde", "v4"] }
-walkdir = "2.4.0"
-yaup = "0.2.1"
+tokio = { version = "1.38.0", features = ["full"] }
+toml = "0.8.14"
+uuid = { version = "1.10.0", features = ["serde", "v4"] }
 serde_urlencoded = "0.7.1"
 termcolor = "1.4.1"
-url = { version = "2.5.0", features = ["serde"] }
+url = { version = "2.5.2", features = ["serde"] }
 tracing = "0.1.40"
 tracing-subscriber = { version = "0.3.18", features = ["json"] }
 tracing-trace = { version = "0.1.0", path = "../tracing-trace" }
-tracing-actix-web = "0.7.9"
+tracing-actix-web = "0.7.11"
 build-info = { version = "1.7.0", path = "../build-info" }
 
 [dev-dependencies]
-actix-rt = "2.9.0"
-assert-json-diff = "2.0.2"
-brotli = "3.4.0"
-insta = "1.34.0"
+actix-rt = "2.10.0"
+brotli = "6.0.0"
+insta = "1.39.0"
 manifest-dir-macros = "0.1.18"
 maplit = "1.0.2"
 meili-snap = { path = "../meili-snap" }
 temp-env = "0.3.6"
 urlencoding = "2.1.3"
-yaup = "0.2.1"
+yaup = "0.3.1"
 
 [build-dependencies]
-anyhow = { version = "1.0.79", optional = true }
-cargo_toml = { version = "0.18.0", optional = true }
+anyhow = { version = "1.0.86", optional = true }
+cargo_toml = { version = "0.20.3", optional = true }
 hex = { version = "0.4.3", optional = true }
-reqwest = { version = "0.11.23", features = [
+reqwest = { version = "0.12.5", features = [
     "blocking",
     "rustls-tls",
 ], default-features = false, optional = true }
 sha-1 = { version = "0.10.1", optional = true }
-static-files = { version = "0.2.3", optional = true }
-tempfile = { version = "3.9.0", optional = true }
-zip = { version = "0.6.6", optional = true }
+static-files = { version = "0.2.4", optional = true }
+tempfile = { version = "3.10.1", optional = true }
+zip = { version = "2.1.3", default-features = false, features = ["deflate"], optional = true }
 
 [features]
 default = ["analytics", "meilisearch-types/all-tokenizations", "mini-dashboard"]
 analytics = ["segment"]
 mini-dashboard = [
-    "actix-web-static-files",
     "static-files",
     "anyhow",
     "cargo_toml",
@@ -152,6 +144,7 @@ chinese = ["meilisearch-types/chinese"]
 chinese-pinyin = ["meilisearch-types/chinese-pinyin"]
 hebrew = ["meilisearch-types/hebrew"]
 japanese = ["meilisearch-types/japanese"]
+korean = ["meilisearch-types/korean"]
 thai = ["meilisearch-types/thai"]
 greek = ["meilisearch-types/greek"]
 khmer = ["meilisearch-types/khmer"]
@@ -159,5 +152,5 @@ vietnamese = ["meilisearch-types/vietnamese"]
 swedish-recomposition = ["meilisearch-types/swedish-recomposition"]
 
 [package.metadata.mini-dashboard]
-assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.13/build.zip"
-sha1 = "e20cc9b390003c6c844f4b8bcc5c5013191a77ff"
+assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.14/build.zip"
+sha1 = "592d1b5a3459d621d0aae1dded8fe3154f5c38fe"

meilisearch/src/analytics/mock_analytics.rs

@@ -25,6 +25,18 @@ impl SearchAggregator {
     pub fn succeed(&mut self, _: &dyn Any) {}
 }
 
+#[derive(Default)]
+pub struct SimilarAggregator;
+
+#[allow(dead_code)]
+impl SimilarAggregator {
+    pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self {
+        Self
+    }
+
+    pub fn succeed(&mut self, _: &dyn Any) {}
+}
+
 #[derive(Default)]
 pub struct MultiSearchAggregator;
 
@@ -66,6 +78,8 @@ impl Analytics for MockAnalytics {
     fn publish(&self, _event_name: String, _send: Value, _request: Option<&HttpRequest>) {}
     fn get_search(&self, _aggregate: super::SearchAggregator) {}
     fn post_search(&self, _aggregate: super::SearchAggregator) {}
+    fn get_similar(&self, _aggregate: super::SimilarAggregator) {}
+    fn post_similar(&self, _aggregate: super::SimilarAggregator) {}
     fn post_multi_search(&self, _aggregate: super::MultiSearchAggregator) {}
     fn post_facet_search(&self, _aggregate: super::FacetSearchAggregator) {}
     fn add_documents(

meilisearch/src/analytics/mod.rs

@@ -22,6 +22,8 @@ pub type SegmentAnalytics = mock_analytics::MockAnalytics;
 #[cfg(not(feature = "analytics"))]
 pub type SearchAggregator = mock_analytics::SearchAggregator;
 #[cfg(not(feature = "analytics"))]
+pub type SimilarAggregator = mock_analytics::SimilarAggregator;
+#[cfg(not(feature = "analytics"))]
 pub type MultiSearchAggregator = mock_analytics::MultiSearchAggregator;
 #[cfg(not(feature = "analytics"))]
 pub type FacetSearchAggregator = mock_analytics::FacetSearchAggregator;
@@ -32,6 +34,8 @@ pub type SegmentAnalytics = segment_analytics::SegmentAnalytics;
 #[cfg(feature = "analytics")]
 pub type SearchAggregator = segment_analytics::SearchAggregator;
 #[cfg(feature = "analytics")]
+pub type SimilarAggregator = segment_analytics::SimilarAggregator;
+#[cfg(feature = "analytics")]
 pub type MultiSearchAggregator = segment_analytics::MultiSearchAggregator;
 #[cfg(feature = "analytics")]
 pub type FacetSearchAggregator = segment_analytics::FacetSearchAggregator;
@@ -70,8 +74,8 @@ pub enum DocumentDeletionKind {
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
 pub enum DocumentFetchKind {
-    PerDocumentId,
-    Normal { with_filter: bool, limit: usize, offset: usize },
+    PerDocumentId { retrieve_vectors: bool },
+    Normal { with_filter: bool, limit: usize, offset: usize, retrieve_vectors: bool },
 }
 
 pub trait Analytics: Sync + Send {
@@ -86,6 +90,12 @@ pub trait Analytics: Sync + Send {
     /// This method should be called to aggregate a post search
     fn post_search(&self, aggregate: SearchAggregator);
 
+    /// This method should be called to aggregate a get similar request
+    fn get_similar(&self, aggregate: SimilarAggregator);
+
+    /// This method should be called to aggregate a post similar request
+    fn post_similar(&self, aggregate: SimilarAggregator);
+
     /// This method should be called to aggregate a post array of searches
     fn post_multi_search(&self, aggregate: MultiSearchAggregator);
 

meilisearch/src/analytics/segment_analytics.rs

@@ -5,10 +5,9 @@ use std::path::{Path, PathBuf};
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 
-use actix_web::http::header::USER_AGENT;
+use actix_web::http::header::{CONTENT_TYPE, USER_AGENT};
 use actix_web::HttpRequest;
 use byte_unit::Byte;
-use http::header::CONTENT_TYPE;
 use index_scheduler::IndexScheduler;
 use meilisearch_auth::{AuthController, AuthFilter};
 use meilisearch_types::InstanceUid;
@@ -36,8 +35,9 @@ use crate::routes::indexes::facet_search::FacetSearchQuery;
 use crate::routes::{create_all_stats, Stats};
 use crate::search::{
     FacetSearchResult, MatchingStrategy, SearchQuery, SearchQueryWithIndex, SearchResult,
-    DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
-    DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, DEFAULT_SEMANTIC_RATIO,
+    SimilarQuery, SimilarResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
+    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
+    DEFAULT_SEMANTIC_RATIO,
 };
 use crate::Opt;
 
@@ -73,6 +73,8 @@ pub enum AnalyticsMsg {
     BatchMessage(Track),
     AggregateGetSearch(SearchAggregator),
     AggregatePostSearch(SearchAggregator),
+    AggregateGetSimilar(SimilarAggregator),
+    AggregatePostSimilar(SimilarAggregator),
     AggregatePostMultiSearch(MultiSearchAggregator),
     AggregatePostFacetSearch(FacetSearchAggregator),
     AggregateAddDocuments(DocumentsAggregator),
@@ -149,6 +151,8 @@ impl SegmentAnalytics {
             update_documents_aggregator: DocumentsAggregator::default(),
             get_fetch_documents_aggregator: DocumentsFetchAggregator::default(),
             post_fetch_documents_aggregator: DocumentsFetchAggregator::default(),
+            get_similar_aggregator: SimilarAggregator::default(),
+            post_similar_aggregator: SimilarAggregator::default(),
         });
         tokio::spawn(segment.run(index_scheduler.clone(), auth_controller.clone()));
 
@@ -184,6 +188,14 @@ impl super::Analytics for SegmentAnalytics {
         let _ = self.sender.try_send(AnalyticsMsg::AggregatePostSearch(aggregate));
     }
 
+    fn get_similar(&self, aggregate: SimilarAggregator) {
+        let _ = self.sender.try_send(AnalyticsMsg::AggregateGetSimilar(aggregate));
+    }
+
+    fn post_similar(&self, aggregate: SimilarAggregator) {
+        let _ = self.sender.try_send(AnalyticsMsg::AggregatePostSimilar(aggregate));
+    }
+
     fn post_facet_search(&self, aggregate: FacetSearchAggregator) {
         let _ = self.sender.try_send(AnalyticsMsg::AggregatePostFacetSearch(aggregate));
     }
@@ -379,6 +391,8 @@ pub struct Segment {
     update_documents_aggregator: DocumentsAggregator,
     get_fetch_documents_aggregator: DocumentsFetchAggregator,
     post_fetch_documents_aggregator: DocumentsFetchAggregator,
+    get_similar_aggregator: SimilarAggregator,
+    post_similar_aggregator: SimilarAggregator,
 }
 
 impl Segment {
@@ -441,6 +455,8 @@ impl Segment {
                         Some(AnalyticsMsg::AggregateUpdateDocuments(agreg)) => self.update_documents_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregateGetFetchDocuments(agreg)) => self.get_fetch_documents_aggregator.aggregate(agreg),
                         Some(AnalyticsMsg::AggregatePostFetchDocuments(agreg)) => self.post_fetch_documents_aggregator.aggregate(agreg),
+                        Some(AnalyticsMsg::AggregateGetSimilar(agreg)) => self.get_similar_aggregator.aggregate(agreg),
+                        Some(AnalyticsMsg::AggregatePostSimilar(agreg)) => self.post_similar_aggregator.aggregate(agreg),
                         None => (),
                     }
                 }
@@ -494,6 +510,8 @@ impl Segment {
             update_documents_aggregator,
             get_fetch_documents_aggregator,
             post_fetch_documents_aggregator,
+            get_similar_aggregator,
+            post_similar_aggregator,
         } = self;
 
         if let Some(get_search) =
@@ -541,6 +559,18 @@ impl Segment {
         {
             let _ = self.batcher.push(post_fetch_documents).await;
         }
+
+        if let Some(get_similar_documents) =
+            take(get_similar_aggregator).into_event(user, "Similar GET")
+        {
+            let _ = self.batcher.push(get_similar_documents).await;
+        }
+
+        if let Some(post_similar_documents) =
+            take(post_similar_aggregator).into_event(user, "Similar POST")
+        {
+            let _ = self.batcher.push(post_similar_documents).await;
+        }
         let _ = self.batcher.flush().await;
     }
 }
@@ -566,6 +596,9 @@ pub struct SearchAggregator {
     // every time a request has a filter, this field must be incremented by one
     sort_total_number_of_criteria: usize,
 
+    // distinct
+    distinct: bool,
+
     // filter
     filter_with_geo_radius: bool,
     filter_with_geo_bounding_box: bool,
@@ -591,6 +624,7 @@ pub struct SearchAggregator {
     // Whether a non-default embedder was specified
     embedder: bool,
     hybrid: bool,
+    retrieve_vectors: bool,
 
     // every time a search is done, we increment the counter linked to the used settings
     matching_strategy: HashMap<String, usize>,
@@ -617,6 +651,7 @@ pub struct SearchAggregator {
     // scoring
     show_ranking_score: bool,
     show_ranking_score_details: bool,
+    ranking_score_threshold: bool,
 }
 
 impl SearchAggregator {
@@ -630,6 +665,7 @@ impl SearchAggregator {
             page,
             hits_per_page,
             attributes_to_retrieve: _,
+            retrieve_vectors,
             attributes_to_crop: _,
             crop_length,
             attributes_to_highlight: _,
@@ -638,6 +674,7 @@ impl SearchAggregator {
             show_ranking_score_details,
             filter,
             sort,
+            distinct,
             facets: _,
             highlight_pre_tag,
             highlight_post_tag,
@@ -645,6 +682,7 @@ impl SearchAggregator {
             matching_strategy,
             attributes_to_search_on,
             hybrid,
+            ranking_score_threshold,
         } = query;
 
         let mut ret = Self::default();
@@ -659,6 +697,8 @@ impl SearchAggregator {
             ret.sort_sum_of_criteria_terms = sort.len();
         }
 
+        ret.distinct = distinct.is_some();
+
         if let Some(ref filter) = filter {
             static RE: Lazy<Regex> = Lazy::new(|| Regex::new("AND | OR").unwrap());
             ret.filter_total_number_of_criteria = 1;
@@ -695,6 +735,7 @@ impl SearchAggregator {
         if let Some(ref vector) = vector {
             ret.max_vector_size = vector.len();
         }
+        ret.retrieve_vectors |= retrieve_vectors;
 
         if query.is_finite_pagination() {
             let limit = hits_per_page.unwrap_or_else(DEFAULT_SEARCH_LIMIT);
@@ -717,6 +758,7 @@ impl SearchAggregator {
 
         ret.show_ranking_score = *show_ranking_score;
         ret.show_ranking_score_details = *show_ranking_score_details;
+        ret.ranking_score_threshold = ranking_score_threshold.is_some();
 
         if let Some(hybrid) = hybrid {
             ret.semantic_ratio = hybrid.semantic_ratio != DEFAULT_SEMANTIC_RATIO();
@@ -761,6 +803,7 @@ impl SearchAggregator {
             sort_with_geo_point,
             sort_sum_of_criteria_terms,
             sort_total_number_of_criteria,
+            distinct,
             filter_with_geo_radius,
             filter_with_geo_bounding_box,
             filter_sum_of_criteria_terms,
@@ -769,6 +812,7 @@ impl SearchAggregator {
             attributes_to_search_on_total_number_of_uses,
             max_terms_number,
             max_vector_size,
+            retrieve_vectors,
             matching_strategy,
             max_limit,
             max_offset,
@@ -790,6 +834,7 @@ impl SearchAggregator {
             hybrid,
             total_degraded,
             total_used_negative_operator,
+            ranking_score_threshold,
         } = other;
 
         if self.timestamp.is_none() {
@@ -816,6 +861,9 @@ impl SearchAggregator {
         self.sort_total_number_of_criteria =
             self.sort_total_number_of_criteria.saturating_add(sort_total_number_of_criteria);
 
+        // distinct
+        self.distinct |= distinct;
+
         // filter
         self.filter_with_geo_radius |= filter_with_geo_radius;
         self.filter_with_geo_bounding_box |= filter_with_geo_bounding_box;
@@ -838,6 +886,7 @@ impl SearchAggregator {
 
         // vector
         self.max_vector_size = self.max_vector_size.max(max_vector_size);
+        self.retrieve_vectors |= retrieve_vectors;
         self.semantic_ratio |= semantic_ratio;
         self.hybrid |= hybrid;
         self.embedder |= embedder;
@@ -873,6 +922,7 @@ impl SearchAggregator {
         // scoring
         self.show_ranking_score |= show_ranking_score;
         self.show_ranking_score_details |= show_ranking_score_details;
+        self.ranking_score_threshold |= ranking_score_threshold;
     }
 
     pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
@@ -885,6 +935,7 @@ impl SearchAggregator {
             sort_with_geo_point,
             sort_sum_of_criteria_terms,
             sort_total_number_of_criteria,
+            distinct,
             filter_with_geo_radius,
             filter_with_geo_bounding_box,
             filter_sum_of_criteria_terms,
@@ -893,6 +944,7 @@ impl SearchAggregator {
             attributes_to_search_on_total_number_of_uses,
             max_terms_number,
             max_vector_size,
+            retrieve_vectors,
             matching_strategy,
             max_limit,
             max_offset,
@@ -914,6 +966,7 @@ impl SearchAggregator {
             hybrid,
             total_degraded,
             total_used_negative_operator,
+            ranking_score_threshold,
         } = self;
 
         if total_received == 0 {
@@ -940,6 +993,7 @@ impl SearchAggregator {
                     "with_geoPoint": sort_with_geo_point,
                     "avg_criteria_number": format!("{:.2}", sort_sum_of_criteria_terms as f64 / sort_total_number_of_criteria as f64),
                 },
+                "distinct": distinct,
                 "filter": {
                    "with_geoRadius": filter_with_geo_radius,
                    "with_geoBoundingBox": filter_with_geo_bounding_box,
@@ -954,6 +1008,7 @@ impl SearchAggregator {
                 },
                 "vector": {
                     "max_vector_size": max_vector_size,
+                    "retrieve_vectors": retrieve_vectors,
                 },
                 "hybrid": {
                     "enabled": hybrid,
@@ -984,6 +1039,7 @@ impl SearchAggregator {
                 "scoring": {
                     "show_ranking_score": show_ranking_score,
                     "show_ranking_score_details": show_ranking_score_details,
+                    "ranking_score_threshold": ranking_score_threshold,
                 },
             });
 
@@ -1041,6 +1097,7 @@ impl MultiSearchAggregator {
                     page: _,
                     hits_per_page: _,
                     attributes_to_retrieve: _,
+                    retrieve_vectors: _,
                     attributes_to_crop: _,
                     crop_length: _,
                     attributes_to_highlight: _,
@@ -1049,6 +1106,7 @@ impl MultiSearchAggregator {
                     show_matches_position: _,
                     filter: _,
                     sort: _,
+                    distinct: _,
                     facets: _,
                     highlight_pre_tag: _,
                     highlight_post_tag: _,
@@ -1056,6 +1114,7 @@ impl MultiSearchAggregator {
                     matching_strategy: _,
                     attributes_to_search_on: _,
                     hybrid: _,
+                    ranking_score_threshold: _,
                 } = query;
 
                 index_uid.as_str()
@@ -1203,6 +1262,7 @@ impl FacetSearchAggregator {
             matching_strategy,
             attributes_to_search_on,
             hybrid,
+            ranking_score_threshold,
         } = query;
 
         let mut ret = Self::default();
@@ -1217,7 +1277,8 @@ impl FacetSearchAggregator {
             || filter.is_some()
             || *matching_strategy != MatchingStrategy::default()
             || attributes_to_search_on.is_some()
-            || hybrid.is_some();
+            || hybrid.is_some()
+            || ranking_score_threshold.is_some();
 
         ret
     }
@@ -1493,6 +1554,9 @@ pub struct DocumentsFetchAggregator {
     // if a filter was used
     per_filter: bool,
 
+    #[serde(rename = "vector.retrieve_vectors")]
+    retrieve_vectors: bool,
+
     // pagination
     #[serde(rename = "pagination.max_limit")]
     max_limit: usize,
@@ -1502,18 +1566,21 @@ pub struct DocumentsFetchAggregator {
 
 impl DocumentsFetchAggregator {
     pub fn from_query(query: &DocumentFetchKind, request: &HttpRequest) -> Self {
-        let (limit, offset) = match query {
-            DocumentFetchKind::PerDocumentId => (1, 0),
-            DocumentFetchKind::Normal { limit, offset, .. } => (*limit, *offset),
+        let (limit, offset, retrieve_vectors) = match query {
+            DocumentFetchKind::PerDocumentId { retrieve_vectors } => (1, 0, *retrieve_vectors),
+            DocumentFetchKind::Normal { limit, offset, retrieve_vectors, .. } => {
+                (*limit, *offset, *retrieve_vectors)
+            }
         };
         Self {
             timestamp: Some(OffsetDateTime::now_utc()),
             user_agents: extract_user_agents(request).into_iter().collect(),
             total_received: 1,
-            per_document_id: matches!(query, DocumentFetchKind::PerDocumentId),
+            per_document_id: matches!(query, DocumentFetchKind::PerDocumentId { .. }),
             per_filter: matches!(query, DocumentFetchKind::Normal { with_filter, .. } if *with_filter),
             max_limit: limit,
             max_offset: offset,
+            retrieve_vectors,
         }
     }
 
@@ -1527,6 +1594,7 @@ impl DocumentsFetchAggregator {
             per_filter,
             max_limit,
             max_offset,
+            retrieve_vectors,
         } = other;
 
         if self.timestamp.is_none() {
@@ -1542,6 +1610,8 @@ impl DocumentsFetchAggregator {
 
         self.max_limit = self.max_limit.max(max_limit);
         self.max_offset = self.max_offset.max(max_offset);
+
+        self.retrieve_vectors |= retrieve_vectors;
     }
 
     pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
@@ -1558,3 +1628,251 @@ impl DocumentsFetchAggregator {
         })
     }
 }
+
+#[derive(Default)]
+pub struct SimilarAggregator {
+    timestamp: Option<OffsetDateTime>,
+
+    // context
+    user_agents: HashSet<String>,
+
+    // requests
+    total_received: usize,
+    total_succeeded: usize,
+    time_spent: BinaryHeap<usize>,
+
+    // filter
+    filter_with_geo_radius: bool,
+    filter_with_geo_bounding_box: bool,
+    // every time a request has a filter, this field must be incremented by the number of terms it contains
+    filter_sum_of_criteria_terms: usize,
+    // every time a request has a filter, this field must be incremented by one
+    filter_total_number_of_criteria: usize,
+    used_syntax: HashMap<String, usize>,
+
+    // Whether a non-default embedder was specified
+    embedder: bool,
+    retrieve_vectors: bool,
+
+    // pagination
+    max_limit: usize,
+    max_offset: usize,
+
+    // formatting
+    max_attributes_to_retrieve: usize,
+
+    // scoring
+    show_ranking_score: bool,
+    show_ranking_score_details: bool,
+    ranking_score_threshold: bool,
+}
+
+impl SimilarAggregator {
+    #[allow(clippy::field_reassign_with_default)]
+    pub fn from_query(query: &SimilarQuery, request: &HttpRequest) -> Self {
+        let SimilarQuery {
+            id: _,
+            embedder,
+            offset,
+            limit,
+            attributes_to_retrieve: _,
+            retrieve_vectors,
+            show_ranking_score,
+            show_ranking_score_details,
+            filter,
+            ranking_score_threshold,
+        } = query;
+
+        let mut ret = Self::default();
+        ret.timestamp = Some(OffsetDateTime::now_utc());
+
+        ret.total_received = 1;
+        ret.user_agents = extract_user_agents(request).into_iter().collect();
+
+        if let Some(ref filter) = filter {
+            static RE: Lazy<Regex> = Lazy::new(|| Regex::new("AND | OR").unwrap());
+            ret.filter_total_number_of_criteria = 1;
+
+            let syntax = match filter {
+                Value::String(_) => "string".to_string(),
+                Value::Array(values) => {
+                    if values.iter().map(|v| v.to_string()).any(|s| RE.is_match(&s)) {
+                        "mixed".to_string()
+                    } else {
+                        "array".to_string()
+                    }
+                }
+                _ => "none".to_string(),
+            };
+            // convert the string to a HashMap
+            ret.used_syntax.insert(syntax, 1);
+
+            let stringified_filters = filter.to_string();
+            ret.filter_with_geo_radius = stringified_filters.contains("_geoRadius(");
+            ret.filter_with_geo_bounding_box = stringified_filters.contains("_geoBoundingBox(");
+            ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count();
+        }
+
+        ret.max_limit = *limit;
+        ret.max_offset = *offset;
+
+        ret.show_ranking_score = *show_ranking_score;
+        ret.show_ranking_score_details = *show_ranking_score_details;
+        ret.ranking_score_threshold = ranking_score_threshold.is_some();
+
+        ret.embedder = embedder.is_some();
+        ret.retrieve_vectors = *retrieve_vectors;
+
+        ret
+    }
+
+    pub fn succeed(&mut self, result: &SimilarResult) {
+        let SimilarResult { id: _, hits: _, processing_time_ms, hits_info: _ } = result;
+
+        self.total_succeeded = self.total_succeeded.saturating_add(1);
+
+        self.time_spent.push(*processing_time_ms as usize);
+    }
+
+    /// Aggregate one [SimilarAggregator] into another.
+    pub fn aggregate(&mut self, mut other: Self) {
+        let Self {
+            timestamp,
+            user_agents,
+            total_received,
+            total_succeeded,
+            ref mut time_spent,
+            filter_with_geo_radius,
+            filter_with_geo_bounding_box,
+            filter_sum_of_criteria_terms,
+            filter_total_number_of_criteria,
+            used_syntax,
+            max_limit,
+            max_offset,
+            max_attributes_to_retrieve,
+            show_ranking_score,
+            show_ranking_score_details,
+            embedder,
+            ranking_score_threshold,
+            retrieve_vectors,
+        } = other;
+
+        if self.timestamp.is_none() {
+            self.timestamp = timestamp;
+        }
+
+        // context
+        for user_agent in user_agents.into_iter() {
+            self.user_agents.insert(user_agent);
+        }
+
+        // request
+        self.total_received = self.total_received.saturating_add(total_received);
+        self.total_succeeded = self.total_succeeded.saturating_add(total_succeeded);
+        self.time_spent.append(time_spent);
+
+        // filter
+        self.filter_with_geo_radius |= filter_with_geo_radius;
+        self.filter_with_geo_bounding_box |= filter_with_geo_bounding_box;
+        self.filter_sum_of_criteria_terms =
+            self.filter_sum_of_criteria_terms.saturating_add(filter_sum_of_criteria_terms);
+        self.filter_total_number_of_criteria =
+            self.filter_total_number_of_criteria.saturating_add(filter_total_number_of_criteria);
+        for (key, value) in used_syntax.into_iter() {
+            let used_syntax = self.used_syntax.entry(key).or_insert(0);
+            *used_syntax = used_syntax.saturating_add(value);
+        }
+
+        self.embedder |= embedder;
+        self.retrieve_vectors |= retrieve_vectors;
+
+        // pagination
+        self.max_limit = self.max_limit.max(max_limit);
+        self.max_offset = self.max_offset.max(max_offset);
+
+        // formatting
+        self.max_attributes_to_retrieve =
+            self.max_attributes_to_retrieve.max(max_attributes_to_retrieve);
+
+        // scoring
+        self.show_ranking_score |= show_ranking_score;
+        self.show_ranking_score_details |= show_ranking_score_details;
+        self.ranking_score_threshold |= ranking_score_threshold;
+    }
+
+    pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
+        let Self {
+            timestamp,
+            user_agents,
+            total_received,
+            total_succeeded,
+            time_spent,
+            filter_with_geo_radius,
+            filter_with_geo_bounding_box,
+            filter_sum_of_criteria_terms,
+            filter_total_number_of_criteria,
+            used_syntax,
+            max_limit,
+            max_offset,
+            max_attributes_to_retrieve,
+            show_ranking_score,
+            show_ranking_score_details,
+            embedder,
+            ranking_score_threshold,
+            retrieve_vectors,
+        } = self;
+
+        if total_received == 0 {
+            None
+        } else {
+            // we get all the values in a sorted manner
+            let time_spent = time_spent.into_sorted_vec();
+            // the index of the 99th percentage of value
+            let percentile_99th = time_spent.len() * 99 / 100;
+            // We are only interested by the slowest value of the 99th fastest results
+            let time_spent = time_spent.get(percentile_99th);
+
+            let properties = json!({
+                "user-agent": user_agents,
+                "requests": {
+                    "99th_response_time": time_spent.map(|t| format!("{:.2}", t)),
+                    "total_succeeded": total_succeeded,
+                    "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics
+                    "total_received": total_received,
+                },
+                "filter": {
+                   "with_geoRadius": filter_with_geo_radius,
+                   "with_geoBoundingBox": filter_with_geo_bounding_box,
+                   "avg_criteria_number": format!("{:.2}", filter_sum_of_criteria_terms as f64 / filter_total_number_of_criteria as f64),
+                   "most_used_syntax": used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)),
+                },
+                "vector": {
+                    "retrieve_vectors": retrieve_vectors,
+                },
+                "hybrid": {
+                    "embedder": embedder,
+                },
+                "pagination": {
+                   "max_limit": max_limit,
+                   "max_offset": max_offset,
+                },
+                "formatting": {
+                    "max_attributes_to_retrieve": max_attributes_to_retrieve,
+                },
+                "scoring": {
+                    "show_ranking_score": show_ranking_score,
+                    "show_ranking_score_details": show_ranking_score_details,
+                    "ranking_score_threshold": ranking_score_threshold,
+                },
+            });
+
+            Some(Track {
+                timestamp,
+                user: user.clone(),
+                event: event_name.to_string(),
+                properties,
+                ..Default::default()
+            })
+        }
+    }
+}

meilisearch/src/error.rs

@@ -1,6 +1,6 @@
 use actix_web as aweb;
 use aweb::error::{JsonPayloadError, QueryPayloadError};
-use byte_unit::Byte;
+use byte_unit::{Byte, UnitType};
 use meilisearch_types::document_formats::{DocumentFormatError, PayloadType};
 use meilisearch_types::error::{Code, ErrorCode, ResponseError};
 use meilisearch_types::index_uid::{IndexUid, IndexUidFormatError};
@@ -33,7 +33,7 @@ pub enum MeilisearchHttpError {
     TooManySearchRequests(usize),
     #[error("Internal error: Search limiter is down.")]
     SearchLimiterIsDown,
-    #[error("The provided payload reached the size limit. The maximum accepted payload size is {}.",  Byte::from_bytes(*.0 as u64).get_appropriate_unit(true))]
+    #[error("The provided payload reached the size limit. The maximum accepted payload size is {}.",  Byte::from_u64(*.0 as u64).get_appropriate_unit(UnitType::Binary))]
     PayloadTooLarge(usize),
     #[error("Two indexes must be given for each swap. The list `[{}]` contains {} indexes.",
         .0.iter().map(|uid| format!("\"{uid}\"")).collect::<Vec<_>>().join(", "), .0.len()
@@ -98,14 +98,29 @@ impl From<MeilisearchHttpError> for aweb::Error {
 
 impl From<aweb::error::PayloadError> for MeilisearchHttpError {
     fn from(error: aweb::error::PayloadError) -> Self {
-        MeilisearchHttpError::Payload(PayloadError::Payload(error))
+        match error {
+            aweb::error::PayloadError::Incomplete(_) => MeilisearchHttpError::Payload(
+                PayloadError::Payload(ActixPayloadError::IncompleteError),
+            ),
+            _ => MeilisearchHttpError::Payload(PayloadError::Payload(
+                ActixPayloadError::OtherError(error),
+            )),
+        }
     }
 }
 
+#[derive(Debug, thiserror::Error)]
+pub enum ActixPayloadError {
+    #[error("The provided payload is incomplete and cannot be parsed")]
+    IncompleteError,
+    #[error(transparent)]
+    OtherError(aweb::error::PayloadError),
+}
+
 #[derive(Debug, thiserror::Error)]
 pub enum PayloadError {
     #[error(transparent)]
-    Payload(aweb::error::PayloadError),
+    Payload(ActixPayloadError),
     #[error(transparent)]
     Json(JsonPayloadError),
     #[error(transparent)]
@@ -122,13 +137,15 @@ impl ErrorCode for PayloadError {
     fn error_code(&self) -> Code {
         match self {
             PayloadError::Payload(e) => match e {
-                aweb::error::PayloadError::Incomplete(_) => Code::Internal,
-                aweb::error::PayloadError::EncodingCorrupted => Code::Internal,
-                aweb::error::PayloadError::Overflow => Code::PayloadTooLarge,
-                aweb::error::PayloadError::UnknownLength => Code::Internal,
-                aweb::error::PayloadError::Http2Payload(_) => Code::Internal,
-                aweb::error::PayloadError::Io(_) => Code::Internal,
-                _ => todo!(),
+                ActixPayloadError::IncompleteError => Code::BadRequest,
+                ActixPayloadError::OtherError(error) => match error {
+                    aweb::error::PayloadError::EncodingCorrupted => Code::Internal,
+                    aweb::error::PayloadError::Overflow => Code::PayloadTooLarge,
+                    aweb::error::PayloadError::UnknownLength => Code::Internal,
+                    aweb::error::PayloadError::Http2Payload(_) => Code::Internal,
+                    aweb::error::PayloadError::Io(_) => Code::Internal,
+                    _ => todo!(),
+                },
             },
             PayloadError::Json(err) => match err {
                 JsonPayloadError::Overflow { .. } => Code::PayloadTooLarge,

meilisearch/src/extractors/authentication/mod.rs

@@ -12,6 +12,8 @@ use futures::Future;
 use meilisearch_auth::{AuthController, AuthFilter};
 use meilisearch_types::error::{Code, ResponseError};
 
+use self::policies::AuthError;
+
 pub struct GuardedData<P, D> {
     data: D,
     filters: AuthFilter,
@@ -35,12 +37,12 @@ impl<P, D> GuardedData<P, D> {
         let missing_master_key = auth.get_master_key().is_none();
 
         match Self::authenticate(auth, token, index).await? {
-            Some(filters) => match data {
+            Ok(filters) => match data {
                 Some(data) => Ok(Self { data, filters, _marker: PhantomData }),
                 None => Err(AuthenticationError::IrretrievableState.into()),
             },
-            None if missing_master_key => Err(AuthenticationError::MissingMasterKey.into()),
-            None => Err(AuthenticationError::InvalidToken.into()),
+            Err(_) if missing_master_key => Err(AuthenticationError::MissingMasterKey.into()),
+            Err(e) => Err(ResponseError::from_msg(e.to_string(), Code::InvalidApiKey)),
         }
     }
 
@@ -51,12 +53,12 @@ impl<P, D> GuardedData<P, D> {
         let missing_master_key = auth.get_master_key().is_none();
 
         match Self::authenticate(auth, String::new(), None).await? {
-            Some(filters) => match data {
+            Ok(filters) => match data {
                 Some(data) => Ok(Self { data, filters, _marker: PhantomData }),
                 None => Err(AuthenticationError::IrretrievableState.into()),
             },
-            None if missing_master_key => Err(AuthenticationError::MissingMasterKey.into()),
-            None => Err(AuthenticationError::MissingAuthorizationHeader.into()),
+            Err(_) if missing_master_key => Err(AuthenticationError::MissingMasterKey.into()),
+            Err(_) => Err(AuthenticationError::MissingAuthorizationHeader.into()),
         }
     }
 
@@ -64,7 +66,7 @@ impl<P, D> GuardedData<P, D> {
         auth: Data<AuthController>,
         token: String,
         index: Option<String>,
-    ) -> Result<Option<AuthFilter>, ResponseError>
+    ) -> Result<Result<AuthFilter, AuthError>, ResponseError>
     where
         P: Policy + 'static,
     {
@@ -127,13 +129,14 @@ pub trait Policy {
         auth: Data<AuthController>,
         token: &str,
         index: Option<&str>,
-    ) -> Option<AuthFilter>;
+    ) -> Result<AuthFilter, policies::AuthError>;
 }
 
 pub mod policies {
     use actix_web::web::Data;
     use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation};
     use meilisearch_auth::{AuthController, AuthFilter, SearchRules};
+    use meilisearch_types::error::{Code, ErrorCode};
     // reexport actions in policies in order to be used in routes configuration.
     pub use meilisearch_types::keys::{actions, Action};
     use serde::{Deserialize, Serialize};
@@ -144,11 +147,53 @@ pub mod policies {
 
     enum TenantTokenOutcome {
         NotATenantToken,
-        Invalid,
-        Expired,
         Valid(Uuid, SearchRules),
     }
 
+    #[derive(thiserror::Error, Debug)]
+    pub enum AuthError {
+        #[error("Tenant token expired. Was valid up to `{exp}` and we're now `{now}`.")]
+        ExpiredTenantToken { exp: i64, now: i64 },
+        #[error("The provided API key is invalid.")]
+        InvalidApiKey,
+        #[error("The provided tenant token cannot acces the index `{index}`, allowed indexes are {allowed:?}.")]
+        TenantTokenAccessingnUnauthorizedIndex { index: String, allowed: Vec<String> },
+        #[error(
+            "The API key used to generate this tenant token cannot acces the index `{index}`."
+        )]
+        TenantTokenApiKeyAccessingnUnauthorizedIndex { index: String },
+        #[error(
+            "The API key cannot acces the index `{index}`, authorized indexes are {allowed:?}."
+        )]
+        ApiKeyAccessingnUnauthorizedIndex { index: String, allowed: Vec<String> },
+        #[error("The provided tenant token is invalid.")]
+        InvalidTenantToken,
+        #[error("Could not decode tenant token, {0}.")]
+        CouldNotDecodeTenantToken(jsonwebtoken::errors::Error),
+        #[error("Invalid action `{0}`.")]
+        InternalInvalidAction(u8),
+    }
+
+    impl From<jsonwebtoken::errors::Error> for AuthError {
+        fn from(error: jsonwebtoken::errors::Error) -> Self {
+            use jsonwebtoken::errors::ErrorKind;
+
+            match error.kind() {
+                ErrorKind::InvalidToken => AuthError::InvalidTenantToken,
+                _ => AuthError::CouldNotDecodeTenantToken(error),
+            }
+        }
+    }
+
+    impl ErrorCode for AuthError {
+        fn error_code(&self) -> Code {
+            match self {
+                AuthError::InternalInvalidAction(_) => Code::Internal,
+                _ => Code::InvalidApiKey,
+            }
+        }
+    }
+
     fn tenant_token_validation() -> Validation {
         let mut validation = Validation::default();
         validation.validate_exp = false;
@@ -158,15 +203,15 @@ pub mod policies {
     }
 
     /// Extracts the key id used to sign the payload, without performing any validation.
-    fn extract_key_id(token: &str) -> Option<Uuid> {
+    fn extract_key_id(token: &str) -> Result<Uuid, AuthError> {
         let mut validation = tenant_token_validation();
         validation.insecure_disable_signature_validation();
         let dummy_key = DecodingKey::from_secret(b"secret");
-        let token_data = decode::<Claims>(token, &dummy_key, &validation).ok()?;
+        let token_data = decode::<Claims>(token, &dummy_key, &validation)?;
 
         // get token fields without validating it.
         let Claims { api_key_uid, .. } = token_data.claims;
-        Some(api_key_uid)
+        Ok(api_key_uid)
     }
 
     fn is_keys_action(action: u8) -> bool {
@@ -187,76 +232,102 @@ pub mod policies {
             auth: Data<AuthController>,
             token: &str,
             index: Option<&str>,
-        ) -> Option<AuthFilter> {
+        ) -> Result<AuthFilter, AuthError> {
             // authenticate if token is the master key.
             // Without a master key, all routes are accessible except the key-related routes.
             if auth.get_master_key().map_or_else(|| !is_keys_action(A), |mk| mk == token) {
-                return Some(AuthFilter::default());
+                return Ok(AuthFilter::default());
             }
 
             let (key_uuid, search_rules) =
                 match ActionPolicy::<A>::authenticate_tenant_token(&auth, token) {
-                    TenantTokenOutcome::Valid(key_uuid, search_rules) => {
+                    Ok(TenantTokenOutcome::Valid(key_uuid, search_rules)) => {
                         (key_uuid, Some(search_rules))
                     }
-                    TenantTokenOutcome::Expired => return None,
-                    TenantTokenOutcome::Invalid => return None,
-                    TenantTokenOutcome::NotATenantToken => {
-                        (auth.get_optional_uid_from_encoded_key(token.as_bytes()).ok()??, None)
-                    }
+                    Ok(TenantTokenOutcome::NotATenantToken)
+                    | Err(AuthError::InvalidTenantToken) => (
+                        auth.get_optional_uid_from_encoded_key(token.as_bytes())
+                            .map_err(|_e| AuthError::InvalidApiKey)?
+                            .ok_or(AuthError::InvalidApiKey)?,
+                        None,
+                    ),
+                    Err(e) => return Err(e),
                 };
 
             // check that the indexes are allowed
-            let action = Action::from_repr(A)?;
-            let auth_filter = auth.get_key_filters(key_uuid, search_rules).ok()?;
-            if auth.is_key_authorized(key_uuid, action, index).unwrap_or(false)
-                && index.map(|index| auth_filter.is_index_authorized(index)).unwrap_or(true)
-            {
-                return Some(auth_filter);
+            let action = Action::from_repr(A).ok_or(AuthError::InternalInvalidAction(A))?;
+            let auth_filter = auth
+                .get_key_filters(key_uuid, search_rules)
+                .map_err(|_e| AuthError::InvalidApiKey)?;
+
+            // First check if the index is authorized in the tenant token, this is a public
+            // information, we can return a nice error message.
+            if let Some(index) = index {
+                if !auth_filter.tenant_token_is_index_authorized(index) {
+                    return Err(AuthError::TenantTokenAccessingnUnauthorizedIndex {
+                        index: index.to_string(),
+                        allowed: auth_filter.tenant_token_list_index_authorized(),
+                    });
+                }
+                if !auth_filter.api_key_is_index_authorized(index) {
+                    if auth_filter.is_tenant_token() {
+                        // If the error comes from a tenant token we cannot share the list
+                        // of authorized indexes in the API key. This is not public information.
+                        return Err(AuthError::TenantTokenApiKeyAccessingnUnauthorizedIndex {
+                            index: index.to_string(),
+                        });
+                    } else {
+                        // Otherwise we can share the list
+                        // of authorized indexes in the API key.
+                        return Err(AuthError::ApiKeyAccessingnUnauthorizedIndex {
+                            index: index.to_string(),
+                            allowed: auth_filter.api_key_list_index_authorized(),
+                        });
+                    }
+                }
+            }
+            if auth.is_key_authorized(key_uuid, action, index).unwrap_or(false) {
+                return Ok(auth_filter);
             }
 
-            None
+            Err(AuthError::InvalidApiKey)
         }
     }
 
     impl<const A: u8> ActionPolicy<A> {
-        fn authenticate_tenant_token(auth: &AuthController, token: &str) -> TenantTokenOutcome {
+        fn authenticate_tenant_token(
+            auth: &AuthController,
+            token: &str,
+        ) -> Result<TenantTokenOutcome, AuthError> {
             // Only search action can be accessed by a tenant token.
             if A != actions::SEARCH {
-                return TenantTokenOutcome::NotATenantToken;
+                return Ok(TenantTokenOutcome::NotATenantToken);
             }
 
-            let uid = if let Some(uid) = extract_key_id(token) {
-                uid
-            } else {
-                return TenantTokenOutcome::NotATenantToken;
-            };
+            let uid = extract_key_id(token)?;
 
             // Check if tenant token is valid.
             let key = if let Some(key) = auth.generate_key(uid) {
                 key
             } else {
-                return TenantTokenOutcome::Invalid;
+                return Err(AuthError::InvalidTenantToken);
             };
 
-            let data = if let Ok(data) = decode::<Claims>(
+            let data = decode::<Claims>(
                 token,
                 &DecodingKey::from_secret(key.as_bytes()),
                 &tenant_token_validation(),
-            ) {
-                data
-            } else {
-                return TenantTokenOutcome::Invalid;
-            };
+            )?;
 
             // Check if token is expired.
             if let Some(exp) = data.claims.exp {
-                if OffsetDateTime::now_utc().unix_timestamp() > exp {
-                    return TenantTokenOutcome::Expired;
+                let now = OffsetDateTime::now_utc().unix_timestamp();
+                if now > exp {
+                    return Err(AuthError::ExpiredTenantToken { exp, now });
                 }
             }
 
-            TenantTokenOutcome::Valid(uid, data.claims.search_rules)
+            Ok(TenantTokenOutcome::Valid(uid, data.claims.search_rules))
         }
     }
 

meilisearch/src/lib.rs

@@ -15,6 +15,7 @@ use std::fs::File;
 use std::io::{BufReader, BufWriter};
 use std::num::NonZeroUsize;
 use std::path::Path;
+use std::str::FromStr;
 use std::sync::Arc;
 use std::thread::{self, available_parallelism};
 use std::time::Duration;
@@ -23,13 +24,13 @@ use actix_cors::Cors;
 use actix_http::body::MessageBody;
 use actix_web::dev::{ServiceFactory, ServiceResponse};
 use actix_web::error::JsonPayloadError;
+use actix_web::http::header::{CONTENT_TYPE, USER_AGENT};
 use actix_web::web::Data;
 use actix_web::{web, HttpRequest};
 use analytics::Analytics;
 use anyhow::bail;
 use error::PayloadError;
 use extractors::payload::PayloadConfig;
-use http::header::CONTENT_TYPE;
 use index_scheduler::{IndexScheduler, IndexSchedulerOptions};
 use meilisearch_auth::AuthController;
 use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
@@ -167,7 +168,7 @@ impl tracing_actix_web::RootSpanBuilder for AwebTracingLogger {
         let conn_info = request.connection_info();
         let headers = request.headers();
         let user_agent = headers
-            .get(http::header::USER_AGENT)
+            .get(USER_AGENT)
             .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned())
             .unwrap_or_default();
         info_span!("HTTP request", method = %request.method(), host = conn_info.host(), route = %request.path(), query_parameters = %request.query_string(), %user_agent, status_code = Empty, error = Empty)
@@ -300,15 +301,15 @@ fn open_or_create_database_unchecked(
             dumps_path: opt.dump_dir.clone(),
             webhook_url: opt.task_webhook_url.as_ref().map(|url| url.to_string()),
             webhook_authorization_header: opt.task_webhook_authorization_header.clone(),
-            task_db_size: opt.max_task_db_size.get_bytes() as usize,
-            index_base_map_size: opt.max_index_size.get_bytes() as usize,
+            task_db_size: opt.max_task_db_size.as_u64() as usize,
+            index_base_map_size: opt.max_index_size.as_u64() as usize,
             enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage,
             indexer_config: (&opt.indexer_options).try_into()?,
             autobatching_enabled: true,
             cleanup_enabled: !opt.experimental_replication_parameters,
             max_number_of_tasks: 1_000_000,
             max_number_of_batched_tasks: opt.experimental_max_number_of_batched_tasks,
-            index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().get_bytes() as usize,
+            index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().as_u64() as usize,
             index_count: DEFAULT_INDEX_COUNT,
             instance_features,
         })?)
@@ -476,7 +477,7 @@ pub fn configure_data(
         opt.experimental_search_queue_size,
         available_parallelism().unwrap_or(NonZeroUsize::new(2).unwrap()),
     );
-    let http_payload_size_limit = opt.http_payload_size_limit.get_bytes() as usize;
+    let http_payload_size_limit = opt.http_payload_size_limit.as_u64() as usize;
     config
         .app_data(index_scheduler)
         .app_data(auth)

meilisearch/src/middleware.rs

@@ -59,10 +59,12 @@ where
             let request_path = req.path();
             let is_registered_resource = req.resource_map().has_resource(request_path);
             if is_registered_resource {
+                let request_pattern = req.match_pattern();
+                let metric_path = request_pattern.as_ref().map_or(request_path, String::as_str);
                 let request_method = req.method().to_string();
                 histogram_timer = Some(
                     crate::metrics::MEILISEARCH_HTTP_RESPONSE_TIME_SECONDS
-                        .with_label_values(&[&request_method, request_path])
+                        .with_label_values(&[&request_method, metric_path])
                         .start_timer(),
                 );
             }

meilisearch/src/option.rs

@@ -9,7 +9,7 @@ use std::str::FromStr;
 use std::sync::Arc;
 use std::{env, fmt, fs};
 
-use byte_unit::{Byte, ByteError};
+use byte_unit::{Byte, ParseError, UnitType};
 use clap::Parser;
 use meilisearch_types::features::InstanceTogglableFeatures;
 use meilisearch_types::milli::update::IndexerConfig;
@@ -674,7 +674,7 @@ impl TryFrom<&IndexerOpts> for IndexerConfig {
 
         Ok(Self {
             log_every_n: Some(DEFAULT_LOG_EVERY_N),
-            max_memory: other.max_indexing_memory.map(|b| b.get_bytes() as usize),
+            max_memory: other.max_indexing_memory.map(|b| b.as_u64() as usize),
             thread_pool: Some(thread_pool),
             max_positions_per_attributes: None,
             skip_index_budget: other.skip_index_budget,
@@ -688,23 +688,25 @@ impl TryFrom<&IndexerOpts> for IndexerConfig {
 pub struct MaxMemory(Option<Byte>);
 
 impl FromStr for MaxMemory {
-    type Err = ByteError;
+    type Err = ParseError;
 
-    fn from_str(s: &str) -> Result<MaxMemory, ByteError> {
+    fn from_str(s: &str) -> Result<MaxMemory, Self::Err> {
         Byte::from_str(s).map(Some).map(MaxMemory)
     }
 }
 
 impl Default for MaxMemory {
     fn default() -> MaxMemory {
-        MaxMemory(total_memory_bytes().map(|bytes| bytes * 2 / 3).map(Byte::from_bytes))
+        MaxMemory(total_memory_bytes().map(|bytes| bytes * 2 / 3).map(Byte::from_u64))
     }
 }
 
 impl fmt::Display for MaxMemory {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self.0 {
-            Some(memory) => write!(f, "{}", memory.get_appropriate_unit(true)),
+            Some(memory) => {
+                write!(f, "{}", memory.get_appropriate_unit(UnitType::Binary))
+            }
             None => f.write_str("unknown"),
         }
     }
@@ -844,11 +846,11 @@ fn default_env() -> String {
 }
 
 fn default_max_index_size() -> Byte {
-    Byte::from_bytes(INDEX_SIZE)
+    Byte::from_u64(INDEX_SIZE)
 }
 
 fn default_max_task_db_size() -> Byte {
-    Byte::from_bytes(TASK_DB_SIZE)
+    Byte::from_u64(TASK_DB_SIZE)
 }
 
 fn default_http_payload_size_limit() -> Byte {

meilisearch/src/routes/features.rs

@@ -47,8 +47,6 @@ pub struct RuntimeTogglableFeatures {
     pub metrics: Option<bool>,
     #[deserr(default)]
     pub logs_route: Option<bool>,
-    #[deserr(default)]
-    pub export_puffin_reports: Option<bool>,
 }
 
 async fn patch_features(
@@ -68,21 +66,13 @@ async fn patch_features(
         vector_store: new_features.0.vector_store.unwrap_or(old_features.vector_store),
         metrics: new_features.0.metrics.unwrap_or(old_features.metrics),
         logs_route: new_features.0.logs_route.unwrap_or(old_features.logs_route),
-        export_puffin_reports: new_features
-            .0
-            .export_puffin_reports
-            .unwrap_or(old_features.export_puffin_reports),
     };
 
     // explicitly destructure for analytics rather than using the `Serialize` implementation, because
     // the it renames to camelCase, which we don't want for analytics.
     // **Do not** ignore fields with `..` or `_` here, because we want to add them in the future.
-    let meilisearch_types::features::RuntimeTogglableFeatures {
-        vector_store,
-        metrics,
-        logs_route,
-        export_puffin_reports,
-    } = new_features;
+    let meilisearch_types::features::RuntimeTogglableFeatures { vector_store, metrics, logs_route } =
+        new_features;
 
     analytics.publish(
         "Experimental features Updated".to_string(),
@@ -90,7 +80,6 @@ async fn patch_features(
             "vector_store": vector_store,
             "metrics": metrics,
             "logs_route": logs_route,
-            "export_puffin_reports": export_puffin_reports,
         }),
         Some(&req),
     );

meilisearch/src/routes/indexes/documents.rs

@@ -16,6 +16,7 @@ use meilisearch_types::error::{Code, ResponseError};
 use meilisearch_types::heed::RoTxn;
 use meilisearch_types::index_uid::IndexUid;
 use meilisearch_types::milli::update::IndexDocumentsMethod;
+use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors;
 use meilisearch_types::milli::DocumentId;
 use meilisearch_types::star_or::OptionStarOrList;
 use meilisearch_types::tasks::KindWithContent;
@@ -39,7 +40,7 @@ use crate::extractors::sequential_extractor::SeqHandler;
 use crate::routes::{
     get_task_id, is_dry_run, PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT,
 };
-use crate::search::parse_filter;
+use crate::search::{parse_filter, RetrieveVectors};
 use crate::Opt;
 
 static ACCEPTED_CONTENT_TYPE: Lazy<Vec<String>> = Lazy::new(|| {
@@ -94,6 +95,8 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
 pub struct GetDocument {
     #[deserr(default, error = DeserrQueryParamError<InvalidDocumentFields>)]
     fields: OptionStarOrList<String>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidDocumentRetrieveVectors>)]
+    retrieve_vectors: Param<bool>,
 }
 
 pub async fn get_document(
@@ -107,13 +110,20 @@ pub async fn get_document(
     debug!(parameters = ?params, "Get document");
     let index_uid = IndexUid::try_from(index_uid)?;
 
-    analytics.get_fetch_documents(&DocumentFetchKind::PerDocumentId, &req);
-
-    let GetDocument { fields } = params.into_inner();
+    let GetDocument { fields, retrieve_vectors: param_retrieve_vectors } = params.into_inner();
     let attributes_to_retrieve = fields.merge_star_and_none();
 
+    let features = index_scheduler.features();
+    let retrieve_vectors = RetrieveVectors::new(param_retrieve_vectors.0, features)?;
+
+    analytics.get_fetch_documents(
+        &DocumentFetchKind::PerDocumentId { retrieve_vectors: param_retrieve_vectors.0 },
+        &req,
+    );
+
     let index = index_scheduler.index(&index_uid)?;
-    let document = retrieve_document(&index, &document_id, attributes_to_retrieve)?;
+    let document =
+        retrieve_document(&index, &document_id, attributes_to_retrieve, retrieve_vectors)?;
     debug!(returns = ?document, "Get document");
     Ok(HttpResponse::Ok().json(document))
 }
@@ -153,6 +163,8 @@ pub struct BrowseQueryGet {
     limit: Param<usize>,
     #[deserr(default, error = DeserrQueryParamError<InvalidDocumentFields>)]
     fields: OptionStarOrList<String>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidDocumentRetrieveVectors>)]
+    retrieve_vectors: Param<bool>,
     #[deserr(default, error = DeserrQueryParamError<InvalidDocumentFilter>)]
     filter: Option<String>,
 }
@@ -166,6 +178,8 @@ pub struct BrowseQuery {
     limit: usize,
     #[deserr(default, error = DeserrJsonError<InvalidDocumentFields>)]
     fields: Option<Vec<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidDocumentRetrieveVectors>)]
+    retrieve_vectors: bool,
     #[deserr(default, error = DeserrJsonError<InvalidDocumentFilter>)]
     filter: Option<Value>,
 }
@@ -185,6 +199,7 @@ pub async fn documents_by_query_post(
             with_filter: body.filter.is_some(),
             limit: body.limit,
             offset: body.offset,
+            retrieve_vectors: body.retrieve_vectors,
         },
         &req,
     );
@@ -201,7 +216,7 @@ pub async fn get_documents(
 ) -> Result<HttpResponse, ResponseError> {
     debug!(parameters = ?params, "Get documents GET");
 
-    let BrowseQueryGet { limit, offset, fields, filter } = params.into_inner();
+    let BrowseQueryGet { limit, offset, fields, retrieve_vectors, filter } = params.into_inner();
 
     let filter = match filter {
         Some(f) => match serde_json::from_str(&f) {
@@ -215,6 +230,7 @@ pub async fn get_documents(
         offset: offset.0,
         limit: limit.0,
         fields: fields.merge_star_and_none(),
+        retrieve_vectors: retrieve_vectors.0,
         filter,
     };
 
@@ -223,6 +239,7 @@ pub async fn get_documents(
             with_filter: query.filter.is_some(),
             limit: query.limit,
             offset: query.offset,
+            retrieve_vectors: query.retrieve_vectors,
         },
         &req,
     );
@@ -236,10 +253,14 @@ fn documents_by_query(
     query: BrowseQuery,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
-    let BrowseQuery { offset, limit, fields, filter } = query;
+    let BrowseQuery { offset, limit, fields, retrieve_vectors, filter } = query;
+
+    let features = index_scheduler.features();
+    let retrieve_vectors = RetrieveVectors::new(retrieve_vectors, features)?;
 
     let index = index_scheduler.index(&index_uid)?;
-    let (total, documents) = retrieve_documents(&index, offset, limit, filter, fields)?;
+    let (total, documents) =
+        retrieve_documents(&index, offset, limit, filter, fields, retrieve_vectors)?;
 
     let ret = PaginationView::new(offset, limit, total as usize, documents);
 
@@ -579,14 +600,54 @@ fn some_documents<'a, 't: 'a>(
     index: &'a Index,
     rtxn: &'t RoTxn,
     doc_ids: impl IntoIterator<Item = DocumentId> + 'a,
+    retrieve_vectors: RetrieveVectors,
 ) -> Result<impl Iterator<Item = Result<Document, ResponseError>> + 'a, ResponseError> {
     let fields_ids_map = index.fields_ids_map(rtxn)?;
+    let dictionary = index.document_decompression_dictionary(rtxn)?;
     let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
+    let embedding_configs = index.embedding_configs(rtxn)?;
+    let mut buffer = Vec::new();
 
-    Ok(index.iter_documents(rtxn, doc_ids)?.map(move |ret| {
-        ret.map_err(ResponseError::from).and_then(|(_key, document)| -> Result<_, ResponseError> {
-            Ok(milli::obkv_to_json(&all_fields, &fields_ids_map, document)?)
-        })
+    Ok(index.iter_compressed_documents(rtxn, doc_ids)?.map(move |ret| {
+        ret.map_err(ResponseError::from).and_then(
+            |(key, compressed_document)| -> Result<_, ResponseError> {
+                let document = compressed_document
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())?;
+                let mut document = milli::obkv_to_json(&all_fields, &fields_ids_map, document)?;
+                match retrieve_vectors {
+                    RetrieveVectors::Ignore => {}
+                    RetrieveVectors::Hide => {
+                        document.remove("_vectors");
+                    }
+                    RetrieveVectors::Retrieve => {
+                        // Clippy is simply wrong
+                        #[allow(clippy::manual_unwrap_or_default)]
+                        let mut vectors = match document.remove("_vectors") {
+                            Some(Value::Object(map)) => map,
+                            _ => Default::default(),
+                        };
+                        for (name, vector) in index.embeddings(rtxn, key)? {
+                            let user_provided = embedding_configs
+                                .iter()
+                                .find(|conf| conf.name == name)
+                                .is_some_and(|conf| conf.user_provided.contains(key));
+                            let embeddings = ExplicitVectors {
+                                embeddings: Some(vector.into()),
+                                regenerate: !user_provided,
+                            };
+                            vectors.insert(
+                                name,
+                                serde_json::to_value(embeddings)
+                                    .map_err(MeilisearchHttpError::from)?,
+                            );
+                        }
+                        document.insert("_vectors".into(), vectors.into());
+                    }
+                }
+
+                Ok(document)
+            },
+        )
     }))
 }
 
@@ -596,6 +657,7 @@ fn retrieve_documents<S: AsRef<str>>(
     limit: usize,
     filter: Option<Value>,
     attributes_to_retrieve: Option<Vec<S>>,
+    retrieve_vectors: RetrieveVectors,
 ) -> Result<(u64, Vec<Document>), ResponseError> {
     let rtxn = index.read_txn()?;
     let filter = &filter;
@@ -620,53 +682,57 @@ fn retrieve_documents<S: AsRef<str>>(
     let (it, number_of_documents) = {
         let number_of_documents = candidates.len();
         (
-            some_documents(index, &rtxn, candidates.into_iter().skip(offset).take(limit))?,
+            some_documents(
+                index,
+                &rtxn,
+                candidates.into_iter().skip(offset).take(limit),
+                retrieve_vectors,
+            )?,
             number_of_documents,
         )
     };
 
-    let documents: Result<Vec<_>, ResponseError> = it
+    let documents: Vec<_> = it
         .map(|document| {
             Ok(match &attributes_to_retrieve {
                 Some(attributes_to_retrieve) => permissive_json_pointer::select_values(
                     &document?,
-                    attributes_to_retrieve.iter().map(|s| s.as_ref()),
+                    attributes_to_retrieve.iter().map(|s| s.as_ref()).chain(
+                        (retrieve_vectors == RetrieveVectors::Retrieve).then_some("_vectors"),
+                    ),
                 ),
                 None => document?,
             })
         })
-        .collect();
+        .collect::<Result<_, ResponseError>>()?;
 
-    Ok((number_of_documents, documents?))
+    Ok((number_of_documents, documents))
 }
 
 fn retrieve_document<S: AsRef<str>>(
     index: &Index,
     doc_id: &str,
     attributes_to_retrieve: Option<Vec<S>>,
+    retrieve_vectors: RetrieveVectors,
 ) -> Result<Document, ResponseError> {
     let txn = index.read_txn()?;
 
-    let fields_ids_map = index.fields_ids_map(&txn)?;
-    let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
-
     let internal_id = index
         .external_documents_ids()
         .get(&txn, doc_id)?
         .ok_or_else(|| MeilisearchHttpError::DocumentNotFound(doc_id.to_string()))?;
 
-    let document = index
-        .documents(&txn, std::iter::once(internal_id))?
-        .into_iter()
+    let document = some_documents(index, &txn, Some(internal_id), retrieve_vectors)?
         .next()
-        .map(|(_, d)| d)
-        .ok_or_else(|| MeilisearchHttpError::DocumentNotFound(doc_id.to_string()))?;
+        .ok_or_else(|| MeilisearchHttpError::DocumentNotFound(doc_id.to_string()))??;
 
-    let document = meilisearch_types::milli::obkv_to_json(&all_fields, &fields_ids_map, document)?;
     let document = match &attributes_to_retrieve {
         Some(attributes_to_retrieve) => permissive_json_pointer::select_values(
             &document,
-            attributes_to_retrieve.iter().map(|s| s.as_ref()),
+            attributes_to_retrieve
+                .iter()
+                .map(|s| s.as_ref())
+                .chain((retrieve_vectors == RetrieveVectors::Retrieve).then_some("_vectors")),
         ),
         None => document,
     };

meilisearch/src/routes/indexes/facet_search.rs

@@ -14,8 +14,8 @@ use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::routes::indexes::search::search_kind;
 use crate::search::{
-    add_search_rules, perform_facet_search, HybridQuery, MatchingStrategy, SearchQuery,
-    DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
+    add_search_rules, perform_facet_search, HybridQuery, MatchingStrategy, RankingScoreThreshold,
+    SearchQuery, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
     DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET,
 };
 use crate::search_queue::SearchQueue;
@@ -46,6 +46,8 @@ pub struct FacetSearchQuery {
     pub matching_strategy: MatchingStrategy,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToSearchOn>, default)]
     pub attributes_to_search_on: Option<Vec<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchRankingScoreThreshold>, default)]
+    pub ranking_score_threshold: Option<RankingScoreThreshold>,
 }
 
 pub async fn search(
@@ -69,7 +71,7 @@ pub async fn search(
 
     // Tenant token search_rules.
     if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(&index_uid) {
-        add_search_rules(&mut search_query, search_rules);
+        add_search_rules(&mut search_query.filter, search_rules);
     }
 
     let index = index_scheduler.index(&index_uid)?;
@@ -103,6 +105,7 @@ impl From<FacetSearchQuery> for SearchQuery {
             matching_strategy,
             attributes_to_search_on,
             hybrid,
+            ranking_score_threshold,
         } = value;
 
         SearchQuery {
@@ -112,6 +115,7 @@ impl From<FacetSearchQuery> for SearchQuery {
             page: None,
             hits_per_page: None,
             attributes_to_retrieve: None,
+            retrieve_vectors: false,
             attributes_to_crop: None,
             crop_length: DEFAULT_CROP_LENGTH(),
             attributes_to_highlight: None,
@@ -120,6 +124,7 @@ impl From<FacetSearchQuery> for SearchQuery {
             show_ranking_score_details: false,
             filter,
             sort: None,
+            distinct: None,
             facets: None,
             highlight_pre_tag: DEFAULT_HIGHLIGHT_PRE_TAG(),
             highlight_post_tag: DEFAULT_HIGHLIGHT_POST_TAG(),
@@ -128,6 +133,7 @@ impl From<FacetSearchQuery> for SearchQuery {
             vector,
             attributes_to_search_on,
             hybrid,
+            ranking_score_threshold,
         }
     }
 }

meilisearch/src/routes/indexes/mod.rs

@@ -29,6 +29,7 @@ pub mod documents;
 pub mod facet_search;
 pub mod search;
 pub mod settings;
+pub mod similar;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(
@@ -48,6 +49,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
             .service(web::scope("/documents").configure(documents::configure))
             .service(web::scope("/search").configure(search::configure))
             .service(web::scope("/facet-search").configure(facet_search::configure))
+            .service(web::scope("/similar").configure(similar::configure))
             .service(web::scope("/settings").configure(settings::configure)),
     );
 }

meilisearch/src/routes/indexes/search.rs

@@ -19,9 +19,10 @@ use crate::extractors::authentication::GuardedData;
 use crate::extractors::sequential_extractor::SeqHandler;
 use crate::metrics::MEILISEARCH_DEGRADED_SEARCH_REQUESTS;
 use crate::search::{
-    add_search_rules, perform_search, HybridQuery, MatchingStrategy, SearchKind, SearchQuery,
-    SemanticRatio, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
-    DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET, DEFAULT_SEMANTIC_RATIO,
+    add_search_rules, perform_search, HybridQuery, MatchingStrategy, RankingScoreThreshold,
+    RetrieveVectors, SearchKind, SearchQuery, SemanticRatio, DEFAULT_CROP_LENGTH,
+    DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG,
+    DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET, DEFAULT_SEMANTIC_RATIO,
 };
 use crate::search_queue::SearchQueue;
 
@@ -50,6 +51,8 @@ pub struct SearchQueryGet {
     hits_per_page: Option<Param<usize>>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchAttributesToRetrieve>)]
     attributes_to_retrieve: Option<CS<String>>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSearchRetrieveVectors>)]
+    retrieve_vectors: Param<bool>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchAttributesToCrop>)]
     attributes_to_crop: Option<CS<String>>,
     #[deserr(default = Param(DEFAULT_CROP_LENGTH()), error = DeserrQueryParamError<InvalidSearchCropLength>)]
@@ -60,6 +63,8 @@ pub struct SearchQueryGet {
     filter: Option<String>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchSort>)]
     sort: Option<String>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSearchDistinct>)]
+    distinct: Option<String>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchShowMatchesPosition>)]
     show_matches_position: Param<bool>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchShowRankingScore>)]
@@ -82,6 +87,21 @@ pub struct SearchQueryGet {
     pub hybrid_embedder: Option<String>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchSemanticRatio>)]
     pub hybrid_semantic_ratio: Option<SemanticRatioGet>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSearchRankingScoreThreshold>)]
+    pub ranking_score_threshold: Option<RankingScoreThresholdGet>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, deserr::Deserr)]
+#[deserr(try_from(String) = TryFrom::try_from -> InvalidSearchRankingScoreThreshold)]
+pub struct RankingScoreThresholdGet(RankingScoreThreshold);
+
+impl std::convert::TryFrom<String> for RankingScoreThresholdGet {
+    type Error = InvalidSearchRankingScoreThreshold;
+
+    fn try_from(s: String) -> Result<Self, Self::Error> {
+        let f: f64 = s.parse().map_err(|_| InvalidSearchRankingScoreThreshold)?;
+        Ok(RankingScoreThresholdGet(RankingScoreThreshold::try_from(f)?))
+    }
 }
 
 #[derive(Debug, Clone, Copy, Default, PartialEq, deserr::Deserr)]
@@ -137,11 +157,13 @@ impl From<SearchQueryGet> for SearchQuery {
             page: other.page.as_deref().copied(),
             hits_per_page: other.hits_per_page.as_deref().copied(),
             attributes_to_retrieve: other.attributes_to_retrieve.map(|o| o.into_iter().collect()),
+            retrieve_vectors: other.retrieve_vectors.0,
             attributes_to_crop: other.attributes_to_crop.map(|o| o.into_iter().collect()),
             crop_length: other.crop_length.0,
             attributes_to_highlight: other.attributes_to_highlight.map(|o| o.into_iter().collect()),
             filter,
             sort: other.sort.map(|attr| fix_sort_query_parameters(&attr)),
+            distinct: other.distinct,
             show_matches_position: other.show_matches_position.0,
             show_ranking_score: other.show_ranking_score.0,
             show_ranking_score_details: other.show_ranking_score_details.0,
@@ -152,6 +174,7 @@ impl From<SearchQueryGet> for SearchQuery {
             matching_strategy: other.matching_strategy,
             attributes_to_search_on: other.attributes_to_search_on.map(|o| o.into_iter().collect()),
             hybrid,
+            ranking_score_threshold: other.ranking_score_threshold.map(|o| o.0),
         }
     }
 }
@@ -196,7 +219,7 @@ pub async fn search_with_url_query(
 
     // Tenant token search_rules.
     if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(&index_uid) {
-        add_search_rules(&mut query, search_rules);
+        add_search_rules(&mut query.filter, search_rules);
     }
 
     let mut aggregate = SearchAggregator::from_query(&query, &req);
@@ -205,10 +228,12 @@ pub async fn search_with_url_query(
     let features = index_scheduler.features();
 
     let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)?;
-
+    let retrieve_vector = RetrieveVectors::new(query.retrieve_vectors, features)?;
     let _permit = search_queue.try_get_search_permit().await?;
-    let search_result =
-        tokio::task::spawn_blocking(move || perform_search(&index, query, search_kind)).await?;
+    let search_result = tokio::task::spawn_blocking(move || {
+        perform_search(&index, query, search_kind, retrieve_vector)
+    })
+    .await?;
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
     }
@@ -235,7 +260,7 @@ pub async fn search_with_post(
 
     // Tenant token search_rules.
     if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(&index_uid) {
-        add_search_rules(&mut query, search_rules);
+        add_search_rules(&mut query.filter, search_rules);
     }
 
     let mut aggregate = SearchAggregator::from_query(&query, &req);
@@ -245,10 +270,13 @@ pub async fn search_with_post(
     let features = index_scheduler.features();
 
     let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)?;
+    let retrieve_vectors = RetrieveVectors::new(query.retrieve_vectors, features)?;
 
     let _permit = search_queue.try_get_search_permit().await?;
-    let search_result =
-        tokio::task::spawn_blocking(move || perform_search(&index, query, search_kind)).await?;
+    let search_result = tokio::task::spawn_blocking(move || {
+        perform_search(&index, query, search_kind, retrieve_vectors)
+    })
+    .await?;
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
         if search_result.degraded {
@@ -270,11 +298,10 @@ pub fn search_kind(
     features: RoFeatures,
 ) -> Result<SearchKind, ResponseError> {
     if query.vector.is_some() {
-        features.check_vector("Passing `vector` as a query parameter")?;
+        features.check_vector("Passing `vector` as a parameter")?;
     }
-
     if query.hybrid.is_some() {
-        features.check_vector("Passing `hybrid` as a query parameter")?;
+        features.check_vector("Passing `hybrid` as a parameter")?;
     }
 
     // regardless of anything, always do a keyword search when we don't have a vector and the query is whitespace or missing

meilisearch/src/routes/indexes/similar.rs

@@ -0,0 +1,192 @@
+use actix_web::web::{self, Data};
+use actix_web::{HttpRequest, HttpResponse};
+use deserr::actix_web::{AwebJson, AwebQueryParameter};
+use index_scheduler::IndexScheduler;
+use meilisearch_types::deserr::query_params::Param;
+use meilisearch_types::deserr::{DeserrJsonError, DeserrQueryParamError};
+use meilisearch_types::error::deserr_codes::*;
+use meilisearch_types::error::{ErrorCode as _, ResponseError};
+use meilisearch_types::index_uid::IndexUid;
+use meilisearch_types::keys::actions;
+use meilisearch_types::serde_cs::vec::CS;
+use serde_json::Value;
+use tracing::debug;
+
+use super::ActionPolicy;
+use crate::analytics::{Analytics, SimilarAggregator};
+use crate::extractors::authentication::GuardedData;
+use crate::extractors::sequential_extractor::SeqHandler;
+use crate::search::{
+    add_search_rules, perform_similar, RankingScoreThresholdSimilar, RetrieveVectors, SearchKind,
+    SimilarQuery, SimilarResult, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET,
+};
+
+pub fn configure(cfg: &mut web::ServiceConfig) {
+    cfg.service(
+        web::resource("")
+            .route(web::get().to(SeqHandler(similar_get)))
+            .route(web::post().to(SeqHandler(similar_post))),
+    );
+}
+
+pub async fn similar_get(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::SEARCH }>, Data<IndexScheduler>>,
+    index_uid: web::Path<String>,
+    params: AwebQueryParameter<SimilarQueryGet, DeserrQueryParamError>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
+) -> Result<HttpResponse, ResponseError> {
+    let index_uid = IndexUid::try_from(index_uid.into_inner())?;
+
+    let query = params.0.try_into()?;
+
+    let mut aggregate = SimilarAggregator::from_query(&query, &req);
+
+    debug!(parameters = ?query, "Similar get");
+
+    let similar = similar(index_scheduler, index_uid, query).await;
+
+    if let Ok(similar) = &similar {
+        aggregate.succeed(similar);
+    }
+    analytics.get_similar(aggregate);
+
+    let similar = similar?;
+
+    debug!(returns = ?similar, "Similar get");
+    Ok(HttpResponse::Ok().json(similar))
+}
+
+pub async fn similar_post(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::SEARCH }>, Data<IndexScheduler>>,
+    index_uid: web::Path<String>,
+    params: AwebJson<SimilarQuery, DeserrJsonError>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
+) -> Result<HttpResponse, ResponseError> {
+    let index_uid = IndexUid::try_from(index_uid.into_inner())?;
+
+    let query = params.into_inner();
+    debug!(parameters = ?query, "Similar post");
+
+    let mut aggregate = SimilarAggregator::from_query(&query, &req);
+
+    let similar = similar(index_scheduler, index_uid, query).await;
+
+    if let Ok(similar) = &similar {
+        aggregate.succeed(similar);
+    }
+    analytics.post_similar(aggregate);
+
+    let similar = similar?;
+
+    debug!(returns = ?similar, "Similar post");
+    Ok(HttpResponse::Ok().json(similar))
+}
+
+async fn similar(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::SEARCH }>, Data<IndexScheduler>>,
+    index_uid: IndexUid,
+    mut query: SimilarQuery,
+) -> Result<SimilarResult, ResponseError> {
+    let features = index_scheduler.features();
+
+    features.check_vector("Using the similar API")?;
+
+    let retrieve_vectors = RetrieveVectors::new(query.retrieve_vectors, features)?;
+
+    // Tenant token search_rules.
+    if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(&index_uid) {
+        add_search_rules(&mut query.filter, search_rules);
+    }
+
+    let index = index_scheduler.index(&index_uid)?;
+
+    let (embedder_name, embedder) =
+        SearchKind::embedder(&index_scheduler, &index, query.embedder.as_deref(), None)?;
+
+    tokio::task::spawn_blocking(move || {
+        perform_similar(&index, query, embedder_name, embedder, retrieve_vectors)
+    })
+    .await?
+}
+
+#[derive(Debug, deserr::Deserr)]
+#[deserr(error = DeserrQueryParamError, rename_all = camelCase, deny_unknown_fields)]
+pub struct SimilarQueryGet {
+    #[deserr(error = DeserrQueryParamError<InvalidSimilarId>)]
+    id: Param<String>,
+    #[deserr(default = Param(DEFAULT_SEARCH_OFFSET()), error = DeserrQueryParamError<InvalidSimilarOffset>)]
+    offset: Param<usize>,
+    #[deserr(default = Param(DEFAULT_SEARCH_LIMIT()), error = DeserrQueryParamError<InvalidSimilarLimit>)]
+    limit: Param<usize>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSimilarAttributesToRetrieve>)]
+    attributes_to_retrieve: Option<CS<String>>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSimilarRetrieveVectors>)]
+    retrieve_vectors: Param<bool>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSimilarFilter>)]
+    filter: Option<String>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSimilarShowRankingScore>)]
+    show_ranking_score: Param<bool>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSimilarShowRankingScoreDetails>)]
+    show_ranking_score_details: Param<bool>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidSimilarRankingScoreThreshold>, default)]
+    pub ranking_score_threshold: Option<RankingScoreThresholdGet>,
+    #[deserr(default, error = DeserrQueryParamError<InvalidEmbedder>)]
+    pub embedder: Option<String>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, deserr::Deserr)]
+#[deserr(try_from(String) = TryFrom::try_from -> InvalidSimilarRankingScoreThreshold)]
+pub struct RankingScoreThresholdGet(RankingScoreThresholdSimilar);
+
+impl std::convert::TryFrom<String> for RankingScoreThresholdGet {
+    type Error = InvalidSimilarRankingScoreThreshold;
+
+    fn try_from(s: String) -> Result<Self, Self::Error> {
+        let f: f64 = s.parse().map_err(|_| InvalidSimilarRankingScoreThreshold)?;
+        Ok(RankingScoreThresholdGet(RankingScoreThresholdSimilar::try_from(f)?))
+    }
+}
+
+impl TryFrom<SimilarQueryGet> for SimilarQuery {
+    type Error = ResponseError;
+
+    fn try_from(
+        SimilarQueryGet {
+            id,
+            offset,
+            limit,
+            attributes_to_retrieve,
+            retrieve_vectors,
+            filter,
+            show_ranking_score,
+            show_ranking_score_details,
+            embedder,
+            ranking_score_threshold,
+        }: SimilarQueryGet,
+    ) -> Result<Self, Self::Error> {
+        let filter = match filter {
+            Some(f) => match serde_json::from_str(&f) {
+                Ok(v) => Some(v),
+                _ => Some(Value::String(f)),
+            },
+            None => None,
+        };
+
+        Ok(SimilarQuery {
+            id: id.0.try_into().map_err(|code: InvalidSimilarId| {
+                ResponseError::from_msg(code.to_string(), code.error_code())
+            })?,
+            offset: offset.0,
+            limit: limit.0,
+            filter,
+            embedder,
+            attributes_to_retrieve: attributes_to_retrieve.map(|o| o.into_iter().collect()),
+            retrieve_vectors: retrieve_vectors.0,
+            show_ranking_score: show_ranking_score.0,
+            show_ranking_score_details: show_ranking_score_details.0,
+            ranking_score_threshold: ranking_score_threshold.map(|x| x.0),
+        })
+    }
+}

meilisearch/src/routes/multi_search.rs

@@ -15,7 +15,7 @@ use crate::extractors::authentication::{AuthenticationError, GuardedData};
 use crate::extractors::sequential_extractor::SeqHandler;
 use crate::routes::indexes::search::search_kind;
 use crate::search::{
-    add_search_rules, perform_search, SearchQueryWithIndex, SearchResultWithIndex,
+    add_search_rules, perform_search, RetrieveVectors, SearchQueryWithIndex, SearchResultWithIndex,
 };
 use crate::search_queue::SearchQueue;
 
@@ -67,7 +67,7 @@ pub async fn multi_search_with_post(
             // Apply search rules from tenant token
             if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(&index_uid)
             {
-                add_search_rules(&mut query, search_rules);
+                add_search_rules(&mut query.filter, search_rules);
             }
 
             let index = index_scheduler
@@ -83,11 +83,14 @@ pub async fn multi_search_with_post(
 
             let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)
                 .with_index(query_index)?;
+            let retrieve_vector =
+                RetrieveVectors::new(query.retrieve_vectors, features).with_index(query_index)?;
 
-            let search_result =
-                tokio::task::spawn_blocking(move || perform_search(&index, query, search_kind))
-                    .await
-                    .with_index(query_index)?;
+            let search_result = tokio::task::spawn_blocking(move || {
+                perform_search(&index, query, search_kind, retrieve_vector)
+            })
+            .await
+            .with_index(query_index)?;
 
             search_results.push(SearchResultWithIndex {
                 index_uid: index_uid.into_inner(),

meilisearch/src/search.rs

@@ -11,10 +11,11 @@ use indexmap::IndexMap;
 use meilisearch_auth::IndexSearchRules;
 use meilisearch_types::deserr::DeserrJsonError;
 use meilisearch_types::error::deserr_codes::*;
-use meilisearch_types::error::ResponseError;
+use meilisearch_types::error::{Code, ResponseError};
 use meilisearch_types::heed::RoTxn;
 use meilisearch_types::index_uid::IndexUid;
 use meilisearch_types::milli::score_details::{ScoreDetails, ScoringStrategy};
+use meilisearch_types::milli::vector::parsed_vectors::ExplicitVectors;
 use meilisearch_types::milli::vector::Embedder;
 use meilisearch_types::milli::{FacetValueHit, OrderBy, SearchForFacetValues, TimeBudget};
 use meilisearch_types::settings::DEFAULT_PAGINATION_MAX_TOTAL_HITS;
@@ -59,6 +60,8 @@ pub struct SearchQuery {
     pub hits_per_page: Option<usize>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToRetrieve>)]
     pub attributes_to_retrieve: Option<BTreeSet<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchRetrieveVectors>)]
+    pub retrieve_vectors: bool,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToCrop>)]
     pub attributes_to_crop: Option<Vec<String>>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchCropLength>, default = DEFAULT_CROP_LENGTH())]
@@ -75,6 +78,8 @@ pub struct SearchQuery {
     pub filter: Option<Value>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchSort>)]
     pub sort: Option<Vec<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchDistinct>)]
+    pub distinct: Option<String>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchFacets>)]
     pub facets: Option<Vec<String>>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchHighlightPreTag>, default = DEFAULT_HIGHLIGHT_PRE_TAG())]
@@ -87,6 +92,44 @@ pub struct SearchQuery {
     pub matching_strategy: MatchingStrategy,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToSearchOn>, default)]
     pub attributes_to_search_on: Option<Vec<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchRankingScoreThreshold>, default)]
+    pub ranking_score_threshold: Option<RankingScoreThreshold>,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Deserr)]
+#[deserr(try_from(f64) = TryFrom::try_from -> InvalidSearchRankingScoreThreshold)]
+pub struct RankingScoreThreshold(f64);
+
+impl std::convert::TryFrom<f64> for RankingScoreThreshold {
+    type Error = InvalidSearchRankingScoreThreshold;
+
+    fn try_from(f: f64) -> Result<Self, Self::Error> {
+        // the suggested "fix" is: `!(0.0..=1.0).contains(&f)`` which is allegedly less readable
+        #[allow(clippy::manual_range_contains)]
+        if f > 1.0 || f < 0.0 {
+            Err(InvalidSearchRankingScoreThreshold)
+        } else {
+            Ok(RankingScoreThreshold(f))
+        }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Deserr)]
+#[deserr(try_from(f64) = TryFrom::try_from -> InvalidSimilarRankingScoreThreshold)]
+pub struct RankingScoreThresholdSimilar(f64);
+
+impl std::convert::TryFrom<f64> for RankingScoreThresholdSimilar {
+    type Error = InvalidSimilarRankingScoreThreshold;
+
+    fn try_from(f: f64) -> Result<Self, Self::Error> {
+        // the suggested "fix" is: `!(0.0..=1.0).contains(&f)`` which is allegedly less readable
+        #[allow(clippy::manual_range_contains)]
+        if f > 1.0 || f < 0.0 {
+            Err(InvalidSimilarRankingScoreThreshold)
+        } else {
+            Ok(Self(f))
+        }
+    }
 }
 
 // Since this structure is logged A LOT we're going to reduce the number of things it logs to the bare minimum.
@@ -103,6 +146,7 @@ impl fmt::Debug for SearchQuery {
             page,
             hits_per_page,
             attributes_to_retrieve,
+            retrieve_vectors,
             attributes_to_crop,
             crop_length,
             attributes_to_highlight,
@@ -111,12 +155,14 @@ impl fmt::Debug for SearchQuery {
             show_ranking_score_details,
             filter,
             sort,
+            distinct,
             facets,
             highlight_pre_tag,
             highlight_post_tag,
             crop_marker,
             matching_strategy,
             attributes_to_search_on,
+            ranking_score_threshold,
         } = self;
 
         let mut debug = f.debug_struct("SearchQuery");
@@ -134,6 +180,9 @@ impl fmt::Debug for SearchQuery {
         if let Some(q) = q {
             debug.field("q", &q);
         }
+        if *retrieve_vectors {
+            debug.field("retrieve_vectors", &retrieve_vectors);
+        }
         if let Some(v) = vector {
             if v.len() < 10 {
                 debug.field("vector", &v);
@@ -156,6 +205,9 @@ impl fmt::Debug for SearchQuery {
         if let Some(sort) = sort {
             debug.field("sort", &sort);
         }
+        if let Some(distinct) = distinct {
+            debug.field("distinct", &distinct);
+        }
         if let Some(facets) = facets {
             debug.field("facets", &facets);
         }
@@ -188,6 +240,9 @@ impl fmt::Debug for SearchQuery {
         debug.field("highlight_pre_tag", &highlight_pre_tag);
         debug.field("highlight_post_tag", &highlight_post_tag);
         debug.field("crop_marker", &crop_marker);
+        if let Some(ranking_score_threshold) = ranking_score_threshold {
+            debug.field("ranking_score_threshold", &ranking_score_threshold);
+        }
 
         debug.finish()
     }
@@ -231,7 +286,7 @@ impl SearchKind {
         Ok(Self::Hybrid { embedder_name, embedder, semantic_ratio })
     }
 
-    fn embedder(
+    pub(crate) fn embedder(
         index_scheduler: &index_scheduler::IndexScheduler,
         index: &Index,
         embedder_name: Option<&str>,
@@ -328,6 +383,8 @@ pub struct SearchQueryWithIndex {
     pub hits_per_page: Option<usize>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToRetrieve>)]
     pub attributes_to_retrieve: Option<BTreeSet<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchRetrieveVectors>)]
+    pub retrieve_vectors: bool,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToCrop>)]
     pub attributes_to_crop: Option<Vec<String>>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchCropLength>, default = DEFAULT_CROP_LENGTH())]
@@ -344,6 +401,8 @@ pub struct SearchQueryWithIndex {
     pub filter: Option<Value>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchSort>)]
     pub sort: Option<Vec<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchDistinct>)]
+    pub distinct: Option<String>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchFacets>)]
     pub facets: Option<Vec<String>>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchHighlightPreTag>, default = DEFAULT_HIGHLIGHT_PRE_TAG())]
@@ -356,6 +415,8 @@ pub struct SearchQueryWithIndex {
     pub matching_strategy: MatchingStrategy,
     #[deserr(default, error = DeserrJsonError<InvalidSearchAttributesToSearchOn>, default)]
     pub attributes_to_search_on: Option<Vec<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSearchRankingScoreThreshold>, default)]
+    pub ranking_score_threshold: Option<RankingScoreThreshold>,
 }
 
 impl SearchQueryWithIndex {
@@ -369,6 +430,7 @@ impl SearchQueryWithIndex {
             page,
             hits_per_page,
             attributes_to_retrieve,
+            retrieve_vectors,
             attributes_to_crop,
             crop_length,
             attributes_to_highlight,
@@ -377,6 +439,7 @@ impl SearchQueryWithIndex {
             show_matches_position,
             filter,
             sort,
+            distinct,
             facets,
             highlight_pre_tag,
             highlight_post_tag,
@@ -384,6 +447,7 @@ impl SearchQueryWithIndex {
             matching_strategy,
             attributes_to_search_on,
             hybrid,
+            ranking_score_threshold,
         } = self;
         (
             index_uid,
@@ -395,6 +459,7 @@ impl SearchQueryWithIndex {
                 page,
                 hits_per_page,
                 attributes_to_retrieve,
+                retrieve_vectors,
                 attributes_to_crop,
                 crop_length,
                 attributes_to_highlight,
@@ -403,6 +468,7 @@ impl SearchQueryWithIndex {
                 show_matches_position,
                 filter,
                 sort,
+                distinct,
                 facets,
                 highlight_pre_tag,
                 highlight_post_tag,
@@ -410,6 +476,7 @@ impl SearchQueryWithIndex {
                 matching_strategy,
                 attributes_to_search_on,
                 hybrid,
+                ranking_score_threshold,
                 // do not use ..Default::default() here,
                 // rather add any missing field from `SearchQuery` to `SearchQueryWithIndex`
             },
@@ -417,6 +484,63 @@ impl SearchQueryWithIndex {
     }
 }
 
+#[derive(Debug, Clone, PartialEq, Deserr)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
+pub struct SimilarQuery {
+    #[deserr(error = DeserrJsonError<InvalidSimilarId>)]
+    pub id: ExternalDocumentId,
+    #[deserr(default = DEFAULT_SEARCH_OFFSET(), error = DeserrJsonError<InvalidSimilarOffset>)]
+    pub offset: usize,
+    #[deserr(default = DEFAULT_SEARCH_LIMIT(), error = DeserrJsonError<InvalidSimilarLimit>)]
+    pub limit: usize,
+    #[deserr(default, error = DeserrJsonError<InvalidSimilarFilter>)]
+    pub filter: Option<Value>,
+    #[deserr(default, error = DeserrJsonError<InvalidEmbedder>, default)]
+    pub embedder: Option<String>,
+    #[deserr(default, error = DeserrJsonError<InvalidSimilarAttributesToRetrieve>)]
+    pub attributes_to_retrieve: Option<BTreeSet<String>>,
+    #[deserr(default, error = DeserrJsonError<InvalidSimilarRetrieveVectors>)]
+    pub retrieve_vectors: bool,
+    #[deserr(default, error = DeserrJsonError<InvalidSimilarShowRankingScore>, default)]
+    pub show_ranking_score: bool,
+    #[deserr(default, error = DeserrJsonError<InvalidSimilarShowRankingScoreDetails>, default)]
+    pub show_ranking_score_details: bool,
+    #[deserr(default, error = DeserrJsonError<InvalidSimilarRankingScoreThreshold>, default)]
+    pub ranking_score_threshold: Option<RankingScoreThresholdSimilar>,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserr)]
+#[deserr(try_from(Value) = TryFrom::try_from -> InvalidSimilarId)]
+pub struct ExternalDocumentId(String);
+
+impl AsRef<str> for ExternalDocumentId {
+    fn as_ref(&self) -> &str {
+        &self.0
+    }
+}
+
+impl ExternalDocumentId {
+    pub fn into_inner(self) -> String {
+        self.0
+    }
+}
+
+impl TryFrom<String> for ExternalDocumentId {
+    type Error = InvalidSimilarId;
+
+    fn try_from(value: String) -> Result<Self, Self::Error> {
+        serde_json::Value::String(value).try_into()
+    }
+}
+
+impl TryFrom<Value> for ExternalDocumentId {
+    type Error = InvalidSimilarId;
+
+    fn try_from(value: Value) -> Result<Self, Self::Error> {
+        Ok(Self(milli::documents::validate_document_id_value(value).map_err(|_| InvalidSimilarId)?))
+    }
+}
+
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr)]
 #[deserr(rename_all = camelCase)]
 pub enum MatchingStrategy {
@@ -424,6 +548,8 @@ pub enum MatchingStrategy {
     Last,
     /// All query words are mandatory
     All,
+    /// Remove query words from the most frequent to the least
+    Frequency,
 }
 
 impl Default for MatchingStrategy {
@@ -437,6 +563,7 @@ impl From<MatchingStrategy> for TermsMatchingStrategy {
         match other {
             MatchingStrategy::Last => Self::Last,
             MatchingStrategy::All => Self::All,
+            MatchingStrategy::Frequency => Self::Frequency,
         }
     }
 }
@@ -538,6 +665,16 @@ impl fmt::Debug for SearchResult {
     }
 }
 
+#[derive(Serialize, Debug, Clone, PartialEq)]
+#[serde(rename_all = "camelCase")]
+pub struct SimilarResult {
+    pub hits: Vec<SearchHit>,
+    pub id: String,
+    pub processing_time_ms: u128,
+    #[serde(flatten)]
+    pub hits_info: HitsInfo,
+}
+
 #[derive(Serialize, Debug, Clone, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct SearchResultWithIndex {
@@ -570,8 +707,8 @@ pub struct FacetSearchResult {
 }
 
 /// Incorporate search rules in search query
-pub fn add_search_rules(query: &mut SearchQuery, rules: IndexSearchRules) {
-    query.filter = match (query.filter.take(), rules.filter) {
+pub fn add_search_rules(filter: &mut Option<Value>, rules: IndexSearchRules) {
+    *filter = match (filter.take(), rules.filter) {
         (None, rules_filter) => rules_filter,
         (filter, None) => filter,
         (Some(filter), Some(rules_filter)) => {
@@ -598,6 +735,13 @@ fn prepare_search<'t>(
 ) -> Result<(milli::Search<'t>, bool, usize, usize), MeilisearchHttpError> {
     let mut search = index.search(rtxn);
     search.time_budget(time_budget);
+    if let Some(ranking_score_threshold) = query.ranking_score_threshold {
+        search.ranking_score_threshold(ranking_score_threshold.0);
+    }
+
+    if let Some(distinct) = &query.distinct {
+        search.distinct(distinct.clone());
+    }
 
     match search_kind {
         SearchKind::KeywordOnly => {
@@ -608,10 +752,15 @@ fn prepare_search<'t>(
         SearchKind::SemanticOnly { embedder_name, embedder } => {
             let vector = match query.vector.clone() {
                 Some(vector) => vector,
-                None => embedder
-                    .embed_one(query.q.clone().unwrap())
-                    .map_err(milli::vector::Error::from)
-                    .map_err(milli::Error::from)?,
+                None => {
+                    let span = tracing::trace_span!(target: "search::vector", "embed_one");
+                    let _entered = span.enter();
+
+                    embedder
+                        .embed_one(query.q.clone().unwrap())
+                        .map_err(milli::vector::Error::from)
+                        .map_err(milli::Error::from)?
+                }
             };
 
             search.semantic(embedder_name.clone(), embedder.clone(), Some(vector));
@@ -639,11 +788,16 @@ fn prepare_search<'t>(
         .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS);
 
     search.exhaustive_number_hits(is_finite_pagination);
-    search.scoring_strategy(if query.show_ranking_score || query.show_ranking_score_details {
-        ScoringStrategy::Detailed
-    } else {
-        ScoringStrategy::Skip
-    });
+    search.scoring_strategy(
+        if query.show_ranking_score
+            || query.show_ranking_score_details
+            || query.ranking_score_threshold.is_some()
+        {
+            ScoringStrategy::Detailed
+        } else {
+            ScoringStrategy::Skip
+        },
+    );
 
     // compute the offset on the limit depending on the pagination mode.
     let (offset, limit) = if is_finite_pagination {
@@ -688,6 +842,7 @@ pub fn perform_search(
     index: &Index,
     query: SearchQuery,
     search_kind: SearchKind,
+    retrieve_vectors: RetrieveVectors,
 ) -> Result<SearchResult, MeilisearchHttpError> {
     let before_search = Instant::now();
     let rtxn = index.read_txn()?;
@@ -719,131 +874,57 @@ pub fn perform_search(
         SearchKind::Hybrid { semantic_ratio, .. } => search.execute_hybrid(*semantic_ratio)?,
     };
 
-    let fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
+    let SearchQuery {
+        q,
+        limit,
+        page,
+        hits_per_page,
+        attributes_to_retrieve,
+        // use the enum passed as parameter
+        retrieve_vectors: _,
+        attributes_to_crop,
+        crop_length,
+        attributes_to_highlight,
+        show_matches_position,
+        show_ranking_score,
+        show_ranking_score_details,
+        sort,
+        facets,
+        highlight_pre_tag,
+        highlight_post_tag,
+        crop_marker,
+        // already used in prepare_search
+        vector: _,
+        hybrid: _,
+        offset: _,
+        ranking_score_threshold: _,
+        matching_strategy: _,
+        attributes_to_search_on: _,
+        filter: _,
+        distinct: _,
+    } = query;
 
-    let displayed_ids = index
-        .displayed_fields_ids(&rtxn)?
-        .map(|fields| fields.into_iter().collect::<BTreeSet<_>>())
-        .unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect());
-
-    let fids = |attrs: &BTreeSet<String>| {
-        let mut ids = BTreeSet::new();
-        for attr in attrs {
-            if attr == "*" {
-                ids = displayed_ids.clone();
-                break;
-            }
-
-            if let Some(id) = fields_ids_map.id(attr) {
-                ids.insert(id);
-            }
-        }
-        ids
+    let format = AttributesFormat {
+        attributes_to_retrieve,
+        retrieve_vectors,
+        attributes_to_highlight,
+        attributes_to_crop,
+        crop_length,
+        crop_marker,
+        highlight_pre_tag,
+        highlight_post_tag,
+        show_matches_position,
+        sort,
+        show_ranking_score,
+        show_ranking_score_details,
     };
 
-    // The attributes to retrieve are the ones explicitly marked as to retrieve (all by default),
-    // but these attributes must be also be present
-    // - in the fields_ids_map
-    // - in the displayed attributes
-    let to_retrieve_ids: BTreeSet<_> = query
-        .attributes_to_retrieve
-        .as_ref()
-        .map(fids)
-        .unwrap_or_else(|| displayed_ids.clone())
-        .intersection(&displayed_ids)
-        .cloned()
-        .collect();
-
-    let attr_to_highlight = query.attributes_to_highlight.unwrap_or_default();
-
-    let attr_to_crop = query.attributes_to_crop.unwrap_or_default();
-
-    // Attributes in `formatted_options` correspond to the attributes that will be in `_formatted`
-    // These attributes are:
-    // - the attributes asked to be highlighted or cropped (with `attributesToCrop` or `attributesToHighlight`)
-    // - the attributes asked to be retrieved: these attributes will not be highlighted/cropped
-    // But these attributes must be also present in displayed attributes
-    let formatted_options = compute_formatted_options(
-        &attr_to_highlight,
-        &attr_to_crop,
-        query.crop_length,
-        &to_retrieve_ids,
-        &fields_ids_map,
-        &displayed_ids,
-    );
-
-    let mut tokenizer_builder = TokenizerBuilder::default();
-    tokenizer_builder.create_char_map(true);
-
-    let script_lang_map = index.script_language(&rtxn)?;
-    if !script_lang_map.is_empty() {
-        tokenizer_builder.allow_list(&script_lang_map);
-    }
-
-    let separators = index.allowed_separators(&rtxn)?;
-    let separators: Option<Vec<_>> =
-        separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
-    if let Some(ref separators) = separators {
-        tokenizer_builder.separators(separators);
-    }
-
-    let dictionary = index.dictionary(&rtxn)?;
-    let dictionary: Option<Vec<_>> =
-        dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
-    if let Some(ref dictionary) = dictionary {
-        tokenizer_builder.words_dict(dictionary);
-    }
-
-    let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer_builder.build());
-    formatter_builder.crop_marker(query.crop_marker);
-    formatter_builder.highlight_prefix(query.highlight_pre_tag);
-    formatter_builder.highlight_suffix(query.highlight_post_tag);
-
-    let mut documents = Vec::new();
-    let documents_iter = index.documents(&rtxn, documents_ids)?;
-
-    for ((_id, obkv), score) in documents_iter.into_iter().zip(document_scores.into_iter()) {
-        // First generate a document with all the displayed fields
-        let displayed_document = make_document(&displayed_ids, &fields_ids_map, obkv)?;
-
-        // select the attributes to retrieve
-        let attributes_to_retrieve = to_retrieve_ids
-            .iter()
-            .map(|&fid| fields_ids_map.name(fid).expect("Missing field name"));
-        let mut document =
-            permissive_json_pointer::select_values(&displayed_document, attributes_to_retrieve);
-
-        let (matches_position, formatted) = format_fields(
-            &displayed_document,
-            &fields_ids_map,
-            &formatter_builder,
-            &formatted_options,
-            query.show_matches_position,
-            &displayed_ids,
-        )?;
-
-        if let Some(sort) = query.sort.as_ref() {
-            insert_geo_distance(sort, &mut document);
-        }
-
-        let ranking_score =
-            query.show_ranking_score.then(|| ScoreDetails::global_score(score.iter()));
-        let ranking_score_details =
-            query.show_ranking_score_details.then(|| ScoreDetails::to_json_map(score.iter()));
-
-        let hit = SearchHit {
-            document,
-            formatted,
-            matches_position,
-            ranking_score_details,
-            ranking_score,
-        };
-        documents.push(hit);
-    }
+    let documents =
+        make_hits(index, &rtxn, format, matching_words, documents_ids, document_scores)?;
 
     let number_of_hits = min(candidates.len() as usize, max_total_hits);
     let hits_info = if is_finite_pagination {
-        let hits_per_page = query.hits_per_page.unwrap_or_else(DEFAULT_SEARCH_LIMIT);
+        let hits_per_page = hits_per_page.unwrap_or_else(DEFAULT_SEARCH_LIMIT);
         // If hit_per_page is 0, then pages can't be computed and so we respond 0.
         let total_pages = (number_of_hits + hits_per_page.saturating_sub(1))
             .checked_div(hits_per_page)
@@ -851,15 +932,15 @@ pub fn perform_search(
 
         HitsInfo::Pagination {
             hits_per_page,
-            page: query.page.unwrap_or(1),
+            page: page.unwrap_or(1),
             total_pages,
             total_hits: number_of_hits,
         }
     } else {
-        HitsInfo::OffsetLimit { limit: query.limit, offset, estimated_total_hits: number_of_hits }
+        HitsInfo::OffsetLimit { limit, offset, estimated_total_hits: number_of_hits }
     };
 
-    let (facet_distribution, facet_stats) = match query.facets {
+    let (facet_distribution, facet_stats) = match facets {
         Some(ref fields) => {
             let mut facet_distribution = index.facets_distribution(&rtxn);
 
@@ -896,7 +977,7 @@ pub fn perform_search(
     let result = SearchResult {
         hits: documents,
         hits_info,
-        query: query.q.unwrap_or_default(),
+        query: q.unwrap_or_default(),
         processing_time_ms: before_search.elapsed().as_millis(),
         facet_distribution,
         facet_stats,
@@ -907,6 +988,222 @@ pub fn perform_search(
     Ok(result)
 }
 
+struct AttributesFormat {
+    attributes_to_retrieve: Option<BTreeSet<String>>,
+    retrieve_vectors: RetrieveVectors,
+    attributes_to_highlight: Option<HashSet<String>>,
+    attributes_to_crop: Option<Vec<String>>,
+    crop_length: usize,
+    crop_marker: String,
+    highlight_pre_tag: String,
+    highlight_post_tag: String,
+    show_matches_position: bool,
+    sort: Option<Vec<String>>,
+    show_ranking_score: bool,
+    show_ranking_score_details: bool,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum RetrieveVectors {
+    /// Do not touch the `_vectors` field
+    ///
+    /// this is the behavior when the vectorStore feature is disabled
+    Ignore,
+    /// Remove the `_vectors` field
+    ///
+    /// this is the behavior when the vectorStore feature is enabled, and `retrieveVectors` is `false`
+    Hide,
+    /// Retrieve vectors from the DB and merge them into the `_vectors` field
+    ///
+    /// this is the behavior when the vectorStore feature is enabled, and `retrieveVectors` is `true`
+    Retrieve,
+}
+
+impl RetrieveVectors {
+    pub fn new(
+        retrieve_vector: bool,
+        features: index_scheduler::RoFeatures,
+    ) -> Result<Self, index_scheduler::Error> {
+        match (retrieve_vector, features.check_vector("Passing `retrieveVectors` as a parameter")) {
+            (true, Ok(())) => Ok(Self::Retrieve),
+            (true, Err(error)) => Err(error),
+            (false, Ok(())) => Ok(Self::Hide),
+            (false, Err(_)) => Ok(Self::Ignore),
+        }
+    }
+}
+
+fn make_hits(
+    index: &Index,
+    rtxn: &RoTxn<'_>,
+    format: AttributesFormat,
+    matching_words: milli::MatchingWords,
+    documents_ids: Vec<u32>,
+    document_scores: Vec<Vec<ScoreDetails>>,
+) -> Result<Vec<SearchHit>, MeilisearchHttpError> {
+    let fields_ids_map = index.fields_ids_map(rtxn).unwrap();
+    let displayed_ids =
+        index.displayed_fields_ids(rtxn)?.map(|fields| fields.into_iter().collect::<BTreeSet<_>>());
+
+    let vectors_fid = fields_ids_map.id(milli::vector::parsed_vectors::RESERVED_VECTORS_FIELD_NAME);
+
+    let vectors_is_hidden = match (&displayed_ids, vectors_fid) {
+        // displayed_ids is a wildcard, so `_vectors` can be displayed regardless of its fid
+        (None, _) => false,
+        // displayed_ids is a finite list, and `_vectors` cannot be part of it because it is not an existing field
+        (Some(_), None) => true,
+        // displayed_ids is a finit list, so hide if `_vectors` is not part of it
+        (Some(map), Some(vectors_fid)) => map.contains(&vectors_fid),
+    };
+
+    let retrieve_vectors = if let RetrieveVectors::Retrieve = format.retrieve_vectors {
+        if vectors_is_hidden {
+            RetrieveVectors::Hide
+        } else {
+            RetrieveVectors::Retrieve
+        }
+    } else {
+        format.retrieve_vectors
+    };
+
+    let displayed_ids =
+        displayed_ids.unwrap_or_else(|| fields_ids_map.iter().map(|(id, _)| id).collect());
+    let fids = |attrs: &BTreeSet<String>| {
+        let mut ids = BTreeSet::new();
+        for attr in attrs {
+            if attr == "*" {
+                ids.clone_from(&displayed_ids);
+                break;
+            }
+
+            if let Some(id) = fields_ids_map.id(attr) {
+                ids.insert(id);
+            }
+        }
+        ids
+    };
+    let to_retrieve_ids: BTreeSet<_> = format
+        .attributes_to_retrieve
+        .as_ref()
+        .map(fids)
+        .unwrap_or_else(|| displayed_ids.clone())
+        .intersection(&displayed_ids)
+        .cloned()
+        .collect();
+
+    let attr_to_highlight = format.attributes_to_highlight.unwrap_or_default();
+    let attr_to_crop = format.attributes_to_crop.unwrap_or_default();
+    let formatted_options = compute_formatted_options(
+        &attr_to_highlight,
+        &attr_to_crop,
+        format.crop_length,
+        &to_retrieve_ids,
+        &fields_ids_map,
+        &displayed_ids,
+    );
+    let mut tokenizer_builder = TokenizerBuilder::default();
+    tokenizer_builder.create_char_map(true);
+    let script_lang_map = index.script_language(rtxn)?;
+    if !script_lang_map.is_empty() {
+        tokenizer_builder.allow_list(&script_lang_map);
+    }
+    let separators = index.allowed_separators(rtxn)?;
+    let separators: Option<Vec<_>> =
+        separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
+    if let Some(ref separators) = separators {
+        tokenizer_builder.separators(separators);
+    }
+    let dictionary = index.dictionary(rtxn)?;
+    let dictionary: Option<Vec<_>> =
+        dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
+    if let Some(ref dictionary) = dictionary {
+        tokenizer_builder.words_dict(dictionary);
+    }
+    let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer_builder.build());
+    formatter_builder.crop_marker(format.crop_marker);
+    formatter_builder.highlight_prefix(format.highlight_pre_tag);
+    formatter_builder.highlight_suffix(format.highlight_post_tag);
+    let decompression_dictionary = index.document_decompression_dictionary(rtxn)?;
+    let mut buffer = Vec::new();
+    let mut documents = Vec::new();
+    let embedding_configs = index.embedding_configs(rtxn)?;
+    let documents_iter = index.compressed_documents(rtxn, documents_ids)?;
+    for ((id, compressed), score) in documents_iter.into_iter().zip(document_scores.into_iter()) {
+        let obkv = compressed
+            .decompress_with_optional_dictionary(&mut buffer, decompression_dictionary.as_ref())
+            // TODO use a better error?
+            .map_err(|e| MeilisearchHttpError::HeedError(e.into()))?;
+        // First generate a document with all the displayed fields
+        let displayed_document = make_document(&displayed_ids, &fields_ids_map, obkv)?;
+
+        let add_vectors_fid =
+            vectors_fid.filter(|_fid| retrieve_vectors == RetrieveVectors::Retrieve);
+
+        // select the attributes to retrieve
+        let attributes_to_retrieve = to_retrieve_ids
+            .iter()
+            // skip the vectors_fid if RetrieveVectors::Hide
+            .filter(|fid| match vectors_fid {
+                Some(vectors_fid) => {
+                    !(retrieve_vectors == RetrieveVectors::Hide && **fid == vectors_fid)
+                }
+                None => true,
+            })
+            // need to retrieve the existing `_vectors` field if the `RetrieveVectors::Retrieve`
+            .chain(add_vectors_fid.iter())
+            .map(|&fid| fields_ids_map.name(fid).expect("Missing field name"));
+        let mut document =
+            permissive_json_pointer::select_values(&displayed_document, attributes_to_retrieve);
+
+        if retrieve_vectors == RetrieveVectors::Retrieve {
+            // Clippy is wrong
+            #[allow(clippy::manual_unwrap_or_default)]
+            let mut vectors = match document.remove("_vectors") {
+                Some(Value::Object(map)) => map,
+                _ => Default::default(),
+            };
+            for (name, vector) in index.embeddings(rtxn, id)? {
+                let user_provided = embedding_configs
+                    .iter()
+                    .find(|conf| conf.name == name)
+                    .is_some_and(|conf| conf.user_provided.contains(id));
+                let embeddings =
+                    ExplicitVectors { embeddings: Some(vector.into()), regenerate: !user_provided };
+                vectors.insert(name, serde_json::to_value(embeddings)?);
+            }
+            document.insert("_vectors".into(), vectors.into());
+        }
+
+        let (matches_position, formatted) = format_fields(
+            &displayed_document,
+            &fields_ids_map,
+            &formatter_builder,
+            &formatted_options,
+            format.show_matches_position,
+            &displayed_ids,
+        )?;
+
+        if let Some(sort) = format.sort.as_ref() {
+            insert_geo_distance(sort, &mut document);
+        }
+
+        let ranking_score =
+            format.show_ranking_score.then(|| ScoreDetails::global_score(score.iter()));
+        let ranking_score_details =
+            format.show_ranking_score_details.then(|| ScoreDetails::to_json_map(score.iter()));
+
+        let hit = SearchHit {
+            document,
+            formatted,
+            matches_position,
+            ranking_score_details,
+            ranking_score,
+        };
+        documents.push(hit);
+    }
+    Ok(documents)
+}
+
 pub fn perform_facet_search(
     index: &Index,
     search_query: SearchQuery,
@@ -941,6 +1238,103 @@ pub fn perform_facet_search(
     })
 }
 
+pub fn perform_similar(
+    index: &Index,
+    query: SimilarQuery,
+    embedder_name: String,
+    embedder: Arc<Embedder>,
+    retrieve_vectors: RetrieveVectors,
+) -> Result<SimilarResult, ResponseError> {
+    let before_search = Instant::now();
+    let rtxn = index.read_txn()?;
+
+    let SimilarQuery {
+        id,
+        offset,
+        limit,
+        filter: _,
+        embedder: _,
+        attributes_to_retrieve,
+        retrieve_vectors: _,
+        show_ranking_score,
+        show_ranking_score_details,
+        ranking_score_threshold,
+    } = query;
+
+    // using let-else rather than `?` so that the borrow checker identifies we're always returning here,
+    // preventing a use-after-move
+    let Some(internal_id) = index.external_documents_ids().get(&rtxn, &id)? else {
+        return Err(ResponseError::from_msg(
+            MeilisearchHttpError::DocumentNotFound(id.into_inner()).to_string(),
+            Code::NotFoundSimilarId,
+        ));
+    };
+
+    let mut similar =
+        milli::Similar::new(internal_id, offset, limit, index, &rtxn, embedder_name, embedder);
+
+    if let Some(ref filter) = query.filter {
+        if let Some(facets) = parse_filter(filter)
+            // inject InvalidSimilarFilter code
+            .map_err(|e| ResponseError::from_msg(e.to_string(), Code::InvalidSimilarFilter))?
+        {
+            similar.filter(facets);
+        }
+    }
+
+    if let Some(ranking_score_threshold) = ranking_score_threshold {
+        similar.ranking_score_threshold(ranking_score_threshold.0);
+    }
+
+    let milli::SearchResult {
+        documents_ids,
+        matching_words: _,
+        candidates,
+        document_scores,
+        degraded: _,
+        used_negative_operator: _,
+    } = similar.execute().map_err(|err| match err {
+        milli::Error::UserError(milli::UserError::InvalidFilter(_)) => {
+            ResponseError::from_msg(err.to_string(), Code::InvalidSimilarFilter)
+        }
+        err => err.into(),
+    })?;
+
+    let format = AttributesFormat {
+        attributes_to_retrieve,
+        retrieve_vectors,
+        attributes_to_highlight: None,
+        attributes_to_crop: None,
+        crop_length: DEFAULT_CROP_LENGTH(),
+        crop_marker: DEFAULT_CROP_MARKER(),
+        highlight_pre_tag: DEFAULT_HIGHLIGHT_PRE_TAG(),
+        highlight_post_tag: DEFAULT_HIGHLIGHT_POST_TAG(),
+        show_matches_position: false,
+        sort: None,
+        show_ranking_score,
+        show_ranking_score_details,
+    };
+
+    let hits = make_hits(index, &rtxn, format, Default::default(), documents_ids, document_scores)?;
+
+    let max_total_hits = index
+        .pagination_max_total_hits(&rtxn)
+        .map_err(milli::Error::from)?
+        .map(|x| x as usize)
+        .unwrap_or(DEFAULT_PAGINATION_MAX_TOTAL_HITS);
+
+    let number_of_hits = min(candidates.len() as usize, max_total_hits);
+    let hits_info = HitsInfo::OffsetLimit { limit, offset, estimated_total_hits: number_of_hits };
+
+    let result = SimilarResult {
+        hits,
+        hits_info,
+        id: id.into_inner(),
+        processing_time_ms: before_search.elapsed().as_millis(),
+    };
+    Ok(result)
+}
+
 fn insert_geo_distance(sorts: &[String], document: &mut Document) {
     lazy_static::lazy_static! {
         static ref GEO_REGEX: Regex =
@@ -950,13 +1344,23 @@ fn insert_geo_distance(sorts: &[String], document: &mut Document) {
         // TODO: TAMO: milli encountered an internal error, what do we want to do?
         let base = [capture_group[1].parse().unwrap(), capture_group[2].parse().unwrap()];
         let geo_point = &document.get("_geo").unwrap_or(&json!(null));
-        if let Some((lat, lng)) = geo_point["lat"].as_f64().zip(geo_point["lng"].as_f64()) {
+        if let Some((lat, lng)) =
+            extract_geo_value(&geo_point["lat"]).zip(extract_geo_value(&geo_point["lng"]))
+        {
             let distance = milli::distance_between_two_points(&base, &[lat, lng]);
             document.insert("_geoDistance".to_string(), json!(distance.round() as usize));
         }
     }
 }
 
+fn extract_geo_value(value: &Value) -> Option<f64> {
+    match value {
+        Value::Number(n) => n.as_f64(),
+        Value::String(s) => s.parse().ok(),
+        _ => None,
+    }
+}
+
 fn compute_formatted_options(
     attr_to_highlight: &HashSet<String>,
     attr_to_crop: &[String],
@@ -1330,4 +1734,54 @@ mod test {
         insert_geo_distance(sorters, &mut document);
         assert_eq!(document.get("_geoDistance"), None);
     }
+
+    #[test]
+    fn test_insert_geo_distance_with_coords_as_string() {
+        let value: Document = serde_json::from_str(
+            r#"{
+              "_geo": {
+                "lat": "50",
+                "lng": 3
+              }
+            }"#,
+        )
+        .unwrap();
+
+        let sorters = &["_geoPoint(50,3):desc".to_string()];
+        let mut document = value.clone();
+        insert_geo_distance(sorters, &mut document);
+        assert_eq!(document.get("_geoDistance"), Some(&json!(0)));
+
+        let value: Document = serde_json::from_str(
+            r#"{
+              "_geo": {
+                "lat": "50",
+                "lng": "3"
+              },
+              "id": "1"
+            }"#,
+        )
+        .unwrap();
+
+        let sorters = &["_geoPoint(50,3):desc".to_string()];
+        let mut document = value.clone();
+        insert_geo_distance(sorters, &mut document);
+        assert_eq!(document.get("_geoDistance"), Some(&json!(0)));
+
+        let value: Document = serde_json::from_str(
+            r#"{
+              "_geo": {
+                "lat": 50,
+                "lng": "3"
+              },
+              "id": "1"
+            }"#,
+        )
+        .unwrap();
+
+        let sorters = &["_geoPoint(50,3):desc".to_string()];
+        let mut document = value.clone();
+        insert_geo_distance(sorters, &mut document);
+        assert_eq!(document.get("_geoDistance"), Some(&json!(0)));
+    }
 }

meilisearch/src/search_queue.rs

@@ -40,8 +40,9 @@ pub struct Permit {
 
 impl Drop for Permit {
     fn drop(&mut self) {
+        let sender = self.sender.clone();
         // if the channel is closed then the whole instance is down
-        let _ = futures::executor::block_on(self.sender.send(()));
+        std::mem::drop(tokio::spawn(async move { sender.send(()).await }));
     }
 }
 
@@ -85,8 +86,13 @@ impl SearchQueue {
                 },
 
                 search_request = receive_new_searches.recv() => {
-                    // this unwrap is safe because we're sure the `SearchQueue` still lives somewhere in actix-web
-                    let search_request = search_request.unwrap();
+                    let search_request = match search_request {
+                        Some(search_request) => search_request,
+                        // This should never happen while actix-web is running, but it's not a reason to crash
+                        // and it can generate a lot of noise in the tests.
+                        None => continue,
+                    };
+
                     if searches_running < usize::from(parallelism) && queue.is_empty() {
                         searches_running += 1;
                         // if the search requests die it's not a hard error on our side

meilisearch/tests/auth/authorization.rs

@@ -78,7 +78,7 @@ pub static ALL_ACTIONS: Lazy<HashSet<&'static str>> = Lazy::new(|| {
 });
 
 static INVALID_RESPONSE: Lazy<Value> = Lazy::new(|| {
-    json!({"message": "The provided API key is invalid.",
+    json!({"message": null,
         "code": "invalid_api_key",
         "type": "auth",
         "link": "https://docs.meilisearch.com/errors#invalid_api_key"
@@ -119,7 +119,8 @@ async fn error_access_expired_key() {
     thread::sleep(time::Duration::new(1, 0));
 
     for (method, route) in AUTHORIZATIONS.keys() {
-        let (response, code) = server.dummy_request(method, route).await;
+        let (mut response, code) = server.dummy_request(method, route).await;
+        response["message"] = serde_json::json!(null);
 
         assert_eq!(response, INVALID_RESPONSE.clone(), "on route: {:?} - {:?}", method, route);
         assert_eq!(403, code, "{:?}", &response);
@@ -149,7 +150,8 @@ async fn error_access_unauthorized_index() {
         // filter `products` index routes
         .filter(|(_, route)| route.starts_with("/indexes/products"))
     {
-        let (response, code) = server.dummy_request(method, route).await;
+        let (mut response, code) = server.dummy_request(method, route).await;
+        response["message"] = serde_json::json!(null);
 
         assert_eq!(response, INVALID_RESPONSE.clone(), "on route: {:?} - {:?}", method, route);
         assert_eq!(403, code, "{:?}", &response);
@@ -176,7 +178,8 @@ async fn error_access_unauthorized_action() {
 
         let key = response["key"].as_str().unwrap();
         server.use_api_key(key);
-        let (response, code) = server.dummy_request(method, route).await;
+        let (mut response, code) = server.dummy_request(method, route).await;
+        response["message"] = serde_json::json!(null);
 
         assert_eq!(response, INVALID_RESPONSE.clone(), "on route: {:?} - {:?}", method, route);
         assert_eq!(403, code, "{:?}", &response);
@@ -280,7 +283,7 @@ async fn access_authorized_no_index_restriction() {
                 route,
                 action
             );
-            assert_ne!(code, 403);
+            assert_ne!(code, 403, "on route: {:?} - {:?} with action: {:?}", method, route, action);
         }
     }
 }

meilisearch/tests/auth/errors.rs

@@ -1,7 +1,10 @@
+use actix_web::http::StatusCode;
+use actix_web::test;
+use jsonwebtoken::{EncodingKey, Header};
 use meili_snap::*;
 use uuid::Uuid;
 
-use crate::common::Server;
+use crate::common::{Server, Value};
 use crate::json;
 
 #[actix_rt::test]
@@ -436,3 +439,262 @@ async fn patch_api_keys_unknown_field() {
     }
     "###);
 }
+
+async fn send_request_with_custom_auth(
+    app: impl actix_web::dev::Service<
+        actix_http::Request,
+        Response = actix_web::dev::ServiceResponse<impl actix_web::body::MessageBody>,
+        Error = actix_web::Error,
+    >,
+    url: &str,
+    auth: &str,
+) -> (Value, StatusCode) {
+    let req = test::TestRequest::get().uri(url).insert_header(("Authorization", auth)).to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+
+    (response, status_code)
+}
+
+#[actix_rt::test]
+async fn invalid_auth_format() {
+    let server = Server::new_auth().await;
+    let app = server.init_web_app().await;
+
+    let req = test::TestRequest::get().uri("/indexes/dog/documents").to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    snapshot!(status_code, @"401 Unauthorized");
+    snapshot!(response, @r###"
+    {
+      "message": "The Authorization header is missing. It must use the bearer authorization method.",
+      "code": "missing_authorization_header",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#missing_authorization_header"
+    }
+    "###);
+
+    let req = test::TestRequest::get().uri("/indexes/dog/documents").to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    snapshot!(status_code, @"401 Unauthorized");
+    snapshot!(response, @r###"
+    {
+      "message": "The Authorization header is missing. It must use the bearer authorization method.",
+      "code": "missing_authorization_header",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#missing_authorization_header"
+    }
+    "###);
+
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/documents", "Bearer").await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "The provided API key is invalid.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+}
+
+#[actix_rt::test]
+async fn invalid_api_key() {
+    let server = Server::new_auth().await;
+    let app = server.init_web_app().await;
+
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/search", "Bearer kefir").await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "The provided API key is invalid.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+
+    let uuid = Uuid::nil();
+    let key = json!({ "actions": ["search"], "indexes": ["dog"], "expiresAt": null, "uid": uuid.to_string() });
+    let req = test::TestRequest::post()
+        .uri("/keys")
+        .insert_header(("Authorization", "Bearer MASTER_KEY"))
+        .set_json(&key)
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    snapshot!(json_string!(response, { ".createdAt" => "[date]",  ".updatedAt" => "[date]" }), @r###"
+    {
+      "name": null,
+      "description": null,
+      "key": "aeb94973e0b6e912d94165430bbe87dee91a7c4f891ce19050c3910ec96977e9",
+      "uid": "00000000-0000-0000-0000-000000000000",
+      "actions": [
+        "search"
+      ],
+      "indexes": [
+        "dog"
+      ],
+      "expiresAt": null,
+      "createdAt": "[date]",
+      "updatedAt": "[date]"
+    }
+    "###);
+    let key = response["key"].as_str().unwrap();
+
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/doggo/search", &format!("Bearer {key}"))
+            .await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "The API key cannot acces the index `doggo`, authorized indexes are [\"dog\"].",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+}
+
+#[actix_rt::test]
+async fn invalid_tenant_token() {
+    let server = Server::new_auth().await;
+    let app = server.init_web_app().await;
+
+    // The tenant token won't be recognized at all if we're not on a search route
+    let claims = json!({ "tamo": "kefir" });
+    let jwt = jsonwebtoken::encode(&Header::default(), &claims, &EncodingKey::from_secret(b"tamo"))
+        .unwrap();
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/documents", &format!("Bearer {jwt}"))
+            .await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "The provided API key is invalid.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+
+    let claims = json!({ "tamo": "kefir" });
+    let jwt = jsonwebtoken::encode(&Header::default(), &claims, &EncodingKey::from_secret(b"tamo"))
+        .unwrap();
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/search", &format!("Bearer {jwt}")).await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "Could not decode tenant token, JSON error: missing field `searchRules` at line 1 column 16.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+
+    // The error messages are not ideal but that's expected since we cannot _yet_ use deserr
+    let claims = json!({ "searchRules": "kefir" });
+    let jwt = jsonwebtoken::encode(&Header::default(), &claims, &EncodingKey::from_secret(b"tamo"))
+        .unwrap();
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/search", &format!("Bearer {jwt}")).await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "Could not decode tenant token, JSON error: data did not match any variant of untagged enum SearchRules at line 1 column 23.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+
+    let uuid = Uuid::nil();
+    let claims = json!({ "searchRules": ["kefir"], "apiKeyUid": uuid.to_string() });
+    let jwt = jsonwebtoken::encode(&Header::default(), &claims, &EncodingKey::from_secret(b"tamo"))
+        .unwrap();
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/search", &format!("Bearer {jwt}")).await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "Could not decode tenant token, InvalidSignature.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+
+    // ~~ For the next tests we first need a valid API key
+    let key = json!({ "actions": ["search"], "indexes": ["dog"], "expiresAt": null, "uid": uuid.to_string() });
+    let req = test::TestRequest::post()
+        .uri("/keys")
+        .insert_header(("Authorization", "Bearer MASTER_KEY"))
+        .set_json(&key)
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    snapshot!(json_string!(response, { ".createdAt" => "[date]",  ".updatedAt" => "[date]" }), @r###"
+    {
+      "name": null,
+      "description": null,
+      "key": "aeb94973e0b6e912d94165430bbe87dee91a7c4f891ce19050c3910ec96977e9",
+      "uid": "00000000-0000-0000-0000-000000000000",
+      "actions": [
+        "search"
+      ],
+      "indexes": [
+        "dog"
+      ],
+      "expiresAt": null,
+      "createdAt": "[date]",
+      "updatedAt": "[date]"
+    }
+    "###);
+    let key = response["key"].as_str().unwrap();
+
+    let claims = json!({ "searchRules": ["doggo", "catto"], "apiKeyUid": uuid.to_string() });
+    let jwt = jsonwebtoken::encode(
+        &Header::default(),
+        &claims,
+        &EncodingKey::from_secret(key.as_bytes()),
+    )
+    .unwrap();
+    // Try to access an index that is not authorized by the tenant token
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/dog/search", &format!("Bearer {jwt}")).await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "The provided tenant token cannot acces the index `dog`, allowed indexes are [\"catto\", \"doggo\"].",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+
+    // Try to access an index that *is* authorized by the tenant token but not by the api key used to generate the tt
+    let (response, status_code) =
+        send_request_with_custom_auth(&app, "/indexes/doggo/search", &format!("Bearer {jwt}"))
+            .await;
+    snapshot!(status_code, @"403 Forbidden");
+    snapshot!(response, @r###"
+    {
+      "message": "The API key used to generate this tenant token cannot acces the index `doggo`.",
+      "code": "invalid_api_key",
+      "type": "auth",
+      "link": "https://docs.meilisearch.com/errors#invalid_api_key"
+    }
+    "###);
+}

meilisearch/tests/auth/tenant_token.rs

@@ -53,7 +53,8 @@ static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
 });
 
 static INVALID_RESPONSE: Lazy<Value> = Lazy::new(|| {
-    json!({"message": "The provided API key is invalid.",
+    json!({
+        "message": null,
         "code": "invalid_api_key",
         "type": "auth",
         "link": "https://docs.meilisearch.com/errors#invalid_api_key"
@@ -191,7 +192,9 @@ macro_rules! compute_forbidden_search {
                 server.use_api_key(&web_token);
                 let index = server.index("sales");
                 index
-                    .search(json!({}), |response, code| {
+                    .search(json!({}), |mut response, code| {
+                        // We don't assert anything on the message since it may change between cases
+                        response["message"] = serde_json::json!(null);
                         assert_eq!(
                             response,
                             INVALID_RESPONSE.clone(),
@@ -495,7 +498,8 @@ async fn error_access_forbidden_routes() {
 
     for ((method, route), actions) in AUTHORIZATIONS.iter() {
         if !actions.contains("search") {
-            let (response, code) = server.dummy_request(method, route).await;
+            let (mut response, code) = server.dummy_request(method, route).await;
+            response["message"] = serde_json::json!(null);
             assert_eq!(response, INVALID_RESPONSE.clone());
             assert_eq!(code, 403);
         }
@@ -529,14 +533,16 @@ async fn error_access_expired_parent_key() {
     server.use_api_key(&web_token);
 
     // test search request while parent_key is not expired
-    let (response, code) = server.dummy_request("POST", "/indexes/products/search").await;
+    let (mut response, code) = server.dummy_request("POST", "/indexes/products/search").await;
+    response["message"] = serde_json::json!(null);
     assert_ne!(response, INVALID_RESPONSE.clone());
     assert_ne!(code, 403);
 
     // wait until the key is expired.
     thread::sleep(time::Duration::new(1, 0));
 
-    let (response, code) = server.dummy_request("POST", "/indexes/products/search").await;
+    let (mut response, code) = server.dummy_request("POST", "/indexes/products/search").await;
+    response["message"] = serde_json::json!(null);
     assert_eq!(response, INVALID_RESPONSE.clone());
     assert_eq!(code, 403);
 }
@@ -585,7 +591,8 @@ async fn error_access_modified_token() {
     .join(".");
 
     server.use_api_key(&altered_token);
-    let (response, code) = server.dummy_request("POST", "/indexes/products/search").await;
+    let (mut response, code) = server.dummy_request("POST", "/indexes/products/search").await;
+    response["message"] = serde_json::json!(null);
     assert_eq!(response, INVALID_RESPONSE.clone());
     assert_eq!(code, 403);
 }