Use an experimental feature to avoid copying 64k in memory

.github/workflows/bench-manual.yml

@@ -18,7 +18,7 @@ jobs:
         timeout-minutes: 180 # 3h
         steps:
             - uses: actions/checkout@v3
-            - uses: dtolnay/rust-toolchain@1.79
+            - uses: helix-editor/rust-toolchain@v1
               with:
                 profile: minimal
 

.github/workflows/bench-pr.yml

@@ -16,37 +16,6 @@ jobs:
       runs-on: benchmarks
       timeout-minutes: 180 # 3h
       steps:
-        - name: Check permissions
-          id: permission
-          env:
-            PR_AUTHOR: ${{github.event.issue.user.login }}
-            COMMENT_AUTHOR: ${{github.event.comment.user.login }}
-            REPOSITORY: ${{github.repository}}
-            PR_ID: ${{github.event.issue.number}}
-          run: |
-            PR_REPOSITORY=$(gh api /repos/"$REPOSITORY"/pulls/"$PR_ID" --jq .head.repo.full_name)
-            if $(gh api /repos/"$REPOSITORY"/collaborators/"$PR_AUTHOR"/permission --jq .user.permissions.push)
-            then
-              echo "::notice title=Authentication success::PR author authenticated"
-            else
-              echo "::error title=Authentication error::PR author doesn't have push permission on this repository"
-              exit 1
-            fi
-            if $(gh api /repos/"$REPOSITORY"/collaborators/"$COMMENT_AUTHOR"/permission --jq .user.permissions.push)
-            then
-              echo "::notice title=Authentication success::Comment author authenticated"
-            else
-              echo "::error title=Authentication error::Comment author doesn't have push permission on this repository"
-              exit 1
-            fi
-            if [ "$PR_REPOSITORY" = "$REPOSITORY" ]
-            then
-              echo "::notice title=Authentication success::PR started from main repository"
-            else
-              echo "::error title=Authentication error::PR started from a fork"
-              exit 1
-            fi
-
         - name: Check for Command
           id: command
           uses: xt0rted/slash-command-action@v2
@@ -66,7 +35,7 @@ jobs:
             fetch-depth: 0 # fetch full history to be able to get main commit sha
             ref: ${{ steps.comment-branch.outputs.head_ref }}
 
-        - uses: dtolnay/rust-toolchain@1.79
+        - uses: helix-editor/rust-toolchain@v1
           with:
             profile: minimal
 

.github/workflows/bench-push-indexing.yml

@@ -12,7 +12,7 @@ jobs:
         timeout-minutes: 180 # 3h
         steps:
           - uses: actions/checkout@v3
-          - uses: dtolnay/rust-toolchain@1.79
+          - uses: helix-editor/rust-toolchain@v1
             with:
               profile: minimal
 

.github/workflows/benchmarks-manual.yml

@@ -18,7 +18,7 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
 

.github/workflows/benchmarks-pr.yml

@@ -13,38 +13,7 @@ jobs:
     runs-on: benchmarks
     timeout-minutes: 4320 # 72h
     steps:
-      - name: Check permissions
-        id: permission
-        env:
-          PR_AUTHOR: ${{github.event.issue.user.login }}
-          COMMENT_AUTHOR: ${{github.event.comment.user.login }}
-          REPOSITORY: ${{github.repository}}
-          PR_ID: ${{github.event.issue.number}}
-        run: |
-          PR_REPOSITORY=$(gh api /repos/"$REPOSITORY"/pulls/"$PR_ID" --jq .head.repo.full_name)
-          if $(gh api /repos/"$REPOSITORY"/collaborators/"$PR_AUTHOR"/permission --jq .user.permissions.push)
-          then
-            echo "::notice title=Authentication success::PR author authenticated"
-          else
-            echo "::error title=Authentication error::PR author doesn't have push permission on this repository"
-            exit 1
-          fi
-          if $(gh api /repos/"$REPOSITORY"/collaborators/"$COMMENT_AUTHOR"/permission --jq .user.permissions.push)
-          then
-            echo "::notice title=Authentication success::Comment author authenticated"
-          else
-            echo "::error title=Authentication error::Comment author doesn't have push permission on this repository"
-            exit 1
-          fi
-          if [ "$PR_REPOSITORY" = "$REPOSITORY" ]
-          then
-            echo "::notice title=Authentication success::PR started from main repository"
-          else
-            echo "::error title=Authentication error::PR started from a fork"
-            exit 1
-          fi
-
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-indexing.yml

@@ -16,7 +16,7 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-search-geo.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-search-songs.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
 

.github/workflows/benchmarks-push-search-wiki.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
 

.github/workflows/flaky-tests.yml

@@ -1,5 +1,6 @@
 name: Look for flaky tests
-
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 on:
   workflow_dispatch:
   schedule:
@@ -9,15 +10,15 @@ jobs:
   flaky:
     runs-on: ubuntu-latest
     container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
     steps:
     - uses: actions/checkout@v3
     - name: Install needed dependencies
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: dtolnay/rust-toolchain@1.79
+    - uses: helix-editor/rust-toolchain@v1
     - name: Install cargo-flaky
       run: cargo install cargo-flaky
     - name: Run cargo flaky in the dumps

.github/workflows/fuzzer-indexing.yml

@@ -1,5 +1,6 @@
 name: Run the indexing fuzzer
-
+env:
+  ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
 on:
   push:
     branches:
@@ -12,7 +13,7 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
 

.github/workflows/publish-apt-brew-pkg.yml

@@ -15,17 +15,19 @@ jobs:
 
   debian:
     name: Publish debian packagge
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     needs: check-version
     container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
+      # Use ubuntu-18.04 to compile with glibc 2.27
+      image: ubuntu:18.04
     steps:
     - name: Install needed dependencies
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: dtolnay/rust-toolchain@1.79
+    - uses: helix-editor/rust-toolchain@v1
     - name: Install cargo-deb
       run: cargo install cargo-deb
     - uses: actions/checkout@v3

.github/workflows/publish-binaries.yml

@@ -35,17 +35,19 @@ jobs:
   publish-linux:
     name: Publish binary for Linux
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     needs: check-version
     container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
+      # Use ubuntu-18.04 to compile with glibc 2.27
+      image: ubuntu:18.04
     steps:
     - uses: actions/checkout@v3
     - name: Install needed dependencies
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: dtolnay/rust-toolchain@1.79
+    - uses: helix-editor/rust-toolchain@v1
     - name: Build
       run: cargo build --release --locked
     # No need to upload binaries for dry run (cron)
@@ -75,7 +77,7 @@ jobs:
             asset_name: meilisearch-windows-amd64.exe
     steps:
     - uses: actions/checkout@v3
-    - uses: dtolnay/rust-toolchain@1.79
+    - uses: helix-editor/rust-toolchain@v1
     - name: Build
       run: cargo build --release --locked
     # No need to upload binaries for dry run (cron)
@@ -101,7 +103,7 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v3
       - name: Installing Rust toolchain
-        uses: dtolnay/rust-toolchain@1.79
+        uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
           target: ${{ matrix.target }}
@@ -125,10 +127,10 @@ jobs:
     runs-on: ubuntu-latest
     needs: check-version
     env:
-      DEBIAN_FRONTEND: noninteractive
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
+      # Use ubuntu-18.04 to compile with glibc 2.27
+      image: ubuntu:18.04
     strategy:
       matrix:
         include:
@@ -148,7 +150,7 @@ jobs:
           add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
           apt-get update -y && apt-get install -y docker-ce
       - name: Installing Rust toolchain
-        uses: dtolnay/rust-toolchain@1.79
+        uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
           target: ${{ matrix.target }}
@@ -162,9 +164,6 @@ jobs:
           echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
           echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
           echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
-      - name: Install a default toolchain that will be used to build cargo cross
-        run: |
-          rustup default stable
       - name: Cargo build
         uses: actions-rs/cargo@v1
         with:

.github/workflows/test-suite.yml

@@ -19,11 +19,13 @@ env:
 
 jobs:
   test-linux:
-    name: Tests on ubuntu-20.04
+    name: Tests on ubuntu-18.04
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
     steps:
       - uses: actions/checkout@v3
       - name: Install needed dependencies
@@ -31,7 +33,7 @@ jobs:
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
       - name: Setup test with Rust stable
-        uses: dtolnay/rust-toolchain@1.79
+        uses: helix-editor/rust-toolchain@v1
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
       - name: Run cargo check without any default features
@@ -56,7 +58,7 @@ jobs:
       - uses: actions/checkout@v3
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -71,9 +73,11 @@ jobs:
   test-all-features:
     name: Tests almost all features
     runs-on: ubuntu-latest
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/checkout@v3
@@ -81,7 +85,7 @@ jobs:
         run: |
           apt-get update
           apt-get install --assume-yes build-essential curl
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
       - name: Run cargo build with almost all features
         run: |
           cargo build --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda)"
@@ -91,9 +95,11 @@ jobs:
 
   test-disabled-tokenization:
     name: Test disabled tokenization
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
-      image: ubuntu:20.04
+      image: ubuntu:18.04
     if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
     steps:
       - uses: actions/checkout@v3
@@ -101,7 +107,7 @@ jobs:
         run: |
           apt-get update
           apt-get install --assume-yes build-essential curl
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
       - name: Run cargo tree without default features and check lindera is not present
         run: |
           if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -qz lindera; then
@@ -115,17 +121,19 @@ jobs:
   # We run tests in debug also, to make sure that the debug_assertions are hit
   test-debug:
     name: Run tests in debug
+    env:
+      ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
     runs-on: ubuntu-latest
     container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
     steps:
       - uses: actions/checkout@v3
       - name: Install needed dependencies
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
       - name: Run tests in debug
@@ -139,7 +147,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
           components: clippy
@@ -156,7 +164,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
           toolchain: nightly-2024-07-09

.github/workflows/update-cargo-toml-version.yml

@@ -18,7 +18,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: helix-editor/rust-toolchain@v1
         with:
           profile: minimal
       - name: Install sd

CONTRIBUTING.md

@@ -62,10 +62,6 @@ We recommend using the standard `$HOME/.cache/lindera` directory:
 export LINDERA_CACHE=$HOME/.cache/lindera
 ```
 
-Furthermore, you can improve incremental compilation by setting the `MEILI_NO_VERGEN` environment variable.
-Setting this variable will prevent the Meilisearch binary from being rebuilt each time the directory that hosts the Meilisearch repository changes.
-Do not enable this environment variable for production builds (as it will break the `version` route, among other things).
-
 #### Snapshot-based tests
 
 We are using [insta](https://insta.rs) to perform snapshot-based testing.

Cargo.toml

@@ -22,7 +22,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.11.1"
+version = "1.9.0"
 authors = [
     "Quentin de Quelen <quentin@dequelen.me>",
     "Clément Renault <clement@meilisearch.com>",

README.md

@@ -45,14 +45,14 @@ See the list of all our example apps in our [demos repository](https://github.co
 ## ✨ Features
 - **Hybrid search:** Combine the best of both [semantic](https://www.meilisearch.com/docs/learn/experimental/vector_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) & full-text search to get the most relevant results
 - **Search-as-you-type:** Find & display results in less than 50 milliseconds to provide an intuitive experience
-- **[Typo tolerance](https://www.meilisearch.com/docs/learn/relevancy/typo_tolerance_settings?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** get relevant matches even when queries contain typos and misspellings
+- **[Typo tolerance](https://www.meilisearch.com/docs/learn/configuration/typo_tolerance?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** get relevant matches even when queries contain typos and misspellings
 - **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your users' search experience with custom filters and build a faceted search interface in a few lines of code
 - **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** sort results based on price, date, or pretty much anything else your users need
-- **[Synonym support](https://www.meilisearch.com/docs/learn/relevancy/synonyms?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** configure synonyms to include more relevant content in your search results
+- **[Synonym support](https://www.meilisearch.com/docs/learn/configuration/synonyms?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** configure synonyms to include more relevant content in your search results
 - **[Geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** filter and sort documents based on geographic data
 - **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
 - **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** control which users can access what data with API keys that allow fine-grained permissions handling
-- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** personalize search results for any number of application tenants
+- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** personalize search results for any number of application tenants
 - **Highly Customizable:** customize Meilisearch to your specific needs or use our out-of-the-box and hassle-free presets
 - **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** integrate Meilisearch in your technical stack with our plugins and SDKs
 - **Easy to install, deploy, and maintain**

bors.toml

@@ -1,5 +1,5 @@
 status = [
-    'Tests on ubuntu-20.04',
+    'Tests on ubuntu-18.04',
     'Tests on macos-12',
     'Tests on windows-2022',
     'Run Clippy',

build-info/build.rs

@@ -5,13 +5,6 @@ fn main() {
 }
 
 fn emit_git_variables() -> anyhow::Result<()> {
-    println!("cargo::rerun-if-env-changed=MEILI_NO_VERGEN");
-
-    let has_vergen =
-        !matches!(std::env::var_os("MEILI_NO_VERGEN"), Some(x) if x != "false" && x != "0");
-
-    anyhow::ensure!(has_vergen, "disabled via `MEILI_NO_VERGEN`");
-
     // Note: any code that needs VERGEN_ environment variables should take care to define them manually in the Dockerfile and pass them
     // in the corresponding GitHub workflow (publish_docker.yml).
     // This is due to the Dockerfile building the binary outside of the git directory.

dump/src/lib.rs

@@ -104,11 +104,6 @@ pub enum KindDump {
     DocumentDeletionByFilter {
         filter: serde_json::Value,
     },
-    DocumentEdition {
-        filter: Option<serde_json::Value>,
-        context: Option<serde_json::Map<String, serde_json::Value>>,
-        function: String,
-    },
     Settings {
         settings: Box<meilisearch_types::settings::Settings<Unchecked>>,
         is_deletion: bool,
@@ -177,9 +172,6 @@ impl From<KindWithContent> for KindDump {
             KindWithContent::DocumentDeletionByFilter { filter_expr, .. } => {
                 KindDump::DocumentDeletionByFilter { filter: filter_expr }
             }
-            KindWithContent::DocumentEdition { filter_expr, context, function, .. } => {
-                KindDump::DocumentEdition { filter: filter_expr, context, function }
-            }
             KindWithContent::DocumentClear { .. } => KindDump::DocumentClear,
             KindWithContent::SettingsUpdate {
                 new_settings,
@@ -286,7 +278,6 @@ pub(crate) mod test {
             pagination: Setting::NotSet,
             embedders: Setting::NotSet,
             search_cutoff_ms: Setting::NotSet,
-            localized_attributes: Setting::NotSet,
             _kind: std::marker::PhantomData,
         };
         settings.check()

dump/src/reader/compat/v5_to_v6.rs

@@ -379,7 +379,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                 v5::Setting::NotSet => v6::Setting::NotSet,
             },
             embedders: v6::Setting::NotSet,
-            localized_attributes: v6::Setting::NotSet,
             search_cutoff_ms: v6::Setting::NotSet,
             _kind: std::marker::PhantomData,
         }

dump/src/reader/mod.rs

@@ -255,8 +255,6 @@ pub(crate) mod test {
         }
         "###);
 
-        insta::assert_json_snapshot!(vector_index.settings().unwrap());
-
         {
             let documents: Result<Vec<_>> = vector_index.documents().unwrap().collect();
             let mut documents = documents.unwrap();

dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-5.snap

@@ -1,56 +1,783 @@
 ---
 source: dump/src/reader/mod.rs
-expression: vector_index.settings().unwrap()
+expression: document
 ---
 {
-  "displayedAttributes": [
-    "*"
+  "id": "e3",
+  "desc": "overriden vector + map",
+  "_vectors": {
+    "default": [
+      0.2,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1,
+      0.1
     ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "sortableAttributes": [],
-  "rankingRules": [
-    "words",
-    "typo",
-    "proximity",
-    "attribute",
-    "sort",
-    "exactness"
-  ],
-  "stopWords": [],
-  "nonSeparatorTokens": [],
-  "separatorTokens": [],
-  "dictionary": [],
-  "synonyms": {},
-  "distinctAttribute": null,
-  "proximityPrecision": "byWord",
-  "typoTolerance": {
-    "enabled": true,
-    "minWordSizeForTypos": {
-      "oneTypo": 5,
-      "twoTypos": 9
-    },
-    "disableOnWords": [],
-    "disableOnAttributes": []
-  },
-  "faceting": {
-    "maxValuesPerFacet": 100,
-    "sortFacetValuesBy": {
-      "*": "alpha"
+    "toto": [
+      0.1
+    ]
   }
-  },
-  "pagination": {
-    "maxTotalHits": 1000
-  },
-  "embedders": {
-    "default": {
-      "source": "huggingFace",
-      "model": "BAAI/bge-base-en-v1.5",
-      "revision": "617ca489d9e86b49b8167676d8220688b99db36e",
-      "documentTemplate": "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}"
-    }
-  },
-  "searchCutoffMs": null
 }

dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-9.snap

@@ -1,780 +0,0 @@
----
-source: dump/src/reader/mod.rs
-expression: document
----
-{
-  "id": "e0",
-  "desc": "overriden vector",
-  "_vectors": {
-    "default": [
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1
-    ]
-  }
-}

filter-parser/src/condition.rs

@@ -26,8 +26,6 @@ pub enum Condition<'a> {
     LowerThan(Token<'a>),
     LowerThanOrEqual(Token<'a>),
     Between { from: Token<'a>, to: Token<'a> },
-    Contains { keyword: Token<'a>, word: Token<'a> },
-    StartsWith { keyword: Token<'a>, word: Token<'a> },
 }
 
 /// condition      = value ("==" | ">" ...) value
@@ -94,62 +92,6 @@ pub fn parse_not_exists(input: Span) -> IResult<FilterCondition> {
     Ok((input, FilterCondition::Not(Box::new(FilterCondition::Condition { fid: key, op: Exists }))))
 }
 
-/// contains        = value "CONTAINS" value
-pub fn parse_contains(input: Span) -> IResult<FilterCondition> {
-    let (input, (fid, contains, value)) =
-        tuple((parse_value, tag("CONTAINS"), cut(parse_value)))(input)?;
-    Ok((
-        input,
-        FilterCondition::Condition {
-            fid,
-            op: Contains { keyword: Token { span: contains, value: None }, word: value },
-        },
-    ))
-}
-
-/// contains        = value "NOT" WS+ "CONTAINS" value
-pub fn parse_not_contains(input: Span) -> IResult<FilterCondition> {
-    let keyword = tuple((tag("NOT"), multispace1, tag("CONTAINS")));
-    let (input, (fid, (_not, _spaces, contains), value)) =
-        tuple((parse_value, keyword, cut(parse_value)))(input)?;
-
-    Ok((
-        input,
-        FilterCondition::Not(Box::new(FilterCondition::Condition {
-            fid,
-            op: Contains { keyword: Token { span: contains, value: None }, word: value },
-        })),
-    ))
-}
-
-/// starts with        = value "CONTAINS" value
-pub fn parse_starts_with(input: Span) -> IResult<FilterCondition> {
-    let (input, (fid, starts_with, value)) =
-        tuple((parse_value, tag("STARTS WITH"), cut(parse_value)))(input)?;
-    Ok((
-        input,
-        FilterCondition::Condition {
-            fid,
-            op: StartsWith { keyword: Token { span: starts_with, value: None }, word: value },
-        },
-    ))
-}
-
-/// starts with        = value "NOT" WS+ "CONTAINS" value
-pub fn parse_not_starts_with(input: Span) -> IResult<FilterCondition> {
-    let keyword = tuple((tag("NOT"), multispace1, tag("STARTS WITH")));
-    let (input, (fid, (_not, _spaces, starts_with), value)) =
-        tuple((parse_value, keyword, cut(parse_value)))(input)?;
-
-    Ok((
-        input,
-        FilterCondition::Not(Box::new(FilterCondition::Condition {
-            fid,
-            op: StartsWith { keyword: Token { span: starts_with, value: None }, word: value },
-        })),
-    ))
-}
-
 /// to             = value value "TO" WS+ value
 pub fn parse_to(input: Span) -> IResult<FilterCondition> {
     let (input, (key, from, _, _, to)) =

filter-parser/src/error.rs

@@ -146,7 +146,7 @@ impl<'a> Display for Error<'a> {
             }
             ErrorKind::InvalidPrimary => {
                 let text = if input.trim().is_empty() { "but instead got nothing.".to_string() } else { format!("at `{}`.", escaped_input) };
-                writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` {}", text)?
+                writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` {}", text)?
             }
             ErrorKind::InvalidEscapedNumber => {
                 writeln!(f, "Found an invalid escaped sequence number: `{}`.", escaped_input)?

filter-parser/src/lib.rs

@@ -48,8 +48,8 @@ use std::fmt::Debug;
 
 pub use condition::{parse_condition, parse_to, Condition};
 use condition::{
-    parse_contains, parse_exists, parse_is_empty, parse_is_not_empty, parse_is_not_null,
-    parse_is_null, parse_not_contains, parse_not_exists, parse_not_starts_with, parse_starts_with,
+    parse_exists, parse_is_empty, parse_is_not_empty, parse_is_not_null, parse_is_null,
+    parse_not_exists,
 };
 use error::{cut_with_err, ExpectedValueKind, NomErrorExt};
 pub use error::{Error, ErrorKind};
@@ -147,38 +147,7 @@ pub enum FilterCondition<'a> {
     GeoBoundingBox { top_right_point: [Token<'a>; 2], bottom_left_point: [Token<'a>; 2] },
 }
 
-pub enum TraversedElement<'a> {
-    FilterCondition(&'a FilterCondition<'a>),
-    Condition(&'a Condition<'a>),
-}
-
 impl<'a> FilterCondition<'a> {
-    pub fn use_contains_operator(&self) -> Option<&Token> {
-        match self {
-            FilterCondition::Condition { fid: _, op } => match op {
-                Condition::GreaterThan(_)
-                | Condition::GreaterThanOrEqual(_)
-                | Condition::Equal(_)
-                | Condition::NotEqual(_)
-                | Condition::Null
-                | Condition::Empty
-                | Condition::Exists
-                | Condition::LowerThan(_)
-                | Condition::LowerThanOrEqual(_)
-                | Condition::Between { .. } => None,
-                Condition::Contains { keyword, word: _ }
-                | Condition::StartsWith { keyword, word: _ } => Some(keyword),
-            },
-            FilterCondition::Not(this) => this.use_contains_operator(),
-            FilterCondition::Or(seq) | FilterCondition::And(seq) => {
-                seq.iter().find_map(|filter| filter.use_contains_operator())
-            }
-            FilterCondition::GeoLowerThan { .. }
-            | FilterCondition::GeoBoundingBox { .. }
-            | FilterCondition::In { .. } => None,
-        }
-    }
-
     /// Returns the first token found at the specified depth, `None` if no token at this depth.
     pub fn token_at_depth(&self, depth: usize) -> Option<&Token> {
         match self {
@@ -483,10 +452,6 @@ fn parse_primary(input: Span, depth: usize) -> IResult<FilterCondition> {
         parse_exists,
         parse_not_exists,
         parse_to,
-        parse_contains,
-        parse_not_contains,
-        parse_starts_with,
-        parse_not_starts_with,
         // the next lines are only for error handling and are written at the end to have the less possible performance impact
         parse_geo,
         parse_geo_distance,
@@ -569,8 +534,6 @@ impl<'a> std::fmt::Display for Condition<'a> {
             Condition::LowerThan(token) => write!(f, "< {token}"),
             Condition::LowerThanOrEqual(token) => write!(f, "<= {token}"),
             Condition::Between { from, to } => write!(f, "{from} TO {to}"),
-            Condition::Contains { word, keyword: _ } => write!(f, "CONTAINS {word}"),
-            Condition::StartsWith { word, keyword: _ } => write!(f, "STARTS WITH {word}"),
         }
     }
 }
@@ -595,7 +558,6 @@ pub mod tests {
         unsafe { Span::new_from_raw_offset(offset, lines as u32, value, "") }.into()
     }
 
-    #[track_caller]
     fn p(s: &str) -> impl std::fmt::Display + '_ {
         Fc::parse(s).unwrap().unwrap()
     }
@@ -677,20 +639,6 @@ pub mod tests {
         insta::assert_snapshot!(p("NOT subscribers NOT EXISTS"), @"{subscribers} EXISTS");
         insta::assert_snapshot!(p("subscribers NOT   EXISTS"), @"NOT ({subscribers} EXISTS)");
 
-        // Test CONTAINS + NOT CONTAINS
-        insta::assert_snapshot!(p("subscribers CONTAINS 'hello'"), @"{subscribers} CONTAINS {hello}");
-        insta::assert_snapshot!(p("NOT subscribers CONTAINS 'hello'"), @"NOT ({subscribers} CONTAINS {hello})");
-        insta::assert_snapshot!(p("subscribers NOT CONTAINS hello"), @"NOT ({subscribers} CONTAINS {hello})");
-        insta::assert_snapshot!(p("NOT subscribers NOT CONTAINS 'hello'"), @"{subscribers} CONTAINS {hello}");
-        insta::assert_snapshot!(p("subscribers NOT   CONTAINS 'hello'"), @"NOT ({subscribers} CONTAINS {hello})");
-
-        // Test STARTS WITH + NOT STARTS WITH
-        insta::assert_snapshot!(p("subscribers STARTS WITH 'hel'"), @"{subscribers} STARTS WITH {hel}");
-        insta::assert_snapshot!(p("NOT subscribers STARTS WITH 'hel'"), @"NOT ({subscribers} STARTS WITH {hel})");
-        insta::assert_snapshot!(p("subscribers NOT STARTS WITH hel"), @"NOT ({subscribers} STARTS WITH {hel})");
-        insta::assert_snapshot!(p("NOT subscribers NOT STARTS WITH 'hel'"), @"{subscribers} STARTS WITH {hel}");
-        insta::assert_snapshot!(p("subscribers NOT   STARTS WITH 'hel'"), @"NOT ({subscribers} STARTS WITH {hel})");
-
         // Test nested NOT
         insta::assert_snapshot!(p("NOT NOT NOT NOT x = 5"), @"{x} = {5}");
         insta::assert_snapshot!(p("NOT NOT (NOT NOT x = 5)"), @"{x} = {5}");
@@ -762,7 +710,7 @@ pub mod tests {
         "###);
 
         insta::assert_snapshot!(p("'OR'"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `\'OR\'`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `\'OR\'`.
         1:5 'OR'
         "###);
 
@@ -772,12 +720,12 @@ pub mod tests {
         "###);
 
         insta::assert_snapshot!(p("channel Ponce"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `channel Ponce`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `channel Ponce`.
         1:14 channel Ponce
         "###);
 
         insta::assert_snapshot!(p("channel = Ponce OR"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.
         19:19 channel = Ponce OR
         "###);
 
@@ -862,12 +810,12 @@ pub mod tests {
         "###);
 
         insta::assert_snapshot!(p("colour NOT EXIST"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `colour NOT EXIST`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `colour NOT EXIST`.
         1:17 colour NOT EXIST
         "###);
 
         insta::assert_snapshot!(p("subscribers 100 TO1000"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `subscribers 100 TO1000`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `subscribers 100 TO1000`.
         1:23 subscribers 100 TO1000
         "###);
 
@@ -930,35 +878,35 @@ pub mod tests {
         "###);
 
         insta::assert_snapshot!(p(r#"value NULL"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value NULL`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value NULL`.
         1:11 value NULL
         "###);
         insta::assert_snapshot!(p(r#"value NOT NULL"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value NOT NULL`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value NOT NULL`.
         1:15 value NOT NULL
         "###);
         insta::assert_snapshot!(p(r#"value EMPTY"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value EMPTY`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value EMPTY`.
         1:12 value EMPTY
         "###);
         insta::assert_snapshot!(p(r#"value NOT EMPTY"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value NOT EMPTY`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value NOT EMPTY`.
         1:16 value NOT EMPTY
         "###);
         insta::assert_snapshot!(p(r#"value IS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS`.
         1:9 value IS
         "###);
         insta::assert_snapshot!(p(r#"value IS NOT"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT`.
         1:13 value IS NOT
         "###);
         insta::assert_snapshot!(p(r#"value IS EXISTS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS EXISTS`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS EXISTS`.
         1:16 value IS EXISTS
         "###);
         insta::assert_snapshot!(p(r#"value IS NOT EXISTS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT EXISTS`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT EXISTS`.
         1:20 value IS NOT EXISTS
         "###);
     }

filter-parser/src/value.rs

@@ -211,9 +211,6 @@ fn is_keyword(s: &str) -> bool {
             | "IS"
             | "NULL"
             | "EMPTY"
-            | "CONTAINS"
-            | "STARTS"
-            | "WITH"
             | "_geoRadius"
             | "_geoBoundingBox"
     )

fuzzers/src/bin/fuzz-indexing.rs

@@ -110,7 +110,7 @@ fn main() {
 
                             // after executing a batch we check if the database is corrupted
                             let res = index.search(&wtxn).execute().unwrap();
-                            index.documents(&wtxn, res.documents_ids).unwrap();
+                            index.compressed_documents(&wtxn, res.documents_ids).unwrap();
                             progression.fetch_add(1, Ordering::Relaxed);
                         }
                         wtxn.abort();

index-scheduler/Cargo.toml

@@ -40,7 +40,7 @@ ureq = "2.10.0"
 uuid = { version = "1.10.0", features = ["serde", "v4"] }
 
 [dev-dependencies]
-arroy = "0.5.0"
+arroy = "0.4.0"
 big_s = "1.0.2"
 crossbeam = "0.8.4"
 insta = { version = "1.39.0", features = ["json", "redactions"] }

index-scheduler/src/autobatcher.rs

@@ -24,10 +24,8 @@ enum AutobatchKind {
         allow_index_creation: bool,
         primary_key: Option<String>,
     },
-    DocumentEdition,
-    DocumentDeletion {
-        by_filter: bool,
-    },
+    DocumentDeletion,
+    DocumentDeletionByFilter,
     DocumentClear,
     Settings {
         allow_index_creation: bool,
@@ -65,13 +63,10 @@ impl From<KindWithContent> for AutobatchKind {
                 primary_key,
                 ..
             } => AutobatchKind::DocumentImport { method, allow_index_creation, primary_key },
-            KindWithContent::DocumentEdition { .. } => AutobatchKind::DocumentEdition,
-            KindWithContent::DocumentDeletion { .. } => {
-                AutobatchKind::DocumentDeletion { by_filter: false }
-            }
+            KindWithContent::DocumentDeletion { .. } => AutobatchKind::DocumentDeletion,
             KindWithContent::DocumentClear { .. } => AutobatchKind::DocumentClear,
             KindWithContent::DocumentDeletionByFilter { .. } => {
-                AutobatchKind::DocumentDeletion { by_filter: true }
+                AutobatchKind::DocumentDeletionByFilter
             }
             KindWithContent::SettingsUpdate { allow_index_creation, is_deletion, .. } => {
                 AutobatchKind::Settings {
@@ -103,12 +98,11 @@ pub enum BatchKind {
         primary_key: Option<String>,
         operation_ids: Vec<TaskId>,
     },
-    DocumentEdition {
-        id: TaskId,
-    },
     DocumentDeletion {
         deletion_ids: Vec<TaskId>,
-        includes_by_filter: bool,
+    },
+    DocumentDeletionByFilter {
+        id: TaskId,
     },
     ClearAndSettings {
         other: Vec<TaskId>,
@@ -205,14 +199,12 @@ impl BatchKind {
                 }),
                 allow_index_creation,
             ),
-            K::DocumentEdition => (Break(BatchKind::DocumentEdition { id: task_id }), false),
-            K::DocumentDeletion { by_filter: includes_by_filter } => (
-                Continue(BatchKind::DocumentDeletion {
-                    deletion_ids: vec![task_id],
-                    includes_by_filter,
-                }),
-                false,
-            ),
+            K::DocumentDeletion => {
+                (Continue(BatchKind::DocumentDeletion { deletion_ids: vec![task_id] }), false)
+            }
+            K::DocumentDeletionByFilter => {
+                (Break(BatchKind::DocumentDeletionByFilter { id: task_id }), false)
+            }
             K::Settings { allow_index_creation } => (
                 Continue(BatchKind::Settings { allow_index_creation, settings_ids: vec![task_id] }),
                 allow_index_creation,
@@ -230,7 +222,7 @@ impl BatchKind {
 
         match (self, kind) {
             // We don't batch any of these operations
-            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => Break(this),
+            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentDeletionByFilter) => Break(this),
             // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists.
             (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => {
                 Break(this)
@@ -266,7 +258,7 @@ impl BatchKind {
             // The index deletion can batch with everything but must stop after
             (
                 BatchKind::DocumentClear { mut ids }
-                | BatchKind::DocumentDeletion { deletion_ids: mut ids, includes_by_filter: _ }
+                | BatchKind::DocumentDeletion { deletion_ids: mut ids }
                 | BatchKind::DocumentOperation { method: _, allow_index_creation: _, primary_key: _, operation_ids: mut ids }
                 | BatchKind::Settings { allow_index_creation: _, settings_ids: mut ids },
                 K::IndexDeletion,
@@ -286,7 +278,7 @@ impl BatchKind {
 
             (
                 BatchKind::DocumentClear { mut ids },
-                K::DocumentClear | K::DocumentDeletion { by_filter: _ },
+                K::DocumentClear | K::DocumentDeletion,
             ) => {
                 ids.push(id);
                 Continue(BatchKind::DocumentClear { ids })
@@ -330,7 +322,7 @@ impl BatchKind {
             }
             (
                 BatchKind::DocumentOperation { method, allow_index_creation, primary_key, mut operation_ids },
-                K::DocumentDeletion { by_filter: false },
+                K::DocumentDeletion,
             ) => {
                 operation_ids.push(id);
 
@@ -341,13 +333,6 @@ impl BatchKind {
                     operation_ids,
                 })
             }
-            // We can't batch a document operation with a delete by filter
-            (
-                this @ BatchKind::DocumentOperation { .. },
-                K::DocumentDeletion { by_filter: true },
-            ) => {
-                Break(this)
-            }
             // but we can't autobatch documents if it's not the same kind
             // this match branch MUST be AFTER the previous one
             (
@@ -366,18 +351,13 @@ impl BatchKind {
                 operation_ids,
             }),
 
-            (BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: _ }, K::DocumentClear) => {
+            (BatchKind::DocumentDeletion { mut deletion_ids }, K::DocumentClear) => {
                 deletion_ids.push(id);
                 Continue(BatchKind::DocumentClear { ids: deletion_ids })
             }
-            // we can't autobatch the deletion and import if the document deletion contained a filter
-            (
-                this @ BatchKind::DocumentDeletion { deletion_ids: _, includes_by_filter: true },
-                K::DocumentImport { .. }
-            ) => Break(this),
             // we can autobatch the deletion and import if the index already exists
             (
-                BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: false },
+                BatchKind::DocumentDeletion { mut deletion_ids },
                 K::DocumentImport { method, allow_index_creation, primary_key }
             ) if index_already_exists => {
                 deletion_ids.push(id);
@@ -391,7 +371,7 @@ impl BatchKind {
             }
             // we can autobatch the deletion and import if both can't create an index
             (
-                BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: false },
+                BatchKind::DocumentDeletion { mut deletion_ids },
                 K::DocumentImport { method, allow_index_creation, primary_key }
             ) if !allow_index_creation => {
                 deletion_ids.push(id);
@@ -410,9 +390,9 @@ impl BatchKind {
             ) => {
                 Break(this)
             }
-            (BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter }, K::DocumentDeletion { by_filter }) => {
+            (BatchKind::DocumentDeletion { mut deletion_ids }, K::DocumentDeletion) => {
                 deletion_ids.push(id);
-                Continue(BatchKind::DocumentDeletion { deletion_ids, includes_by_filter: includes_by_filter | by_filter })
+                Continue(BatchKind::DocumentDeletion { deletion_ids })
             }
             (this @ BatchKind::DocumentDeletion { .. }, K::Settings { .. }) => Break(this),
 
@@ -426,7 +406,7 @@ impl BatchKind {
             }),
             (
                 this @ BatchKind::Settings { .. },
-                K::DocumentImport { .. } | K::DocumentDeletion { .. },
+                K::DocumentImport { .. } | K::DocumentDeletion,
             ) => Break(this),
             (
                 BatchKind::Settings { mut settings_ids, allow_index_creation },
@@ -457,7 +437,7 @@ impl BatchKind {
                     settings_ids,
                     allow_index_creation,
                 },
-                K::DocumentDeletion { .. },
+                K::DocumentDeletion,
             ) => {
                 other.push(id);
                 Continue(BatchKind::ClearAndSettings {
@@ -519,7 +499,7 @@ impl BatchKind {
             // this MUST be AFTER the two previous branch
             (
                 this @ BatchKind::SettingsAndDocumentOperation { .. },
-                K::DocumentDeletion { .. } | K::DocumentImport { .. },
+                K::DocumentDeletion | K::DocumentImport { .. },
             ) => Break(this),
             (
                 BatchKind::SettingsAndDocumentOperation { mut settings_ids, method, allow_index_creation,primary_key, operation_ids },
@@ -539,7 +519,7 @@ impl BatchKind {
                 | BatchKind::IndexDeletion { .. }
                 | BatchKind::IndexUpdate { .. }
                 | BatchKind::IndexSwap { .. }
-                | BatchKind::DocumentEdition { .. },
+                | BatchKind::DocumentDeletionByFilter { .. },
                 _,
             ) => {
                 unreachable!()
@@ -629,13 +609,6 @@ mod tests {
         }
     }
 
-    fn doc_del_fil() -> KindWithContent {
-        KindWithContent::DocumentDeletionByFilter {
-            index_uid: String::from("doggo"),
-            filter_expr: serde_json::json!("cuteness > 100"),
-        }
-    }
-
     fn doc_clr() -> KindWithContent {
         KindWithContent::DocumentClear { index_uid: String::from("doggo") }
     }
@@ -696,16 +669,10 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1, 2] }, false))");
 
         // we can autobatch one or multiple DocumentDeletion together
-        debug_snapshot!(autobatch_from(true, None, [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: false }, false))");
-
-        // we can autobatch one or multiple DocumentDeletionByFilter together
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_del_fil(), doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del_fil(), doc_del_fil(), doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2] }, false))");
 
         // we can autobatch one or multiple Settings together
         debug_snapshot!(autobatch_from(true, None, [settings(true)]), @"Some((Settings { allow_index_creation: true, settings_ids: [0] }, true))");
@@ -748,63 +715,25 @@ mod tests {
         debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
         debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-
-        // But we can't autobatch document addition with document deletion by filter
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
-        // And the other way around
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
     }
 
     #[test]
     fn simple_document_operation_dont_autobatch_with_other() {
-        // addition, updates and deletion by filter can't batch together
+        // addition, updates and deletion can't batch together
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
 
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_create()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_create()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
 
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_update()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_update()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
 
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_swap()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_swap()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
     }
 
     #[test]
@@ -871,7 +800,6 @@ mod tests {
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_del()]), @"Some((IndexDeletion { ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_del_fil()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_clr()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), settings(true)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), settings(false)]), @"Some((IndexDeletion { ids: [0] }, false))");
@@ -881,7 +809,6 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_del()]), @"Some((IndexDeletion { ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_del_fil()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_clr()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), settings(true)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), settings(false)]), @"Some((IndexDeletion { ids: [0] }, false))");
@@ -893,7 +820,6 @@ mod tests {
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
         debug_snapshot!(autobatch_from(true, None, [settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
@@ -903,7 +829,6 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_del(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del_fil(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
         debug_snapshot!(autobatch_from(false,None,  [settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
@@ -969,10 +894,10 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), settings(true)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
 
         // batch deletion and addition
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
     }
 
     #[test]

index-scheduler/src/batch.rs

@@ -34,7 +34,7 @@ use meilisearch_types::milli::update::{
 use meilisearch_types::milli::vector::parsed_vectors::{
     ExplicitVectors, VectorOrArrayOfVectors, RESERVED_VECTORS_FIELD_NAME,
 };
-use meilisearch_types::milli::{self, Filter, Object};
+use meilisearch_types::milli::{self, Filter};
 use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked};
 use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status, Task};
 use meilisearch_types::{compression, Index, VERSION_FILE_NAME};
@@ -106,14 +106,10 @@ pub(crate) enum IndexOperation {
         operations: Vec<DocumentOperation>,
         tasks: Vec<Task>,
     },
-    DocumentEdition {
+    IndexDocumentDeletionByFilter {
         index_uid: String,
         task: Task,
     },
-    DocumentDeletion {
-        index_uid: String,
-        tasks: Vec<Task>,
-    },
     DocumentClear {
         index_uid: String,
         tasks: Vec<Task>,
@@ -165,11 +161,10 @@ impl Batch {
             Batch::IndexOperation { op, .. } => match op {
                 IndexOperation::DocumentOperation { tasks, .. }
                 | IndexOperation::Settings { tasks, .. }
-                | IndexOperation::DocumentDeletion { tasks, .. }
                 | IndexOperation::DocumentClear { tasks, .. } => {
                     RoaringBitmap::from_iter(tasks.iter().map(|task| task.uid))
                 }
-                IndexOperation::DocumentEdition { task, .. } => {
+                IndexOperation::IndexDocumentDeletionByFilter { task, .. } => {
                     RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
                 }
                 IndexOperation::SettingsAndDocumentOperation {
@@ -233,8 +228,7 @@ impl IndexOperation {
     pub fn index_uid(&self) -> &str {
         match self {
             IndexOperation::DocumentOperation { index_uid, .. }
-            | IndexOperation::DocumentEdition { index_uid, .. }
-            | IndexOperation::DocumentDeletion { index_uid, .. }
+            | IndexOperation::IndexDocumentDeletionByFilter { index_uid, .. }
             | IndexOperation::DocumentClear { index_uid, .. }
             | IndexOperation::Settings { index_uid, .. }
             | IndexOperation::DocumentClearAndSetting { index_uid, .. }
@@ -249,11 +243,8 @@ impl fmt::Display for IndexOperation {
             IndexOperation::DocumentOperation { .. } => {
                 f.write_str("IndexOperation::DocumentOperation")
             }
-            IndexOperation::DocumentEdition { .. } => {
-                f.write_str("IndexOperation::DocumentEdition")
-            }
-            IndexOperation::DocumentDeletion { .. } => {
-                f.write_str("IndexOperation::DocumentDeletion")
+            IndexOperation::IndexDocumentDeletionByFilter { .. } => {
+                f.write_str("IndexOperation::IndexDocumentDeletionByFilter")
             }
             IndexOperation::DocumentClear { .. } => f.write_str("IndexOperation::DocumentClear"),
             IndexOperation::Settings { .. } => f.write_str("IndexOperation::Settings"),
@@ -289,12 +280,12 @@ impl IndexScheduler {
                 },
                 must_create_index,
             })),
-            BatchKind::DocumentEdition { id } => {
+            BatchKind::DocumentDeletionByFilter { id } => {
                 let task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
                 match &task.kind {
-                    KindWithContent::DocumentEdition { index_uid, .. } => {
+                    KindWithContent::DocumentDeletionByFilter { index_uid, .. } => {
                         Ok(Some(Batch::IndexOperation {
-                            op: IndexOperation::DocumentEdition {
+                            op: IndexOperation::IndexDocumentDeletionByFilter {
                                 index_uid: index_uid.clone(),
                                 task,
                             },
@@ -351,11 +342,30 @@ impl IndexScheduler {
                     must_create_index,
                 }))
             }
-            BatchKind::DocumentDeletion { deletion_ids, includes_by_filter: _ } => {
+            BatchKind::DocumentDeletion { deletion_ids } => {
                 let tasks = self.get_existing_tasks(rtxn, deletion_ids)?;
 
+                let mut operations = Vec::with_capacity(tasks.len());
+                let mut documents_counts = Vec::with_capacity(tasks.len());
+                for task in &tasks {
+                    match task.kind {
+                        KindWithContent::DocumentDeletion { ref documents_ids, .. } => {
+                            operations.push(DocumentOperation::Delete(documents_ids.clone()));
+                            documents_counts.push(documents_ids.len() as u64);
+                        }
+                        _ => unreachable!(),
+                    }
+                }
+
                 Ok(Some(Batch::IndexOperation {
-                    op: IndexOperation::DocumentDeletion { index_uid, tasks },
+                    op: IndexOperation::DocumentOperation {
+                        index_uid,
+                        primary_key: None,
+                        method: IndexDocumentsMethod::ReplaceDocuments,
+                        documents_counts,
+                        operations,
+                        tasks,
+                    },
                     must_create_index,
                 }))
             }
@@ -898,16 +908,22 @@ impl IndexScheduler {
                     let mut index_dumper = dump.create_index(uid, &metadata)?;
 
                     let fields_ids_map = index.fields_ids_map(&rtxn)?;
+                    let dictionary = index.document_decompression_dictionary(&rtxn)?;
                     let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
                     let embedding_configs = index.embedding_configs(&rtxn)?;
+                    let mut buffer = Vec::new();
 
                     // 3.1. Dump the documents
-                    for ret in index.all_documents(&rtxn)? {
+                    for ret in index.all_compressed_documents(&rtxn)? {
                         if self.must_stop_processing.get() {
                             return Err(Error::AbortedTask);
                         }
 
-                        let (id, doc) = ret?;
+                        let (id, compressed) = ret?;
+                        let doc = compressed.decompress_with_optional_dictionary(
+                            &mut buffer,
+                            dictionary.as_ref(),
+                        )?;
 
                         let mut document = milli::obkv_to_json(&all_fields, &fields_ids_map, doc)?;
 
@@ -1247,7 +1263,6 @@ impl IndexScheduler {
                 operations,
                 mut tasks,
             } => {
-                let started_processing_at = std::time::Instant::now();
                 let mut primary_key_has_been_set = false;
                 let must_stop_processing = self.must_stop_processing.clone();
                 let indexer_config = self.index_mapper.indexer_config();
@@ -1362,7 +1377,7 @@ impl IndexScheduler {
 
                 if !tasks.iter().all(|res| res.error.is_some()) {
                     let addition = builder.execute()?;
-                    tracing::info!(indexing_result = ?addition, processed_in = ?started_processing_at.elapsed(), "document indexing done");
+                    tracing::info!(indexing_result = ?addition, "document indexing done");
                 } else if primary_key_has_been_set {
                     // Everything failed but we've set a primary key.
                     // We need to remove it.
@@ -1377,57 +1392,46 @@ impl IndexScheduler {
 
                 Ok(tasks)
             }
-            IndexOperation::DocumentEdition { mut task, .. } => {
-                let (filter, context, function) =
-                    if let KindWithContent::DocumentEdition {
-                        filter_expr, context, function, ..
-                    } = &task.kind
+            IndexOperation::IndexDocumentDeletionByFilter { mut task, index_uid: _ } => {
+                let filter =
+                    if let KindWithContent::DocumentDeletionByFilter { filter_expr, .. } =
+                        &task.kind
                     {
-                        (filter_expr, context, function)
+                        filter_expr
                     } else {
                         unreachable!()
                     };
-                let result_count = edit_documents_by_function(
+                let deleted_documents = delete_document_by_filter(
                     index_wtxn,
                     filter,
-                    context.clone(),
-                    function,
                     self.index_mapper.indexer_config(),
                     self.must_stop_processing.clone(),
                     index,
                 );
-                let (original_filter, context, function) = if let Some(Details::DocumentEdition {
+                let original_filter = if let Some(Details::DocumentDeletionByFilter {
                     original_filter,
-                    context,
-                    function,
-                    ..
+                    deleted_documents: _,
                 }) = task.details
                 {
-                    (original_filter, context, function)
+                    original_filter
                 } else {
-                    // In the case of a `documentEdition` the details MUST be set
+                    // In the case of a `documentDeleteByFilter` the details MUST be set
                     unreachable!();
                 };
 
-                match result_count {
-                    Ok((deleted_documents, edited_documents)) => {
+                match deleted_documents {
+                    Ok(deleted_documents) => {
                         task.status = Status::Succeeded;
-                        task.details = Some(Details::DocumentEdition {
+                        task.details = Some(Details::DocumentDeletionByFilter {
                             original_filter,
-                            context,
-                            function,
                             deleted_documents: Some(deleted_documents),
-                            edited_documents: Some(edited_documents),
                         });
                     }
                     Err(e) => {
                         task.status = Status::Failed;
-                        task.details = Some(Details::DocumentEdition {
+                        task.details = Some(Details::DocumentDeletionByFilter {
                             original_filter,
-                            context,
-                            function,
                             deleted_documents: Some(0),
-                            edited_documents: Some(0),
                         });
                         task.error = Some(e.into());
                     }
@@ -1435,103 +1439,6 @@ impl IndexScheduler {
 
                 Ok(vec![task])
             }
-            IndexOperation::DocumentDeletion { mut tasks, index_uid: _ } => {
-                let mut to_delete = RoaringBitmap::new();
-                let external_documents_ids = index.external_documents_ids();
-
-                for task in tasks.iter_mut() {
-                    let before = to_delete.len();
-                    task.status = Status::Succeeded;
-
-                    match &task.kind {
-                        KindWithContent::DocumentDeletion { index_uid: _, documents_ids } => {
-                            for id in documents_ids {
-                                if let Some(id) = external_documents_ids.get(index_wtxn, id)? {
-                                    to_delete.insert(id);
-                                }
-                            }
-                            let will_be_removed = to_delete.len() - before;
-                            task.details = Some(Details::DocumentDeletion {
-                                provided_ids: documents_ids.len(),
-                                deleted_documents: Some(will_be_removed),
-                            });
-                        }
-                        KindWithContent::DocumentDeletionByFilter { index_uid: _, filter_expr } => {
-                            let before = to_delete.len();
-                            let filter = match Filter::from_json(filter_expr) {
-                                Ok(filter) => filter,
-                                Err(err) => {
-                                    // theorically, this should be catched by deserr before reaching the index-scheduler and cannot happens
-                                    task.status = Status::Failed;
-                                    task.error = match err {
-                                        milli::Error::UserError(
-                                            milli::UserError::InvalidFilterExpression { .. },
-                                        ) => Some(
-                                            Error::from(err)
-                                                .with_custom_error_code(Code::InvalidDocumentFilter)
-                                                .into(),
-                                        ),
-                                        e => Some(e.into()),
-                                    };
-                                    None
-                                }
-                            };
-                            if let Some(filter) = filter {
-                                let candidates =
-                                    filter.evaluate(index_wtxn, index).map_err(|err| match err {
-                                        milli::Error::UserError(
-                                            milli::UserError::InvalidFilter(_),
-                                        ) => Error::from(err)
-                                            .with_custom_error_code(Code::InvalidDocumentFilter),
-                                        e => e.into(),
-                                    });
-                                match candidates {
-                                    Ok(candidates) => to_delete |= candidates,
-                                    Err(err) => {
-                                        task.status = Status::Failed;
-                                        task.error = Some(err.into());
-                                    }
-                                };
-                            }
-                            let will_be_removed = to_delete.len() - before;
-                            if let Some(Details::DocumentDeletionByFilter {
-                                original_filter: _,
-                                deleted_documents,
-                            }) = &mut task.details
-                            {
-                                *deleted_documents = Some(will_be_removed);
-                            } else {
-                                // In the case of a `documentDeleteByFilter` the details MUST be set
-                                unreachable!()
-                            }
-                        }
-                        _ => unreachable!(),
-                    }
-                }
-
-                let config = IndexDocumentsConfig {
-                    update_method: IndexDocumentsMethod::ReplaceDocuments,
-                    ..Default::default()
-                };
-
-                let must_stop_processing = self.must_stop_processing.clone();
-                let mut builder = milli::update::IndexDocuments::new(
-                    index_wtxn,
-                    index,
-                    self.index_mapper.indexer_config(),
-                    config,
-                    |indexing_step| tracing::debug!(update = ?indexing_step),
-                    || must_stop_processing.get(),
-                )?;
-
-                let (new_builder, _count) =
-                    builder.remove_documents_from_db_no_batch(&to_delete)?;
-                builder = new_builder;
-
-                let _ = builder.execute()?;
-
-                Ok(tasks)
-            }
             IndexOperation::Settings { index_uid: _, settings, mut tasks } => {
                 let indexer_config = self.index_mapper.indexer_config();
                 let mut builder = milli::update::Settings::new(index_wtxn, index, indexer_config);
@@ -1734,25 +1641,21 @@ impl IndexScheduler {
     }
 }
 
-fn edit_documents_by_function<'a>(
+fn delete_document_by_filter<'a>(
     wtxn: &mut RwTxn<'a>,
-    filter: &Option<serde_json::Value>,
-    context: Option<Object>,
-    code: &str,
+    filter: &serde_json::Value,
     indexer_config: &IndexerConfig,
     must_stop_processing: MustStopProcessing,
     index: &'a Index,
-) -> Result<(u64, u64)> {
-    let candidates = match filter.as_ref().map(Filter::from_json) {
-        Some(Ok(Some(filter))) => filter.evaluate(wtxn, index).map_err(|err| match err {
+) -> Result<u64> {
+    let filter = Filter::from_json(filter)?;
+    Ok(if let Some(filter) = filter {
+        let candidates = filter.evaluate(wtxn, index).map_err(|err| match err {
             milli::Error::UserError(milli::UserError::InvalidFilter(_)) => {
                 Error::from(err).with_custom_error_code(Code::InvalidDocumentFilter)
             }
             e => e.into(),
-        })?,
-        None | Some(Ok(None)) => index.documents_ids(wtxn)?,
-        Some(Err(e)) => return Err(e.into()),
-    };
+        })?;
 
         let config = IndexDocumentsConfig {
             update_method: IndexDocumentsMethod::ReplaceDocuments,
@@ -1768,9 +1671,12 @@ fn edit_documents_by_function<'a>(
             || must_stop_processing.get(),
         )?;
 
-    let (new_builder, count) = builder.edit_documents(&candidates, context, code)?;
+        let (new_builder, count) = builder.remove_documents_from_db_no_batch(&candidates)?;
         builder = new_builder;
 
         let _ = builder.execute()?;
-    Ok(count.unwrap())
+        count
+    } else {
+        0
+    })
 }

index-scheduler/src/features.rs

@@ -68,32 +68,6 @@ impl RoFeatures {
             .into())
         }
     }
-
-    pub fn check_edit_documents_by_function(&self, disabled_action: &'static str) -> Result<()> {
-        if self.runtime.edit_documents_by_function {
-            Ok(())
-        } else {
-            Err(FeatureNotEnabledError {
-                disabled_action,
-                feature: "edit documents by function",
-                issue_link: "https://github.com/orgs/meilisearch/discussions/762",
-            }
-            .into())
-        }
-    }
-
-    pub fn check_contains_filter(&self) -> Result<()> {
-        if self.runtime.contains_filter {
-            Ok(())
-        } else {
-            Err(FeatureNotEnabledError {
-                disabled_action: "Using `CONTAINS` or `STARTS WITH` in a filter",
-                feature: "contains filter",
-                issue_link: "https://github.com/orgs/meilisearch/discussions/763",
-            }
-            .into())
-        }
-    }
 }
 
 impl FeatureData {
@@ -105,11 +79,9 @@ impl FeatureData {
         let txn = env.read_txn()?;
         let persisted_features: RuntimeTogglableFeatures =
             runtime_features_db.get(&txn, EXPERIMENTAL_FEATURES)?.unwrap_or_default();
-        let InstanceTogglableFeatures { metrics, logs_route, contains_filter } = instance_features;
         let runtime = Arc::new(RwLock::new(RuntimeTogglableFeatures {
-            metrics: metrics || persisted_features.metrics,
-            logs_route: logs_route || persisted_features.logs_route,
-            contains_filter: contains_filter || persisted_features.contains_filter,
+            metrics: instance_features.metrics || persisted_features.metrics,
+            logs_route: instance_features.logs_route || persisted_features.logs_route,
             ..persisted_features
         }));
 

index-scheduler/src/index_mapper/mod.rs

@@ -108,10 +108,8 @@ pub struct IndexStats {
     /// Association of every field name with the number of times it occurs in the documents.
     pub field_distribution: FieldDistribution,
     /// Creation date of the index.
-    #[serde(with = "time::serde::rfc3339")]
     pub created_at: OffsetDateTime,
     /// Date of the last update of the index.
-    #[serde(with = "time::serde::rfc3339")]
     pub updated_at: OffsetDateTime,
 }
 

index-scheduler/src/insta_snapshot.rs

@@ -11,9 +11,6 @@ use crate::index_mapper::IndexMapper;
 use crate::{IndexScheduler, Kind, Status, BEI128};
 
 pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
-    // Since we'll snapshot the index right afterward, we don't need to ensure it's internally consistent for every run.
-    // We can only do it for the release run, where the function runs way faster.
-    #[cfg(not(debug_assertions))]
     scheduler.assert_internally_consistent();
 
     let IndexScheduler {
@@ -180,17 +177,6 @@ fn snapshot_details(d: &Details) -> String {
         } => {
             format!("{{ received_documents: {received_documents}, indexed_documents: {indexed_documents:?} }}")
         }
-        Details::DocumentEdition {
-            deleted_documents,
-            edited_documents,
-            original_filter,
-            context,
-            function,
-        } => {
-            format!(
-                "{{ deleted_documents: {deleted_documents:?}, edited_documents: {edited_documents:?}, context: {context:?}, function: {function:?}, original_filter: {original_filter:?} }}"
-            )
-        }
         Details::SettingsUpdate { settings } => {
             format!("{{ settings: {settings:?} }}")
         }

index-scheduler/src/lib.rs

@@ -35,7 +35,6 @@ pub type TaskId = u32;
 use std::collections::{BTreeMap, HashMap};
 use std::io::{self, BufReader, Read};
 use std::ops::{Bound, RangeBounds};
-use std::panic::{catch_unwind, AssertUnwindSafe};
 use std::path::{Path, PathBuf};
 use std::sync::atomic::Ordering::{self, Relaxed};
 use std::sync::atomic::{AtomicBool, AtomicU32};
@@ -613,24 +612,19 @@ impl IndexScheduler {
                 #[cfg(test)]
                 run.breakpoint(Breakpoint::Init);
 
-                run.wake_up.wait_timeout(std::time::Duration::from_secs(60));
+                run.wake_up.wait();
 
                 loop {
-                    let ret = catch_unwind(AssertUnwindSafe(|| run.tick()));
-                    match ret {
-                        Ok(Ok(TickOutcome::TickAgain(_))) => (),
-                        Ok(Ok(TickOutcome::WaitForSignal)) => run.wake_up.wait(),
-                        Ok(Err(e)) => {
+                    match run.tick() {
+                        Ok(TickOutcome::TickAgain(_)) => (),
+                        Ok(TickOutcome::WaitForSignal) => run.wake_up.wait(),
+                        Err(e) => {
                             tracing::error!("{e}");
                             // Wait one second when an irrecoverable error occurs.
                             if !e.is_recoverable() {
                                 std::thread::sleep(Duration::from_secs(1));
                             }
                         }
-                        Err(_panic) => {
-                            tracing::error!("Internal error: Unexpected panic in the `IndexScheduler::run` method.");
-
-                        }
                     }
                 }
             })
@@ -668,11 +662,7 @@ impl IndexScheduler {
         let rtxn = self.env.read_txn()?;
         self.index_mapper.index(&rtxn, name)
     }
-    /// Return the boolean referring if index exists.
-    pub fn index_exists(&self, name: &str) -> Result<bool> {
-        let rtxn = self.env.read_txn()?;
-        self.index_mapper.index_exists(&rtxn, name)
-    }
+
     /// Return the name of all indexes without opening them.
     pub fn index_names(&self) -> Result<Vec<String>> {
         let rtxn = self.env.read_txn()?;
@@ -1263,7 +1253,7 @@ impl IndexScheduler {
                     #[cfg(test)]
                     self.maybe_fail(tests::FailureLocation::UpdatingTaskAfterProcessBatchFailure)?;
 
-                    tracing::error!("Batch failed {}", error);
+                    tracing::info!("Batch failed {}", error);
 
                     self.update_task(&mut wtxn, &task)
                         .map_err(|e| Error::TaskDatabaseUpdate(Box::new(e)))?;
@@ -1477,7 +1467,7 @@ impl IndexScheduler {
             .map(
                 |IndexEmbeddingConfig {
                      name,
-                     config: milli::vector::EmbeddingConfig { embedder_options, prompt, quantized },
+                     config: milli::vector::EmbeddingConfig { embedder_options, prompt },
                      ..
                  }| {
                     let prompt =
@@ -1486,10 +1476,7 @@ impl IndexScheduler {
                     {
                         let embedders = self.embedders.read().unwrap();
                         if let Some(embedder) = embedders.get(&embedder_options) {
-                            return Ok((
-                                name,
-                                (embedder.clone(), prompt, quantized.unwrap_or_default()),
-                            ));
+                            return Ok((name, (embedder.clone(), prompt)));
                         }
                     }
 
@@ -1503,7 +1490,7 @@ impl IndexScheduler {
                         let mut embedders = self.embedders.write().unwrap();
                         embedders.insert(embedder_options, embedder.clone());
                     }
-                    Ok((name, (embedder, prompt, quantized.unwrap_or_default())))
+                    Ok((name, (embedder, prompt)))
                 },
             )
             .collect();
@@ -1616,14 +1603,6 @@ impl<'a> Dump<'a> {
                         index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
                     }
                 }
-                KindDump::DocumentEdition { filter, context, function } => {
-                    KindWithContent::DocumentEdition {
-                        index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
-                        filter_expr: filter,
-                        context,
-                        function,
-                    }
-                }
                 KindDump::DocumentClear => KindWithContent::DocumentClear {
                     index_uid: task.index_uid.ok_or(Error::CorruptedDump)?,
                 },
@@ -1767,7 +1746,6 @@ mod tests {
     use crossbeam::channel::RecvTimeoutError;
     use file_store::File;
     use insta::assert_json_snapshot;
-    use maplit::btreeset;
     use meili_snap::{json_string, snapshot};
     use meilisearch_auth::AuthFilter;
     use meilisearch_types::document_formats::DocumentFormatError;
@@ -2010,13 +1988,11 @@ mod tests {
         fn advance_till(&mut self, breakpoints: impl IntoIterator<Item = Breakpoint>) {
             for breakpoint in breakpoints {
                 let b = self.advance();
+                let state = snapshot_index_scheduler(&self.index_scheduler);
                 assert_eq!(
-                    b,
-                    breakpoint,
-                    "Was expecting the breakpoint `{:?}` but instead got `{:?}`.\n{}",
-                    breakpoint,
-                    b,
-                    snapshot_index_scheduler(&self.index_scheduler)
+                    b, breakpoint,
+                    "Was expecting the breakpoint `{:?}` but instead got `{:?}`.\n{state}",
+                    breakpoint, b
                 );
             }
         }
@@ -2040,6 +2016,7 @@ mod tests {
         // Wait for one successful batch.
         #[track_caller]
         fn advance_one_successful_batch(&mut self) {
+            self.index_scheduler.assert_internally_consistent();
             self.advance_till([Start, BatchCreated]);
             loop {
                 match self.advance() {
@@ -2058,6 +2035,7 @@ mod tests {
             }
 
             self.advance_till([AfterProcessing]);
+            self.index_scheduler.assert_internally_consistent();
         }
 
         // Wait for one failed batch.
@@ -2487,12 +2465,20 @@ mod tests {
 
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -2547,127 +2533,24 @@ mod tests {
 
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
 
-    #[test]
-    fn fail_in_process_batch_for_document_deletion() {
-        let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
-
-        use meilisearch_types::settings::{Settings, Unchecked};
-        let mut new_settings: Box<Settings<Unchecked>> = Box::default();
-        new_settings.filterable_attributes = Setting::Set(btreeset!(S("catto")));
-
-        index_scheduler
-            .register(
-                KindWithContent::SettingsUpdate {
-                    index_uid: S("doggos"),
-                    new_settings,
-                    is_deletion: false,
-                    allow_index_creation: true,
-                },
-                None,
-                false,
-            )
-            .unwrap();
-
-        let content = r#"[
-            { "id": 1, "doggo": "jean bob" },
-            { "id": 2, "catto": "jorts" },
-            { "id": 3, "doggo": "bork" }
-        ]"#;
-
-        let (uuid, mut file) = index_scheduler.create_update_file_with_uuid(0).unwrap();
-        let documents_count = read_json(content.as_bytes(), &mut file).unwrap();
-        file.persist().unwrap();
-        index_scheduler
-            .register(
-                KindWithContent::DocumentAdditionOrUpdate {
-                    index_uid: S("doggos"),
-                    primary_key: Some(S("id")),
-                    method: ReplaceDocuments,
-                    content_file: uuid,
-                    documents_count,
-                    allow_index_creation: true,
-                },
-                None,
-                false,
-            )
-            .unwrap();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_setting_and_document_addition");
-
-        handle.advance_one_successful_batch();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_adding_the_settings");
-        handle.advance_one_successful_batch();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_adding_the_documents");
-
-        index_scheduler
-            .register(
-                KindWithContent::DocumentDeletion {
-                    index_uid: S("doggos"),
-                    documents_ids: vec![S("1")],
-                },
-                None,
-                false,
-            )
-            .unwrap();
-        // This one should not be catched by Meilisearch but it's still nice to handle it because if one day we break the filters it could happens
-        index_scheduler
-            .register(
-                KindWithContent::DocumentDeletionByFilter {
-                    index_uid: S("doggos"),
-                    filter_expr: serde_json::json!(true),
-                },
-                None,
-                false,
-            )
-            .unwrap();
-        // Should fail because the ids are not filterable
-        index_scheduler
-            .register(
-                KindWithContent::DocumentDeletionByFilter {
-                    index_uid: S("doggos"),
-                    filter_expr: serde_json::json!("id = 2"),
-                },
-                None,
-                false,
-            )
-            .unwrap();
-        index_scheduler
-            .register(
-                KindWithContent::DocumentDeletionByFilter {
-                    index_uid: S("doggos"),
-                    filter_expr: serde_json::json!("catto EXISTS"),
-                },
-                None,
-                false,
-            )
-            .unwrap();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "registered_the_document_deletions");
-
-        // Everything should be batched together
-        handle.advance_one_successful_batch();
-        snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_removing_the_documents");
-
-        let index = index_scheduler.index("doggos").unwrap();
-        let rtxn = index.read_txn().unwrap();
-        let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
-        let field_ids = field_ids_map.ids().collect::<Vec<_>>();
-        let documents = index
-            .all_documents(&rtxn)
-            .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
-            .collect::<Vec<_>>();
-        snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents_remaining_should_only_be_bork");
-    }
-
     #[test]
     fn do_not_batch_task_of_different_indexes() {
         let (index_scheduler, mut handle) = IndexScheduler::test(true, vec![]);
@@ -3037,12 +2920,20 @@ mod tests {
         // has everything being pushed successfully in milli?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -3088,12 +2979,20 @@ mod tests {
         // has everything being pushed successfully in milli?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -3144,12 +3043,20 @@ mod tests {
         // has everything being pushed successfully in milli?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -3168,8 +3075,6 @@ mod tests {
             api_key: Setting::Set(S("My super secret")),
             url: Setting::Set(S("http://localhost:7777")),
             dimensions: Setting::Set(4),
-            request: Setting::Set(serde_json::json!("{{text}}")),
-            response: Setting::Set(serde_json::json!("{{embedding}}")),
             ..Default::default()
         };
         embedders.insert(S("default"), Setting::Set(embedding_settings));
@@ -3264,12 +3169,20 @@ mod tests {
         // has everything being pushed successfully in milli?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -3319,12 +3232,20 @@ mod tests {
         // has everything being pushed successfully in milli?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -3922,15 +3843,15 @@ mod tests {
         ]);
         snapshot!(snapshot_index_scheduler(&index_scheduler), name: "after_processing_the_10_tasks");
 
-        // The index should not exist.
-        snapshot!(matches!(index_scheduler.index_exists("doggos"), Ok(true)), @"false");
+        // The index should not exists.
+        snapshot!(format!("{}", index_scheduler.index("doggos").map(|_| ()).unwrap_err()), @"Index `doggos` not found.");
     }
 
     #[test]
     fn test_document_addition_cant_create_index_without_index_without_autobatching() {
         // We're going to execute multiple document addition that don't have
         // the right to create an index while there is no index currently.
-        // Since the auto-batching is disabled, every task should be processed
+        // Since the autobatching is disabled, every tasks should be processed
         // sequentially and throw an IndexDoesNotExists.
         let (index_scheduler, mut handle) = IndexScheduler::test(false, vec![]);
 
@@ -3972,8 +3893,8 @@ mod tests {
         handle.advance_n_failed_batches(5);
         snapshot!(snapshot_index_scheduler(&index_scheduler), name: "all_tasks_processed");
 
-        // The index should not exist.
-        snapshot!(matches!(index_scheduler.index_exists("doggos"), Ok(true)), @"false");
+        // The index should not exists.
+        snapshot!(format!("{}", index_scheduler.index("doggos").map(|_| ()).unwrap_err()), @"Index `doggos` not found.");
     }
 
     #[test]
@@ -4033,12 +3954,20 @@ mod tests {
         // Has everything being pushed successfully in milli?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -4104,12 +4033,20 @@ mod tests {
         // Has everything being pushed successfully in milli?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -4172,12 +4109,20 @@ mod tests {
         // Has everything being pushed successfully in milli?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -4233,12 +4178,20 @@ mod tests {
         // Has everything being pushed successfully in milli?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -4294,6 +4247,8 @@ mod tests {
         // Is the primary key still what we expect?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let primary_key = index.primary_key(&rtxn).unwrap().unwrap();
         snapshot!(primary_key, @"id");
 
@@ -4301,9 +4256,15 @@ mod tests {
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -4355,6 +4316,8 @@ mod tests {
         // Is the primary key still what we expect?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let primary_key = index.primary_key(&rtxn).unwrap().unwrap();
         snapshot!(primary_key, @"id");
 
@@ -4362,9 +4325,15 @@ mod tests {
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -4438,6 +4407,8 @@ mod tests {
         // Is the primary key still what we expect?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let primary_key = index.primary_key(&rtxn).unwrap().unwrap();
         snapshot!(primary_key, @"id");
 
@@ -4445,9 +4416,15 @@ mod tests {
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -4524,6 +4501,8 @@ mod tests {
         // Is the primary key still what we expect?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let primary_key = index.primary_key(&rtxn).unwrap().unwrap();
         snapshot!(primary_key, @"paw");
 
@@ -4531,9 +4510,15 @@ mod tests {
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -4603,6 +4588,8 @@ mod tests {
         // Is the primary key still what we expect?
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let primary_key = index.primary_key(&rtxn).unwrap().unwrap();
         snapshot!(primary_key, @"doggoid");
 
@@ -4610,9 +4597,15 @@ mod tests {
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string_pretty(&documents).unwrap(), name: "documents");
     }
@@ -4879,7 +4872,6 @@ mod tests {
           "types": {
             "documentAdditionOrUpdate": 0,
             "documentDeletion": 0,
-            "documentEdition": 0,
             "dumpCreation": 0,
             "indexCreation": 3,
             "indexDeletion": 0,
@@ -4911,7 +4903,6 @@ mod tests {
           "types": {
             "documentAdditionOrUpdate": 0,
             "documentDeletion": 0,
-            "documentEdition": 0,
             "dumpCreation": 0,
             "indexCreation": 3,
             "indexDeletion": 0,
@@ -4950,7 +4941,6 @@ mod tests {
           "types": {
             "documentAdditionOrUpdate": 0,
             "documentDeletion": 0,
-            "documentEdition": 0,
             "dumpCreation": 0,
             "indexCreation": 3,
             "indexDeletion": 0,
@@ -4990,7 +4980,6 @@ mod tests {
           "types": {
             "documentAdditionOrUpdate": 0,
             "documentDeletion": 0,
-            "documentEdition": 0,
             "dumpCreation": 0,
             "indexCreation": 3,
             "indexDeletion": 0,
@@ -5129,8 +5118,6 @@ mod tests {
             api_key: Setting::Set(S("My super secret")),
             url: Setting::Set(S("http://localhost:7777")),
             dimensions: Setting::Set(384),
-            request: Setting::Set(serde_json::json!("{{text}}")),
-            response: Setting::Set(serde_json::json!("{{embedding}}")),
             ..Default::default()
         };
         embedders.insert(S("A_fakerest"), Setting::Set(embedding_settings));
@@ -5200,11 +5187,10 @@ mod tests {
             let simple_hf_name = name.clone();
 
             let configs = index_scheduler.embedders(configs).unwrap();
-            let (hf_embedder, _, _) = configs.get(&simple_hf_name).unwrap();
-            let beagle_embed =
-                hf_embedder.embed_one(S("Intel the beagle best doggo"), None).unwrap();
-            let lab_embed = hf_embedder.embed_one(S("Max the lab best doggo"), None).unwrap();
-            let patou_embed = hf_embedder.embed_one(S("kefir the patou best doggo"), None).unwrap();
+            let (hf_embedder, _) = configs.get(&simple_hf_name).unwrap();
+            let beagle_embed = hf_embedder.embed_one(S("Intel the beagle best doggo")).unwrap();
+            let lab_embed = hf_embedder.embed_one(S("Max the lab best doggo")).unwrap();
+            let patou_embed = hf_embedder.embed_one(S("kefir the patou best doggo")).unwrap();
             (fakerest_name, simple_hf_name, beagle_embed, lab_embed, patou_embed)
         };
 
@@ -5262,6 +5248,8 @@ mod tests {
         {
             let index = index_scheduler.index("doggos").unwrap();
             let rtxn = index.read_txn().unwrap();
+            let mut buffer = Vec::new();
+            let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
 
             // Ensure the document have been inserted into the relevant bitamp
             let configs = index.embedding_configs(&rtxn).unwrap();
@@ -5281,8 +5269,12 @@ mod tests {
             assert_json_snapshot!(embeddings[&simple_hf_name][0] == lab_embed, @"true");
             assert_json_snapshot!(embeddings[&fakerest_name][0] == beagle_embed, @"true");
 
-            let doc = index.documents(&rtxn, std::iter::once(0)).unwrap()[0].1;
+            let (_id, compressed_doc) =
+                index.compressed_documents(&rtxn, std::iter::once(0)).unwrap().remove(0);
             let fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
+            let doc = compressed_doc
+                .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                .unwrap();
             let doc = obkv_to_json(
                 &[
                     fields_ids_map.id("doggo").unwrap(),
@@ -5336,6 +5328,8 @@ mod tests {
             {
                 let index = index_scheduler.index("doggos").unwrap();
                 let rtxn = index.read_txn().unwrap();
+                let mut buffer = Vec::new();
+                let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
 
                 // Ensure the document have been inserted into the relevant bitamp
                 let configs = index.embedding_configs(&rtxn).unwrap();
@@ -5358,8 +5352,12 @@ mod tests {
                 // remained beagle
                 assert_json_snapshot!(embeddings[&fakerest_name][0] == beagle_embed, @"true");
 
-                let doc = index.documents(&rtxn, std::iter::once(0)).unwrap()[0].1;
+                let (_id, compressed_doc) =
+                    index.compressed_documents(&rtxn, std::iter::once(0)).unwrap().remove(0);
                 let fields_ids_map = index.fields_ids_map(&rtxn).unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
                 let doc = obkv_to_json(
                     &[
                         fields_ids_map.id("doggo").unwrap(),
@@ -5451,12 +5449,20 @@ mod tests {
 
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string(&documents).unwrap(), name: "documents after initial push");
 
@@ -5490,12 +5496,20 @@ mod tests {
 
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         // the all the vectors linked to the new specified embedder have been removed
         // Only the unknown embedders stays in the document DB
@@ -5519,11 +5533,7 @@ mod tests {
                     ),
                     prompt: PromptData {
                         template: "{{doc.doggo}}",
-                        max_bytes: Some(
-                            400,
-                        ),
                     },
-                    quantized: None,
                 },
                 user_provided: RoaringBitmap<[1, 2]>,
             },
@@ -5536,8 +5546,28 @@ mod tests {
 
         // the document with the id 3 should keep its original embedding
         let docid = index.external_documents_ids.get(&rtxn, "3").unwrap().unwrap();
-        let embeddings = index.embeddings(&rtxn, docid).unwrap();
-        let embeddings = &embeddings["my_doggo_embedder"];
+        let mut embeddings = Vec::new();
+
+        'vectors: for i in 0..=u8::MAX {
+            let reader = arroy::Reader::open(&rtxn, i as u16, index.vector_arroy)
+                .map(Some)
+                .or_else(|e| match e {
+                    arroy::Error::MissingMetadata(_) => Ok(None),
+                    e => Err(e),
+                })
+                .transpose();
+
+            let Some(reader) = reader else {
+                break 'vectors;
+            };
+
+            let embedding = reader.unwrap().item_vector(&rtxn, docid).unwrap();
+            if let Some(embedding) = embedding {
+                embeddings.push(embedding)
+            } else {
+                break 'vectors;
+            }
+        }
 
         snapshot!(embeddings.len(), @"1");
         assert!(embeddings[0].iter().all(|i| *i == 3.0), "{:?}", embeddings[0]);
@@ -5582,9 +5612,15 @@ mod tests {
 
         // the document with the id 3 should have its original embedding updated
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let docid = index.external_documents_ids.get(&rtxn, "3").unwrap().unwrap();
-        let doc = index.documents(&rtxn, Some(docid)).unwrap()[0];
-        let doc = obkv_to_json(&field_ids, &field_ids_map, doc.1).unwrap();
+        let (_id, compressed_doc) =
+            index.compressed_documents(&rtxn, Some(docid)).unwrap().remove(0);
+        let doc = compressed_doc
+            .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+            .unwrap();
+        let doc = obkv_to_json(&field_ids, &field_ids_map, doc).unwrap();
         snapshot!(json_string!(doc), @r###"
         {
           "id": 3,
@@ -5696,12 +5732,20 @@ mod tests {
 
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string(&documents).unwrap(), @r###"[{"id":0,"doggo":"kefir"}]"###);
         let conf = index.embedding_configs(&rtxn).unwrap();
@@ -5717,12 +5761,8 @@ mod tests {
                         },
                     ),
                     prompt: PromptData {
-                        template: "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
-                        max_bytes: Some(
-                            400,
-                        ),
+                        template: "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}",
                     },
-                    quantized: None,
                 },
                 user_provided: RoaringBitmap<[0]>,
             },
@@ -5740,12 +5780,20 @@ mod tests {
 
         let index = index_scheduler.index("doggos").unwrap();
         let rtxn = index.read_txn().unwrap();
+        let mut buffer = Vec::new();
+        let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
         let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
         let field_ids = field_ids_map.ids().collect::<Vec<_>>();
         let documents = index
-            .all_documents(&rtxn)
+            .all_compressed_documents(&rtxn)
             .unwrap()
-            .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+            .map(|ret| {
+                let (_id, compressed_doc) = ret.unwrap();
+                let doc = compressed_doc
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                    .unwrap();
+                obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+            })
             .collect::<Vec<_>>();
         snapshot!(serde_json::to_string(&documents).unwrap(), @"[]");
         let conf = index.embedding_configs(&rtxn).unwrap();
@@ -5761,12 +5809,8 @@ mod tests {
                         },
                     ),
                     prompt: PromptData {
-                        template: "{% for field in fields %}{% if field.is_searchable and field.value != nil %}{{ field.name }}: {{ field.value }}\n{% endif %}{% endfor %}",
-                        max_bytes: Some(
-                            400,
-                        ),
+                        template: "{% for field in fields %} {{ field.name }}: {{ field.value }}\n{% endfor %}",
                     },
-                    quantized: None,
                 },
                 user_provided: RoaringBitmap<[]>,
             },
@@ -5860,12 +5904,20 @@ mod tests {
         {
             let index = index_scheduler.index("doggos").unwrap();
             let rtxn = index.read_txn().unwrap();
+            let mut buffer = Vec::new();
+            let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
             let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
             let field_ids = field_ids_map.ids().collect::<Vec<_>>();
             let documents = index
-                .all_documents(&rtxn)
+                .all_compressed_documents(&rtxn)
                 .unwrap()
-                .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+                .map(|ret| {
+                    let (_id, compressed_doc) = ret.unwrap();
+                    let doc = compressed_doc
+                        .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                        .unwrap();
+                    obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+                })
                 .collect::<Vec<_>>();
             snapshot!(serde_json::to_string(&documents).unwrap(), @r###"[{"id":0,"doggo":"kefir"},{"id":1,"doggo":"intel"}]"###);
         }
@@ -5895,12 +5947,20 @@ mod tests {
         {
             let index = index_scheduler.index("doggos").unwrap();
             let rtxn = index.read_txn().unwrap();
+            let mut buffer = Vec::new();
+            let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
             let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
             let field_ids = field_ids_map.ids().collect::<Vec<_>>();
             let documents = index
-                .all_documents(&rtxn)
+                .all_compressed_documents(&rtxn)
                 .unwrap()
-                .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+                .map(|ret| {
+                    let (_id, compressed_doc) = ret.unwrap();
+                    let doc = compressed_doc
+                        .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                        .unwrap();
+                    obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+                })
                 .collect::<Vec<_>>();
             snapshot!(serde_json::to_string(&documents).unwrap(), @r###"[{"id":0,"doggo":"kefir","_vectors":{"manual":{"embeddings":[[0.0,0.0,0.0]],"regenerate":false}}},{"id":1,"doggo":"intel","_vectors":{"manual":{"embeddings":[[1.0,1.0,1.0]],"regenerate":false}}}]"###);
         }
@@ -5928,12 +5988,20 @@ mod tests {
         {
             let index = index_scheduler.index("doggos").unwrap();
             let rtxn = index.read_txn().unwrap();
+            let mut buffer = Vec::new();
+            let dictionary = index.document_decompression_dictionary(&rtxn).unwrap();
             let field_ids_map = index.fields_ids_map(&rtxn).unwrap();
             let field_ids = field_ids_map.ids().collect::<Vec<_>>();
             let documents = index
-                .all_documents(&rtxn)
+                .all_compressed_documents(&rtxn)
                 .unwrap()
-                .map(|ret| obkv_to_json(&field_ids, &field_ids_map, ret.unwrap().1).unwrap())
+                .map(|ret| {
+                    let (_id, compressed_doc) = ret.unwrap();
+                    let doc = compressed_doc
+                        .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())
+                        .unwrap();
+                    obkv_to_json(&field_ids, &field_ids_map, doc).unwrap()
+                })
                 .collect::<Vec<_>>();
 
             // FIXME: redaction

index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-2.snap

@@ -8,9 +8,7 @@ expression: task.details
       "source": "rest",
       "apiKey": "MyXXXX...",
       "dimensions": 384,
-      "url": "http://localhost:7777",
-      "request": "{{text}}",
-      "response": "{{embedding}}"
+      "url": "http://localhost:7777"
     },
     "B_small_hf": {
       "source": "huggingFace",

index-scheduler/src/snapshots/index_scheduler__tests__import_vectors-5.snap

@@ -8,8 +8,16 @@ expression: fakerest_config.embedder_options
     "distribution": null,
     "dimensions": 384,
     "url": "http://localhost:7777",
-    "request": "{{text}}",
-    "response": "{{embedding}}",
-    "headers": {}
+    "query": null,
+    "input_field": [
+      "input"
+    ],
+    "path_to_embeddings": [
+      "data"
+    ],
+    "embedding_object": [
+      "embedding"
+    ],
+    "input_type": "text"
   }
 }

index-scheduler/src/snapshots/index_scheduler__tests__import_vectors.snap

@@ -8,9 +8,7 @@ expression: task.details
       "source": "rest",
       "apiKey": "MyXXXX...",
       "dimensions": 384,
-      "url": "http://localhost:7777",
-      "request": "{{text}}",
-      "response": "{{embedding}}"
+      "url": "http://localhost:7777"
     },
     "B_small_hf": {
       "source": "huggingFace",

index-scheduler/src/snapshots/index_scheduler__tests__settings_update-2.snap

@@ -8,9 +8,7 @@ expression: task.details
       "source": "rest",
       "apiKey": "MyXXXX...",
       "dimensions": 4,
-      "url": "http://localhost:7777",
-      "request": "{{text}}",
-      "response": "{{embedding}}"
+      "url": "http://localhost:7777"
     }
   }
 }

index-scheduler/src/snapshots/index_scheduler__tests__settings_update-5.snap

@@ -1,6 +1,6 @@
 ---
 source: index-scheduler/src/lib.rs
-expression: config.embedder_options
+expression: embedding_config.embedder_options
 ---
 {
   "Rest": {
@@ -8,8 +8,16 @@ expression: config.embedder_options
     "distribution": null,
     "dimensions": 4,
     "url": "http://localhost:7777",
-    "request": "{{text}}",
-    "response": "{{embedding}}",
-    "headers": {}
+    "query": null,
+    "input_field": [
+      "input"
+    ],
+    "path_to_embeddings": [
+      "data"
+    ],
+    "embedding_object": [
+      "embedding"
+    ],
+    "input_type": "text"
   }
 }

index-scheduler/src/snapshots/index_scheduler__tests__settings_update.snap

@@ -8,9 +8,7 @@ expression: task.details
       "source": "rest",
       "apiKey": "MyXXXX...",
       "dimensions": 4,
-      "url": "http://localhost:7777",
-      "request": "{{text}}",
-      "response": "{{embedding}}"
+      "url": "http://localhost:7777"
     }
   }
 }

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_documents.snap

@@ -1,44 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
-1 {uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-----------------------------------------------------------------------
-### Status:
-enqueued []
-succeeded [0,1,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [1,]
-"settingsUpdate" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-doggos: { number_of_documents: 3, field_distribution: {"catto": 1, "doggo": 2, "id": 3} }
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Finished At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### File Store:
-
-----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings.snap

@@ -1,43 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
-1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-----------------------------------------------------------------------
-### Status:
-enqueued [1,]
-succeeded [0,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [1,]
-"settingsUpdate" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-doggos: { number_of_documents: 0, field_distribution: {} }
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-[timestamp] [0,]
-----------------------------------------------------------------------
-### Finished At:
-[timestamp] [0,]
-----------------------------------------------------------------------
-### File Store:
-00000000-0000-0000-0000-000000000000
-
-----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_adding_the_settings_and_documents.snap

@@ -1,43 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
-1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-----------------------------------------------------------------------
-### Status:
-enqueued [1,]
-succeeded [0,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [1,]
-"settingsUpdate" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-doggos: { number_of_documents: 0, field_distribution: {} }
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-[timestamp] [0,]
-----------------------------------------------------------------------
-### Finished At:
-[timestamp] [0,]
-----------------------------------------------------------------------
-### File Store:
-00000000-0000-0000-0000-000000000000
-
-----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/after_removing_the_documents.snap

@@ -1,56 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
-1 {uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-2 {uid: 2, status: succeeded, details: { received_document_ids: 1, deleted_documents: Some(1) }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1"] }}
-3 {uid: 3, status: failed, error: ResponseError { code: 200, message: "Invalid type for filter subexpression: expected: String, Array, found: true.", error_code: "invalid_document_filter", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#invalid_document_filter" }, details: { original_filter: true, deleted_documents: Some(0) }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: Bool(true) }}
-4 {uid: 4, status: failed, error: ResponseError { code: 200, message: "Attribute `id` is not filterable. Available filterable attributes are: `catto`.\n1:3 id = 2", error_code: "invalid_document_filter", error_type: "invalid_request", error_link: "https://docs.meilisearch.com/errors#invalid_document_filter" }, details: { original_filter: "id = 2", deleted_documents: Some(0) }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: String("id = 2") }}
-5 {uid: 5, status: succeeded, details: { original_filter: "catto EXISTS", deleted_documents: Some(1) }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: String("catto EXISTS") }}
-----------------------------------------------------------------------
-### Status:
-enqueued []
-succeeded [0,1,2,5,]
-failed [3,4,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [1,]
-"documentDeletion" [2,3,4,5,]
-"settingsUpdate" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,2,3,4,5,]
-----------------------------------------------------------------------
-### Index Mapper:
-doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-[timestamp] [2,]
-[timestamp] [3,]
-[timestamp] [4,]
-[timestamp] [5,]
-----------------------------------------------------------------------
-### Started At:
-[timestamp] [0,]
-[timestamp] [1,]
-[timestamp] [2,3,4,5,]
-----------------------------------------------------------------------
-### Finished At:
-[timestamp] [0,]
-[timestamp] [1,]
-[timestamp] [2,3,4,5,]
-----------------------------------------------------------------------
-### File Store:
-
-----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/documents_remaining_should_only_be_bork.snap

@@ -1,9 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-[
-  {
-    "id": 3,
-    "doggo": "bork"
-  }
-]

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_document_deletions.snap

@@ -1,53 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
-1 {uid: 1, status: succeeded, details: { received_documents: 3, indexed_documents: Some(3) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-2 {uid: 2, status: enqueued, details: { received_document_ids: 1, deleted_documents: None }, kind: DocumentDeletion { index_uid: "doggos", documents_ids: ["1"] }}
-3 {uid: 3, status: enqueued, details: { original_filter: true, deleted_documents: None }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: Bool(true) }}
-4 {uid: 4, status: enqueued, details: { original_filter: "id = 2", deleted_documents: None }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: String("id = 2") }}
-5 {uid: 5, status: enqueued, details: { original_filter: "catto EXISTS", deleted_documents: None }, kind: DocumentDeletionByFilter { index_uid: "doggos", filter_expr: String("catto EXISTS") }}
-----------------------------------------------------------------------
-### Status:
-enqueued [2,3,4,5,]
-succeeded [0,1,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [1,]
-"documentDeletion" [2,3,4,5,]
-"settingsUpdate" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,2,3,4,5,]
-----------------------------------------------------------------------
-### Index Mapper:
-doggos: { number_of_documents: 3, field_distribution: {"catto": 1, "doggo": 2, "id": 3} }
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-[timestamp] [2,]
-[timestamp] [3,]
-[timestamp] [4,]
-[timestamp] [5,]
-----------------------------------------------------------------------
-### Started At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Finished At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### File Store:
-
-----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/fail_in_process_batch_for_document_deletion/registered_the_setting_and_document_addition.snap

@@ -1,39 +0,0 @@
----
-source: index-scheduler/src/lib.rs
----
-### Autobatching Enabled = true
-### Processing Tasks:
-[]
-----------------------------------------------------------------------
-### All Tasks:
-0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: Set({"catto"}), sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: NotSet, search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
-1 {uid: 1, status: enqueued, details: { received_documents: 3, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: ReplaceDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 3, allow_index_creation: true }}
-----------------------------------------------------------------------
-### Status:
-enqueued [0,1,]
-----------------------------------------------------------------------
-### Kind:
-"documentAdditionOrUpdate" [1,]
-"settingsUpdate" [0,]
-----------------------------------------------------------------------
-### Index Tasks:
-doggos [0,1,]
-----------------------------------------------------------------------
-### Index Mapper:
-
-----------------------------------------------------------------------
-### Canceled By:
-
-----------------------------------------------------------------------
-### Enqueued At:
-[timestamp] [0,]
-[timestamp] [1,]
-----------------------------------------------------------------------
-### Started At:
-----------------------------------------------------------------------
-### Finished At:
-----------------------------------------------------------------------
-### File Store:
-00000000-0000-0000-0000-000000000000
-
-----------------------------------------------------------------------

index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir succeeds.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: None, method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
@@ -46,3 +46,4 @@ doggos: { number_of_documents: 1, field_distribution: {"_vectors": 1, "breed": 1
 ### File Store:
 
 ----------------------------------------------------------------------
+

index-scheduler/src/snapshots/lib.rs/import_vectors/Intel to kefir.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 2 {uid: 2, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: None, method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000001, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
@@ -45,3 +45,4 @@ doggos: { number_of_documents: 1, field_distribution: {"_vectors": 1, "breed": 1
 00000000-0000-0000-0000-000000000001
 
 ----------------------------------------------------------------------
+

index-scheduler/src/snapshots/lib.rs/import_vectors/adding Intel succeeds.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: succeeded, details: { received_documents: 1, indexed_documents: Some(1) }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
@@ -42,3 +42,4 @@ doggos: { number_of_documents: 1, field_distribution: {"_vectors": 1, "breed": 1
 ### File Store:
 
 ----------------------------------------------------------------------
+

index-scheduler/src/snapshots/lib.rs/import_vectors/after adding Intel.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 1 {uid: 1, status: enqueued, details: { received_documents: 1, indexed_documents: None }, kind: DocumentAdditionOrUpdate { index_uid: "doggos", primary_key: Some("id"), method: UpdateDocuments, content_file: 00000000-0000-0000-0000-000000000000, documents_count: 1, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
@@ -41,3 +41,4 @@ doggos: { number_of_documents: 0, field_distribution: {} }
 00000000-0000-0000-0000-000000000000
 
 ----------------------------------------------------------------------
+

index-scheduler/src/snapshots/lib.rs/import_vectors/after_registering_settings_task_vectors.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]
@@ -33,3 +33,4 @@ doggos [0,]
 ### File Store:
 
 ----------------------------------------------------------------------
+

index-scheduler/src/snapshots/lib.rs/import_vectors/settings_update_processed_vectors.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, binary_quantized: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), document_template_max_bytes: NotSet, url: NotSet, request: NotSet, response: NotSet, headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"A_fakerest": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(384), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet }), "B_small_hf": Set(EmbeddingSettings { source: Set(HuggingFace), model: Set("sentence-transformers/all-MiniLM-L6-v2"), revision: Set("e4ce9877abf3edfe10b0d82785e83bdcb973e22e"), api_key: NotSet, dimensions: NotSet, document_template: Set("{{doc.doggo}} the {{doc.breed}} best doggo"), url: NotSet, query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []
@@ -37,3 +37,4 @@ doggos: { number_of_documents: 0, field_distribution: {} }
 ### File Store:
 
 ----------------------------------------------------------------------
+

index-scheduler/src/snapshots/lib.rs/test_settings_update/after_registering_settings_task.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: enqueued, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued [0,]
@@ -33,3 +33,4 @@ doggos [0,]
 ### File Store:
 
 ----------------------------------------------------------------------
+

index-scheduler/src/snapshots/lib.rs/test_settings_update/settings_update_processed.snap

@@ -6,7 +6,7 @@ source: index-scheduler/src/lib.rs
 []
 ----------------------------------------------------------------------
 ### All Tasks:
-0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), binary_quantized: NotSet, document_template: NotSet, document_template_max_bytes: NotSet, url: Set("http://localhost:7777"), request: Set(String("{{text}}")), response: Set(String("{{embedding}}")), headers: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, localized_attributes: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
+0 {uid: 0, status: succeeded, details: { settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> } }, kind: SettingsUpdate { index_uid: "doggos", new_settings: Settings { displayed_attributes: WildcardSetting(NotSet), searchable_attributes: WildcardSetting(NotSet), filterable_attributes: NotSet, sortable_attributes: NotSet, ranking_rules: NotSet, stop_words: NotSet, non_separator_tokens: NotSet, separator_tokens: NotSet, dictionary: NotSet, synonyms: NotSet, distinct_attribute: NotSet, proximity_precision: NotSet, typo_tolerance: NotSet, faceting: NotSet, pagination: NotSet, embedders: Set({"default": Set(EmbeddingSettings { source: Set(Rest), model: NotSet, revision: NotSet, api_key: Set("My super secret"), dimensions: Set(4), document_template: NotSet, url: Set("http://localhost:7777"), query: NotSet, input_field: NotSet, path_to_embeddings: NotSet, embedding_object: NotSet, input_type: NotSet, distribution: NotSet })}), search_cutoff_ms: NotSet, _kind: PhantomData<meilisearch_types::settings::Unchecked> }, is_deletion: false, allow_index_creation: true }}
 ----------------------------------------------------------------------
 ### Status:
 enqueued []
@@ -37,3 +37,4 @@ doggos: { number_of_documents: 0, field_distribution: {} }
 ### File Store:
 
 ----------------------------------------------------------------------
+

index-scheduler/src/utils.rs

@@ -238,7 +238,6 @@ pub fn swap_index_uid_in_task(task: &mut Task, swap: (&str, &str)) {
     let mut index_uids = vec![];
     match &mut task.kind {
         K::DocumentAdditionOrUpdate { index_uid, .. } => index_uids.push(index_uid),
-        K::DocumentEdition { index_uid, .. } => index_uids.push(index_uid),
         K::DocumentDeletion { index_uid, .. } => index_uids.push(index_uid),
         K::DocumentDeletionByFilter { index_uid, .. } => index_uids.push(index_uid),
         K::DocumentClear { index_uid } => index_uids.push(index_uid),
@@ -409,26 +408,7 @@ impl IndexScheduler {
                                 match status {
                                     Status::Succeeded => assert!(indexed_documents <= received_documents),
                                     Status::Failed | Status::Canceled => assert_eq!(indexed_documents, 0),
-                                    status => panic!("DocumentAddition can't have an indexed_documents set if it's {}", status),
-                                }
-                            }
-                            None => {
-                                assert!(matches!(status, Status::Enqueued | Status::Processing))
-                            }
-                        }
-                    }
-                    Details::DocumentEdition { edited_documents, .. } => {
-                        assert_eq!(kind.as_kind(), Kind::DocumentEdition);
-                        match edited_documents {
-                            Some(edited_documents) => {
-                                assert!(matches!(
-                                    status,
-                                    Status::Succeeded | Status::Failed | Status::Canceled
-                                ));
-                                match status {
-                                    Status::Succeeded => (),
-                                    Status::Failed | Status::Canceled => assert_eq!(edited_documents, 0),
-                                    status => panic!("DocumentEdition can't have an edited_documents set if it's {}", status),
+                                    status => panic!("DocumentAddition can't have an indexed_document set if it's {}", status),
                                 }
                             }
                             None => {

meilisearch-types/Cargo.toml

@@ -66,8 +66,3 @@ khmer = ["milli/khmer"]
 vietnamese = ["milli/vietnamese"]
 # force swedish character recomposition
 swedish-recomposition = ["milli/swedish-recomposition"]
-# allow german tokenization
-german = ["milli/german"]
-# allow turkish normalization
-turkish = ["milli/turkish"]
-

meilisearch-types/src/deserr/mod.rs

@@ -155,10 +155,6 @@ make_missing_field_convenience_builder!(
     MissingFacetSearchFacetName,
     missing_facet_search_facet_name
 );
-make_missing_field_convenience_builder!(
-    MissingDocumentEditionFunction,
-    missing_document_edition_function
-);
 
 // Integrate a sub-error into a [`DeserrError`] by taking its error message but using
 // the default error code (C) from `Self`
@@ -192,7 +188,6 @@ merge_with_error_impl_take_error_message!(ParseOffsetDateTimeError);
 merge_with_error_impl_take_error_message!(ParseTaskKindError);
 merge_with_error_impl_take_error_message!(ParseTaskStatusError);
 merge_with_error_impl_take_error_message!(IndexUidFormatError);
-merge_with_error_impl_take_error_message!(InvalidMultiSearchWeight);
 merge_with_error_impl_take_error_message!(InvalidSearchSemanticRatio);
 merge_with_error_impl_take_error_message!(InvalidSearchRankingScoreThreshold);
 merge_with_error_impl_take_error_message!(InvalidSimilarRankingScoreThreshold);

meilisearch-types/src/error.rs

@@ -224,7 +224,6 @@ InvalidDocumentCsvDelimiter           , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentFields                 , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentRetrieveVectors        , InvalidRequest       , BAD_REQUEST ;
 MissingDocumentFilter                 , InvalidRequest       , BAD_REQUEST ;
-MissingDocumentEditionFunction        , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentFilter                 , InvalidRequest       , BAD_REQUEST ;
 InvalidDocumentGeoField               , InvalidRequest       , BAD_REQUEST ;
 InvalidVectorDimensions               , InvalidRequest       , BAD_REQUEST ;
@@ -238,17 +237,6 @@ InvalidIndexLimit                     , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexOffset                    , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexPrimaryKey                , InvalidRequest       , BAD_REQUEST ;
 InvalidIndexUid                       , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchFacets              , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchFacetsByIndex       , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchFacetOrder          , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchFederated           , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchFederationOptions   , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchMaxValuesPerFacet   , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchMergeFacets         , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchQueryFacets         , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchQueryPagination     , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchQueryRankingRules   , InvalidRequest       , BAD_REQUEST ;
-InvalidMultiSearchWeight              , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToSearchOn     , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToCrop         , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchAttributesToHighlight    , InvalidRequest       , BAD_REQUEST ;
@@ -262,7 +250,6 @@ InvalidSearchCropLength               , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchCropMarker               , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchFacets                   , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchSemanticRatio            , InvalidRequest       , BAD_REQUEST ;
-InvalidSearchLocales                  , InvalidRequest       , BAD_REQUEST ;
 InvalidFacetSearchFacetName           , InvalidRequest       , BAD_REQUEST ;
 InvalidSimilarId                      , InvalidRequest       , BAD_REQUEST ;
 InvalidSearchFilter                   , InvalidRequest       , BAD_REQUEST ;
@@ -304,7 +291,6 @@ InvalidSettingsSeparatorTokens        , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsDictionary             , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSynonyms               , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsTypoTolerance          , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsLocalizedAttributes    , InvalidRequest       , BAD_REQUEST ;
 InvalidState                          , Internal             , INTERNAL_SERVER_ERROR ;
 InvalidStoreFile                      , Internal             , INTERNAL_SERVER_ERROR ;
 InvalidSwapDuplicateIndexFound        , InvalidRequest       , BAD_REQUEST ;
@@ -350,10 +336,7 @@ UnsupportedMediaType                  , InvalidRequest       , UNSUPPORTED_MEDIA
 
 // Experimental features
 VectorEmbeddingError                  , InvalidRequest       , BAD_REQUEST ;
-NotFoundSimilarId                     , InvalidRequest       , BAD_REQUEST ;
-InvalidDocumentEditionContext         , InvalidRequest       , BAD_REQUEST ;
-InvalidDocumentEditionFunctionFilter  , InvalidRequest       , BAD_REQUEST ;
-EditDocumentsByFunctionError          , InvalidRequest       , BAD_REQUEST
+NotFoundSimilarId                     , InvalidRequest       , BAD_REQUEST
 }
 
 impl ErrorCode for JoinError {
@@ -394,11 +377,7 @@ impl ErrorCode for milli::Error {
                     | UserError::InvalidOpenAiModelDimensionsMax { .. }
                     | UserError::InvalidSettingsDimensions { .. }
                     | UserError::InvalidUrl { .. }
-                    | UserError::InvalidSettingsDocumentTemplateMaxBytes { .. }
-                    | UserError::InvalidPrompt(_)
-                    | UserError::InvalidDisableBinaryQuantization { .. } => {
-                        Code::InvalidSettingsEmbedders
-                    }
+                    | UserError::InvalidPrompt(_) => Code::InvalidSettingsEmbedders,
                     UserError::TooManyEmbedders(_) => Code::InvalidSettingsEmbedders,
                     UserError::InvalidPromptForEmbeddings(..) => Code::InvalidSettingsEmbedders,
                     UserError::NoPrimaryKeyCandidateFound => Code::IndexPrimaryKeyNoCandidateFound,
@@ -427,15 +406,7 @@ impl ErrorCode for milli::Error {
                         Code::InvalidSettingsTypoTolerance
                     }
                     UserError::InvalidEmbedder(_) => Code::InvalidEmbedder,
-                    UserError::VectorEmbeddingError(_) | UserError::DocumentEmbeddingError(_) => {
-                        Code::VectorEmbeddingError
-                    }
-                    UserError::DocumentEditionCannotModifyPrimaryKey
-                    | UserError::DocumentEditionDocumentMustBeObject
-                    | UserError::DocumentEditionRuntimeError(_)
-                    | UserError::DocumentEditionCompilationError(_) => {
-                        Code::EditDocumentsByFunctionError
-                    }
+                    UserError::VectorEmbeddingError(_) => Code::VectorEmbeddingError,
                 }
             }
         }
@@ -531,12 +502,6 @@ impl fmt::Display for deserr_codes::InvalidSearchSemanticRatio {
     }
 }
 
-impl fmt::Display for deserr_codes::InvalidMultiSearchWeight {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(f, "the value of `weight` is invalid, expected a positive float (>= 0.0).")
-    }
-}
-
 impl fmt::Display for deserr_codes::InvalidSimilarId {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         write!(

meilisearch-types/src/features.rs

@@ -6,13 +6,10 @@ pub struct RuntimeTogglableFeatures {
     pub vector_store: bool,
     pub metrics: bool,
     pub logs_route: bool,
-    pub edit_documents_by_function: bool,
-    pub contains_filter: bool,
 }
 
 #[derive(Default, Debug, Clone, Copy)]
 pub struct InstanceTogglableFeatures {
     pub metrics: bool,
     pub logs_route: bool,
-    pub contains_filter: bool,
 }

meilisearch-types/src/index_uid.rs

@@ -1,4 +1,3 @@
-use std::borrow::Borrow;
 use std::error::Error;
 use std::fmt;
 use std::str::FromStr;
@@ -9,7 +8,7 @@ use crate::error::{Code, ErrorCode};
 
 /// An index uid is composed of only ascii alphanumeric characters, - and _, between 1 and 400
 /// bytes long
-#[derive(Debug, Clone, PartialEq, Eq, Deserr, PartialOrd, Ord)]
+#[derive(Debug, Clone, PartialEq, Eq, Deserr)]
 #[deserr(try_from(String) = IndexUid::try_from -> IndexUidFormatError)]
 pub struct IndexUid(String);
 
@@ -71,12 +70,6 @@ impl From<IndexUid> for String {
     }
 }
 
-impl Borrow<String> for IndexUid {
-    fn borrow(&self) -> &String {
-        &self.0
-    }
-}
-
 #[derive(Debug)]
 pub struct IndexUidFormatError {
     pub invalid_uid: String,

meilisearch-types/src/lib.rs

@@ -7,13 +7,12 @@ pub mod features;
 pub mod index_uid;
 pub mod index_uid_pattern;
 pub mod keys;
-pub mod locales;
 pub mod settings;
 pub mod star_or;
 pub mod task_view;
 pub mod tasks;
 pub mod versioning;
-pub use milli::{heed, Index};
+pub use milli::{heed, zstd, Index};
 use uuid::Uuid;
 pub use versioning::VERSION_FILE_NAME;
 pub use {milli, serde_cs};

meilisearch-types/src/locales.rs

@@ -1,166 +0,0 @@
-use deserr::Deserr;
-use milli::LocalizedAttributesRule;
-use serde::{Deserialize, Serialize};
-
-#[derive(Debug, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize)]
-#[deserr(rename_all = camelCase)]
-#[serde(rename_all = "camelCase")]
-pub struct LocalizedAttributesRuleView {
-    pub attribute_patterns: Vec<String>,
-    pub locales: Vec<Locale>,
-}
-
-impl From<LocalizedAttributesRule> for LocalizedAttributesRuleView {
-    fn from(rule: LocalizedAttributesRule) -> Self {
-        Self {
-            attribute_patterns: rule.attribute_patterns,
-            locales: rule.locales.into_iter().map(|l| l.into()).collect(),
-        }
-    }
-}
-
-impl From<LocalizedAttributesRuleView> for LocalizedAttributesRule {
-    fn from(view: LocalizedAttributesRuleView) -> Self {
-        Self {
-            attribute_patterns: view.attribute_patterns,
-            locales: view.locales.into_iter().map(|l| l.into()).collect(),
-        }
-    }
-}
-
-/// Generate a Locale enum and its From and Into implementations for milli::tokenizer::Language.
-///
-/// this enum implements `Deserr` in order to be used in the API.
-macro_rules! make_locale {
-    ($(($iso_639_1:ident, $iso_639_1_str:expr) => ($iso_639_3:ident, $iso_639_3_str:expr),)+) => {
-        #[derive(Debug, Copy, Clone, PartialEq, Eq, Deserr, Serialize, Deserialize, Ord, PartialOrd)]
-        #[deserr(rename_all = camelCase)]
-        #[serde(rename_all = "camelCase")]
-        pub enum Locale {
-            $($iso_639_1,)+
-            $($iso_639_3,)+
-            Cmn,
-        }
-
-        impl From<milli::tokenizer::Language> for Locale {
-            fn from(other: milli::tokenizer::Language) -> Locale {
-                match other {
-                    $(milli::tokenizer::Language::$iso_639_3 => Locale::$iso_639_3,)+
-                    milli::tokenizer::Language::Cmn => Locale::Cmn,
-                }
-            }
-        }
-
-        impl From<Locale> for milli::tokenizer::Language {
-            fn from(other: Locale) -> milli::tokenizer::Language {
-                match other {
-                    $(Locale::$iso_639_1 => milli::tokenizer::Language::$iso_639_3,)+
-                    $(Locale::$iso_639_3 => milli::tokenizer::Language::$iso_639_3,)+
-                    Locale::Cmn => milli::tokenizer::Language::Cmn,
-                }
-            }
-        }
-
-        impl std::str::FromStr for Locale {
-            type Err = LocaleFormatError;
-
-            fn from_str(s: &str) -> Result<Self, Self::Err> {
-                let locale = match s {
-                    $($iso_639_1_str => Locale::$iso_639_1,)+
-                    $($iso_639_3_str => Locale::$iso_639_3,)+
-                    "cmn" => Locale::Cmn,
-                    _ => return Err(LocaleFormatError { invalid_locale: s.to_string() }),
-                };
-
-                Ok(locale)
-            }
-        }
-
-        #[derive(Debug)]
-        pub struct LocaleFormatError {
-            pub invalid_locale: String,
-        }
-
-        impl std::fmt::Display for LocaleFormatError {
-            fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-                let mut valid_locales = [$($iso_639_1_str),+,$($iso_639_3_str),+,"cmn"];
-                valid_locales.sort_by(|left, right| left.len().cmp(&right.len()).then(left.cmp(right)));
-                write!(f, "Unsupported locale `{}`, expected one of {}", self.invalid_locale, valid_locales.join(", "))
-            }
-        }
-
-        impl std::error::Error for LocaleFormatError {}
-    };
-}
-
-make_locale!(
-    (Af, "af") => (Afr, "afr"),
-    (Ak, "ak") => (Aka, "aka"),
-    (Am, "am") => (Amh, "amh"),
-    (Ar, "ar") => (Ara, "ara"),
-    (Az, "az") => (Aze, "aze"),
-    (Be, "be") => (Bel, "bel"),
-    (Bn, "bn") => (Ben, "ben"),
-    (Bg, "bg") => (Bul, "bul"),
-    (Ca, "ca") => (Cat, "cat"),
-    (Cs, "cs") => (Ces, "ces"),
-    (Da, "da") => (Dan, "dan"),
-    (De, "de") => (Deu, "deu"),
-    (El, "el") => (Ell, "ell"),
-    (En, "en") => (Eng, "eng"),
-    (Eo, "eo") => (Epo, "epo"),
-    (Et, "et") => (Est, "est"),
-    (Fi, "fi") => (Fin, "fin"),
-    (Fr, "fr") => (Fra, "fra"),
-    (Gu, "gu") => (Guj, "guj"),
-    (He, "he") => (Heb, "heb"),
-    (Hi, "hi") => (Hin, "hin"),
-    (Hr, "hr") => (Hrv, "hrv"),
-    (Hu, "hu") => (Hun, "hun"),
-    (Hy, "hy") => (Hye, "hye"),
-    (Id, "id") => (Ind, "ind"),
-    (It, "it") => (Ita, "ita"),
-    (Jv, "jv") => (Jav, "jav"),
-    (Ja, "ja") => (Jpn, "jpn"),
-    (Kn, "kn") => (Kan, "kan"),
-    (Ka, "ka") => (Kat, "kat"),
-    (Km, "km") => (Khm, "khm"),
-    (Ko, "ko") => (Kor, "kor"),
-    (La, "la") => (Lat, "lat"),
-    (Lv, "lv") => (Lav, "lav"),
-    (Lt, "lt") => (Lit, "lit"),
-    (Ml, "ml") => (Mal, "mal"),
-    (Mr, "mr") => (Mar, "mar"),
-    (Mk, "mk") => (Mkd, "mkd"),
-    (My, "my") => (Mya, "mya"),
-    (Ne, "ne") => (Nep, "nep"),
-    (Nl, "nl") => (Nld, "nld"),
-    (Nb, "nb") => (Nob, "nob"),
-    (Or, "or") => (Ori, "ori"),
-    (Pa, "pa") => (Pan, "pan"),
-    (Fa, "fa") => (Pes, "pes"),
-    (Pl, "pl") => (Pol, "pol"),
-    (Pt, "pt") => (Por, "por"),
-    (Ro, "ro") => (Ron, "ron"),
-    (Ru, "ru") => (Rus, "rus"),
-    (Si, "si") => (Sin, "sin"),
-    (Sk, "sk") => (Slk, "slk"),
-    (Sl, "sl") => (Slv, "slv"),
-    (Sn, "sn") => (Sna, "sna"),
-    (Es, "es") => (Spa, "spa"),
-    (Sr, "sr") => (Srp, "srp"),
-    (Sv, "sv") => (Swe, "swe"),
-    (Ta, "ta") => (Tam, "tam"),
-    (Te, "te") => (Tel, "tel"),
-    (Tl, "tl") => (Tgl, "tgl"),
-    (Th, "th") => (Tha, "tha"),
-    (Tk, "tk") => (Tuk, "tuk"),
-    (Tr, "tr") => (Tur, "tur"),
-    (Uk, "uk") => (Ukr, "ukr"),
-    (Ur, "ur") => (Urd, "urd"),
-    (Uz, "uz") => (Uzb, "uzb"),
-    (Vi, "vi") => (Vie, "vie"),
-    (Yi, "yi") => (Yid, "yid"),
-    (Zh, "zh") => (Zho, "zho"),
-    (Zu, "zu") => (Zul, "zul"),
-);

meilisearch-types/src/settings.rs

@@ -17,7 +17,6 @@ use serde::{Deserialize, Serialize, Serializer};
 use crate::deserr::DeserrJsonError;
 use crate::error::deserr_codes::*;
 use crate::facet_values_sort::FacetValuesSort;
-use crate::locales::LocalizedAttributesRuleView;
 
 /// The maximum number of results that the engine
 /// will be able to return in one search call.
@@ -199,9 +198,6 @@ pub struct Settings<T> {
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsSearchCutoffMs>)]
     pub search_cutoff_ms: Setting<u64>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    #[deserr(default, error = DeserrJsonError<InvalidSettingsLocalizedAttributes>)]
-    pub localized_attributes: Setting<Vec<LocalizedAttributesRuleView>>,
 
     #[serde(skip)]
     #[deserr(skip)]
@@ -265,7 +261,6 @@ impl Settings<Checked> {
             pagination: Setting::Reset,
             embedders: Setting::Reset,
             search_cutoff_ms: Setting::Reset,
-            localized_attributes: Setting::Reset,
             _kind: PhantomData,
         }
     }
@@ -289,8 +284,7 @@ impl Settings<Checked> {
             pagination,
             embedders,
             search_cutoff_ms,
-            localized_attributes: localized_attributes_rules,
-            _kind,
+            ..
         } = self;
 
         Settings {
@@ -311,7 +305,6 @@ impl Settings<Checked> {
             pagination,
             embedders,
             search_cutoff_ms,
-            localized_attributes: localized_attributes_rules,
             _kind: PhantomData,
         }
     }
@@ -359,7 +352,6 @@ impl Settings<Unchecked> {
             pagination: self.pagination,
             embedders: self.embedders,
             search_cutoff_ms: self.search_cutoff_ms,
-            localized_attributes: self.localized_attributes,
             _kind: PhantomData,
         }
     }
@@ -410,7 +402,6 @@ pub fn apply_settings_to_builder(
         pagination,
         embedders,
         search_cutoff_ms,
-        localized_attributes: localized_attributes_rules,
         _kind,
     } = settings;
 
@@ -494,13 +485,6 @@ pub fn apply_settings_to_builder(
         Setting::NotSet => (),
     }
 
-    match localized_attributes_rules {
-        Setting::Set(ref rules) => builder
-            .set_localized_attributes_rules(rules.iter().cloned().map(|r| r.into()).collect()),
-        Setting::Reset => builder.reset_localized_attributes_rules(),
-        Setting::NotSet => (),
-    }
-
     match typo_tolerance {
         Setting::Set(ref value) => {
             match value.enabled {
@@ -695,8 +679,6 @@ pub fn settings(
 
     let search_cutoff_ms = index.search_cutoff(rtxn)?;
 
-    let localized_attributes_rules = index.localized_attributes_rules(rtxn)?;
-
     let mut settings = Settings {
         displayed_attributes: match displayed_attributes {
             Some(attrs) => Setting::Set(attrs),
@@ -729,10 +711,6 @@ pub fn settings(
             Some(cutoff) => Setting::Set(cutoff),
             None => Setting::Reset,
         },
-        localized_attributes: match localized_attributes_rules {
-            Some(rules) => Setting::Set(rules.into_iter().map(|r| r.into()).collect()),
-            None => Setting::Reset,
-        },
         _kind: PhantomData,
     };
 
@@ -924,7 +902,6 @@ pub(crate) mod test {
             faceting: Setting::NotSet,
             pagination: Setting::NotSet,
             embedders: Setting::NotSet,
-            localized_attributes: Setting::NotSet,
             search_cutoff_ms: Setting::NotSet,
             _kind: PhantomData::<Unchecked>,
         };
@@ -953,7 +930,6 @@ pub(crate) mod test {
             faceting: Setting::NotSet,
             pagination: Setting::NotSet,
             embedders: Setting::NotSet,
-            localized_attributes: Setting::NotSet,
             search_cutoff_ms: Setting::NotSet,
             _kind: PhantomData::<Unchecked>,
         };

meilisearch-types/src/task_view.rs

@@ -1,4 +1,3 @@
-use milli::Object;
 use serde::Serialize;
 use time::{Duration, OffsetDateTime};
 
@@ -55,8 +54,6 @@ pub struct DetailsView {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub indexed_documents: Option<Option<u64>>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub edited_documents: Option<Option<u64>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     pub primary_key: Option<Option<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub provided_ids: Option<usize>,
@@ -73,10 +70,6 @@ pub struct DetailsView {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub dump_uid: Option<Option<String>>,
     #[serde(skip_serializing_if = "Option::is_none")]
-    pub context: Option<Option<Object>>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub function: Option<String>,
-    #[serde(skip_serializing_if = "Option::is_none")]
     #[serde(flatten)]
     pub settings: Option<Box<Settings<Unchecked>>>,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -93,20 +86,6 @@ impl From<Details> for DetailsView {
                     ..DetailsView::default()
                 }
             }
-            Details::DocumentEdition {
-                deleted_documents,
-                edited_documents,
-                original_filter,
-                context,
-                function,
-            } => DetailsView {
-                deleted_documents: Some(deleted_documents),
-                edited_documents: Some(edited_documents),
-                original_filter: Some(original_filter),
-                context: Some(context),
-                function: Some(function),
-                ..DetailsView::default()
-            },
             Details::SettingsUpdate { mut settings } => {
                 settings.hide_secrets();
                 DetailsView { settings: Some(settings), ..DetailsView::default() }

meilisearch-types/src/tasks.rs

@@ -5,7 +5,6 @@ use std::str::FromStr;
 
 use enum_iterator::Sequence;
 use milli::update::IndexDocumentsMethod;
-use milli::Object;
 use roaring::RoaringBitmap;
 use serde::{Deserialize, Serialize, Serializer};
 use time::{Duration, OffsetDateTime};
@@ -49,7 +48,6 @@ impl Task {
             | TaskDeletion { .. }
             | IndexSwap { .. } => None,
             DocumentAdditionOrUpdate { index_uid, .. }
-            | DocumentEdition { index_uid, .. }
             | DocumentDeletion { index_uid, .. }
             | DocumentDeletionByFilter { index_uid, .. }
             | DocumentClear { index_uid }
@@ -69,8 +67,7 @@ impl Task {
     pub fn content_uuid(&self) -> Option<Uuid> {
         match self.kind {
             KindWithContent::DocumentAdditionOrUpdate { content_file, .. } => Some(content_file),
-            KindWithContent::DocumentEdition { .. }
-            | KindWithContent::DocumentDeletion { .. }
+            KindWithContent::DocumentDeletion { .. }
             | KindWithContent::DocumentDeletionByFilter { .. }
             | KindWithContent::DocumentClear { .. }
             | KindWithContent::SettingsUpdate { .. }
@@ -105,12 +102,6 @@ pub enum KindWithContent {
         index_uid: String,
         filter_expr: serde_json::Value,
     },
-    DocumentEdition {
-        index_uid: String,
-        filter_expr: Option<serde_json::Value>,
-        context: Option<milli::Object>,
-        function: String,
-    },
     DocumentClear {
         index_uid: String,
     },
@@ -159,7 +150,6 @@ impl KindWithContent {
     pub fn as_kind(&self) -> Kind {
         match self {
             KindWithContent::DocumentAdditionOrUpdate { .. } => Kind::DocumentAdditionOrUpdate,
-            KindWithContent::DocumentEdition { .. } => Kind::DocumentEdition,
             KindWithContent::DocumentDeletion { .. } => Kind::DocumentDeletion,
             KindWithContent::DocumentDeletionByFilter { .. } => Kind::DocumentDeletion,
             KindWithContent::DocumentClear { .. } => Kind::DocumentDeletion,
@@ -184,7 +174,6 @@ impl KindWithContent {
             | TaskCancelation { .. }
             | TaskDeletion { .. } => vec![],
             DocumentAdditionOrUpdate { index_uid, .. }
-            | DocumentEdition { index_uid, .. }
             | DocumentDeletion { index_uid, .. }
             | DocumentDeletionByFilter { index_uid, .. }
             | DocumentClear { index_uid }
@@ -213,15 +202,6 @@ impl KindWithContent {
                     indexed_documents: None,
                 })
             }
-            KindWithContent::DocumentEdition { index_uid: _, filter_expr, context, function } => {
-                Some(Details::DocumentEdition {
-                    deleted_documents: None,
-                    edited_documents: None,
-                    original_filter: filter_expr.as_ref().map(|v| v.to_string()),
-                    context: context.clone(),
-                    function: function.clone(),
-                })
-            }
             KindWithContent::DocumentDeletion { index_uid: _, documents_ids } => {
                 Some(Details::DocumentDeletion {
                     provided_ids: documents_ids.len(),
@@ -270,15 +250,6 @@ impl KindWithContent {
                     indexed_documents: Some(0),
                 })
             }
-            KindWithContent::DocumentEdition { index_uid: _, filter_expr, context, function } => {
-                Some(Details::DocumentEdition {
-                    deleted_documents: Some(0),
-                    edited_documents: Some(0),
-                    original_filter: filter_expr.as_ref().map(|v| v.to_string()),
-                    context: context.clone(),
-                    function: function.clone(),
-                })
-            }
             KindWithContent::DocumentDeletion { index_uid: _, documents_ids } => {
                 Some(Details::DocumentDeletion {
                     provided_ids: documents_ids.len(),
@@ -330,7 +301,6 @@ impl From<&KindWithContent> for Option<Details> {
                     indexed_documents: None,
                 })
             }
-            KindWithContent::DocumentEdition { .. } => None,
             KindWithContent::DocumentDeletion { .. } => None,
             KindWithContent::DocumentDeletionByFilter { .. } => None,
             KindWithContent::DocumentClear { .. } => None,
@@ -424,7 +394,6 @@ impl std::error::Error for ParseTaskStatusError {}
 #[serde(rename_all = "camelCase")]
 pub enum Kind {
     DocumentAdditionOrUpdate,
-    DocumentEdition,
     DocumentDeletion,
     SettingsUpdate,
     IndexCreation,
@@ -441,7 +410,6 @@ impl Kind {
     pub fn related_to_one_index(&self) -> bool {
         match self {
             Kind::DocumentAdditionOrUpdate
-            | Kind::DocumentEdition
             | Kind::DocumentDeletion
             | Kind::SettingsUpdate
             | Kind::IndexCreation
@@ -459,7 +427,6 @@ impl Display for Kind {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
             Kind::DocumentAdditionOrUpdate => write!(f, "documentAdditionOrUpdate"),
-            Kind::DocumentEdition => write!(f, "documentEdition"),
             Kind::DocumentDeletion => write!(f, "documentDeletion"),
             Kind::SettingsUpdate => write!(f, "settingsUpdate"),
             Kind::IndexCreation => write!(f, "indexCreation"),
@@ -487,8 +454,6 @@ impl FromStr for Kind {
             Ok(Kind::IndexDeletion)
         } else if kind.eq_ignore_ascii_case("documentAdditionOrUpdate") {
             Ok(Kind::DocumentAdditionOrUpdate)
-        } else if kind.eq_ignore_ascii_case("documentEdition") {
-            Ok(Kind::DocumentEdition)
         } else if kind.eq_ignore_ascii_case("documentDeletion") {
             Ok(Kind::DocumentDeletion)
         } else if kind.eq_ignore_ascii_case("settingsUpdate") {
@@ -530,50 +495,16 @@ impl std::error::Error for ParseTaskKindError {}
 
 #[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]
 pub enum Details {
-    DocumentAdditionOrUpdate {
-        received_documents: u64,
-        indexed_documents: Option<u64>,
-    },
-    SettingsUpdate {
-        settings: Box<Settings<Unchecked>>,
-    },
-    IndexInfo {
-        primary_key: Option<String>,
-    },
-    DocumentDeletion {
-        provided_ids: usize,
-        deleted_documents: Option<u64>,
-    },
-    DocumentDeletionByFilter {
-        original_filter: String,
-        deleted_documents: Option<u64>,
-    },
-    DocumentEdition {
-        deleted_documents: Option<u64>,
-        edited_documents: Option<u64>,
-        original_filter: Option<String>,
-        context: Option<Object>,
-        function: String,
-    },
-    ClearAll {
-        deleted_documents: Option<u64>,
-    },
-    TaskCancelation {
-        matched_tasks: u64,
-        canceled_tasks: Option<u64>,
-        original_filter: String,
-    },
-    TaskDeletion {
-        matched_tasks: u64,
-        deleted_tasks: Option<u64>,
-        original_filter: String,
-    },
-    Dump {
-        dump_uid: Option<String>,
-    },
-    IndexSwap {
-        swaps: Vec<IndexSwap>,
-    },
+    DocumentAdditionOrUpdate { received_documents: u64, indexed_documents: Option<u64> },
+    SettingsUpdate { settings: Box<Settings<Unchecked>> },
+    IndexInfo { primary_key: Option<String> },
+    DocumentDeletion { provided_ids: usize, deleted_documents: Option<u64> },
+    DocumentDeletionByFilter { original_filter: String, deleted_documents: Option<u64> },
+    ClearAll { deleted_documents: Option<u64> },
+    TaskCancelation { matched_tasks: u64, canceled_tasks: Option<u64>, original_filter: String },
+    TaskDeletion { matched_tasks: u64, deleted_tasks: Option<u64>, original_filter: String },
+    Dump { dump_uid: Option<String> },
+    IndexSwap { swaps: Vec<IndexSwap> },
 }
 
 impl Details {
@@ -583,7 +514,6 @@ impl Details {
             Self::DocumentAdditionOrUpdate { indexed_documents, .. } => {
                 *indexed_documents = Some(0)
             }
-            Self::DocumentEdition { edited_documents, .. } => *edited_documents = Some(0),
             Self::DocumentDeletion { deleted_documents, .. } => *deleted_documents = Some(0),
             Self::DocumentDeletionByFilter { deleted_documents, .. } => {
                 *deleted_documents = Some(0)

meilisearch-types/src/versioning.rs

@@ -10,52 +10,38 @@ static VERSION_MINOR: &str = env!("CARGO_PKG_VERSION_MINOR");
 static VERSION_PATCH: &str = env!("CARGO_PKG_VERSION_PATCH");
 
 /// Persists the version of the current Meilisearch binary to a VERSION file
-pub fn create_current_version_file(db_path: &Path) -> io::Result<()> {
-    create_version_file(db_path, VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH)
-}
-
-pub fn create_version_file(
-    db_path: &Path,
-    major: &str,
-    minor: &str,
-    patch: &str,
-) -> io::Result<()> {
+pub fn create_version_file(db_path: &Path) -> io::Result<()> {
     let version_path = db_path.join(VERSION_FILE_NAME);
-    fs::write(version_path, format!("{}.{}.{}", major, minor, patch))
+    fs::write(version_path, format!("{}.{}.{}", VERSION_MAJOR, VERSION_MINOR, VERSION_PATCH))
 }
 
 /// Ensures Meilisearch version is compatible with the database, returns an error versions mismatch.
 pub fn check_version_file(db_path: &Path) -> anyhow::Result<()> {
-    let (major, minor, patch) = get_version(db_path)?;
+    let version_path = db_path.join(VERSION_FILE_NAME);
+
+    match fs::read_to_string(version_path) {
+        Ok(version) => {
+            let version_components = version.split('.').collect::<Vec<_>>();
+            let (major, minor, patch) = match &version_components[..] {
+                [major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()),
+                _ => return Err(VersionFileError::MalformedVersionFile.into()),
+            };
 
             if major != VERSION_MAJOR || minor != VERSION_MINOR {
                 return Err(VersionFileError::VersionMismatch { major, minor, patch }.into());
             }
+        }
+        Err(error) => {
+            return match error.kind() {
+                ErrorKind::NotFound => Err(VersionFileError::MissingVersionFile.into()),
+                _ => Err(error.into()),
+            }
+        }
+    }
 
     Ok(())
 }
 
-pub fn get_version(db_path: &Path) -> Result<(String, String, String), VersionFileError> {
-    let version_path = db_path.join(VERSION_FILE_NAME);
-
-    match fs::read_to_string(version_path) {
-        Ok(version) => parse_version(&version),
-        Err(error) => match error.kind() {
-            ErrorKind::NotFound => Err(VersionFileError::MissingVersionFile),
-            _ => Err(error.into()),
-        },
-    }
-}
-
-pub fn parse_version(version: &str) -> Result<(String, String, String), VersionFileError> {
-    let version_components = version.split('.').collect::<Vec<_>>();
-    let (major, minor, patch) = match &version_components[..] {
-        [major, minor, patch] => (major.to_string(), minor.to_string(), patch.to_string()),
-        _ => return Err(VersionFileError::MalformedVersionFile),
-    };
-    Ok((major, minor, patch))
-}
-
 #[derive(thiserror::Error, Debug)]
 pub enum VersionFileError {
     #[error(
@@ -72,7 +58,4 @@ pub enum VersionFileError {
         env!("CARGO_PKG_VERSION").to_string()
     )]
     VersionMismatch { major: String, minor: String, patch: String },
-
-    #[error(transparent)]
-    IoError(#[from] std::io::Error),
 }

meilisearch/Cargo.toml

@@ -17,7 +17,7 @@ actix-cors = "0.7.0"
 actix-http = { version = "3.8.0", default-features = false, features = [
     "compress-brotli",
     "compress-gzip",
-    "rustls-0_23",
+    "rustls-0_21",
 ] }
 actix-utils = "3.0.1"
 actix-web = { version = "4.8.0", default-features = false, features = [
@@ -25,7 +25,7 @@ actix-web = { version = "4.8.0", default-features = false, features = [
     "compress-brotli",
     "compress-gzip",
     "cookies",
-    "rustls-0_23",
+    "rustls-0_21",
 ] }
 anyhow = { version = "1.0.86", features = ["backtrace"] }
 async-trait = "0.1.81"
@@ -72,10 +72,9 @@ reqwest = { version = "0.12.5", features = [
     "rustls-tls",
     "json",
 ], default-features = false }
-rustls = { version = "0.23.11", features = ["ring"], default-features = false }
-rustls-pki-types = { version = "1.7.0", features = ["alloc"] }
-rustls-pemfile = "2.1.2"
-segment = { version = "0.2.4" }
+rustls = "0.21.12"
+rustls-pemfile = "1.0.4"
+segment = { version = "0.2.4", optional = true }
 serde = { version = "1.0.204", features = ["derive"] }
 serde_json = { version = "1.0.120", features = ["preserve_order"] }
 sha2 = "0.10.8"
@@ -103,8 +102,6 @@ tracing-subscriber = { version = "0.3.18", features = ["json"] }
 tracing-trace = { version = "0.1.0", path = "../tracing-trace" }
 tracing-actix-web = "0.7.11"
 build-info = { version = "1.7.0", path = "../build-info" }
-roaring = "0.10.2"
-mopa-maintained = "0.2.3"
 
 [dev-dependencies]
 actix-rt = "2.10.0"
@@ -115,7 +112,6 @@ maplit = "1.0.2"
 meili-snap = { path = "../meili-snap" }
 temp-env = "0.3.6"
 urlencoding = "2.1.3"
-wiremock = "0.6.0"
 yaup = "0.3.1"
 
 [build-dependencies]
@@ -129,10 +125,11 @@ reqwest = { version = "0.12.5", features = [
 sha-1 = { version = "0.10.1", optional = true }
 static-files = { version = "0.2.4", optional = true }
 tempfile = { version = "3.10.1", optional = true }
-zip = { version = "2.1.3", optional = true }
+zip = { version = "2.1.3", default-features = false, features = ["deflate"], optional = true }
 
 [features]
-default = ["meilisearch-types/all-tokenizations", "mini-dashboard"]
+default = ["analytics", "meilisearch-types/all-tokenizations", "mini-dashboard"]
+analytics = ["segment"]
 mini-dashboard = [
     "static-files",
     "anyhow",
@@ -153,9 +150,7 @@ greek = ["meilisearch-types/greek"]
 khmer = ["meilisearch-types/khmer"]
 vietnamese = ["meilisearch-types/vietnamese"]
 swedish-recomposition = ["meilisearch-types/swedish-recomposition"]
-german = ["meilisearch-types/german"]
-turkish = ["meilisearch-types/turkish"]
 
 [package.metadata.mini-dashboard]
-assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.15/build.zip"
-sha1 = "d057600b4a839a2e0c0be7a372cd1b2683f3ca7e"
+assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.14/build.zip"
+sha1 = "592d1b5a3459d621d0aae1dded8fe3154f5c38fe"

meilisearch/src/analytics/mock_analytics.rs

@@ -0,0 +1,102 @@
+use std::any::Any;
+use std::sync::Arc;
+
+use actix_web::HttpRequest;
+use meilisearch_types::InstanceUid;
+use serde_json::Value;
+
+use super::{find_user_id, Analytics, DocumentDeletionKind, DocumentFetchKind};
+use crate::routes::indexes::documents::UpdateDocumentsQuery;
+use crate::Opt;
+
+pub struct MockAnalytics {
+    instance_uid: Option<InstanceUid>,
+}
+
+#[derive(Default)]
+pub struct SearchAggregator;
+
+#[allow(dead_code)]
+impl SearchAggregator {
+    pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self {
+        Self
+    }
+
+    pub fn succeed(&mut self, _: &dyn Any) {}
+}
+
+#[derive(Default)]
+pub struct SimilarAggregator;
+
+#[allow(dead_code)]
+impl SimilarAggregator {
+    pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self {
+        Self
+    }
+
+    pub fn succeed(&mut self, _: &dyn Any) {}
+}
+
+#[derive(Default)]
+pub struct MultiSearchAggregator;
+
+#[allow(dead_code)]
+impl MultiSearchAggregator {
+    pub fn from_queries(_: &dyn Any, _: &dyn Any) -> Self {
+        Self
+    }
+
+    pub fn succeed(&mut self) {}
+}
+
+#[derive(Default)]
+pub struct FacetSearchAggregator;
+
+#[allow(dead_code)]
+impl FacetSearchAggregator {
+    pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self {
+        Self
+    }
+
+    pub fn succeed(&mut self, _: &dyn Any) {}
+}
+
+impl MockAnalytics {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(opt: &Opt) -> Arc<dyn Analytics> {
+        let instance_uid = find_user_id(&opt.db_path);
+        Arc::new(Self { instance_uid })
+    }
+}
+
+impl Analytics for MockAnalytics {
+    fn instance_uid(&self) -> Option<&meilisearch_types::InstanceUid> {
+        self.instance_uid.as_ref()
+    }
+
+    // These methods are noop and should be optimized out
+    fn publish(&self, _event_name: String, _send: Value, _request: Option<&HttpRequest>) {}
+    fn get_search(&self, _aggregate: super::SearchAggregator) {}
+    fn post_search(&self, _aggregate: super::SearchAggregator) {}
+    fn get_similar(&self, _aggregate: super::SimilarAggregator) {}
+    fn post_similar(&self, _aggregate: super::SimilarAggregator) {}
+    fn post_multi_search(&self, _aggregate: super::MultiSearchAggregator) {}
+    fn post_facet_search(&self, _aggregate: super::FacetSearchAggregator) {}
+    fn add_documents(
+        &self,
+        _documents_query: &UpdateDocumentsQuery,
+        _index_creation: bool,
+        _request: &HttpRequest,
+    ) {
+    }
+    fn delete_documents(&self, _kind: DocumentDeletionKind, _request: &HttpRequest) {}
+    fn update_documents(
+        &self,
+        _documents_query: &UpdateDocumentsQuery,
+        _index_creation: bool,
+        _request: &HttpRequest,
+    ) {
+    }
+    fn get_fetch_documents(&self, _documents_query: &DocumentFetchKind, _request: &HttpRequest) {}
+    fn post_fetch_documents(&self, _documents_query: &DocumentFetchKind, _request: &HttpRequest) {}
+}

meilisearch/src/analytics/mod.rs

@@ -1,45 +1,44 @@
-pub mod segment_analytics;
+mod mock_analytics;
+#[cfg(feature = "analytics")]
+mod segment_analytics;
 
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
-use std::sync::Arc;
 
 use actix_web::HttpRequest;
-use index_scheduler::IndexScheduler;
-use meilisearch_auth::AuthController;
 use meilisearch_types::InstanceUid;
-use mopa::mopafy;
+pub use mock_analytics::MockAnalytics;
 use once_cell::sync::Lazy;
 use platform_dirs::AppDirs;
+use serde_json::Value;
+
+use crate::routes::indexes::documents::UpdateDocumentsQuery;
+
+// if the analytics feature is disabled
+// the `SegmentAnalytics` point to the mock instead of the real analytics
+#[cfg(not(feature = "analytics"))]
+pub type SegmentAnalytics = mock_analytics::MockAnalytics;
+#[cfg(not(feature = "analytics"))]
+pub type SearchAggregator = mock_analytics::SearchAggregator;
+#[cfg(not(feature = "analytics"))]
+pub type SimilarAggregator = mock_analytics::SimilarAggregator;
+#[cfg(not(feature = "analytics"))]
+pub type MultiSearchAggregator = mock_analytics::MultiSearchAggregator;
+#[cfg(not(feature = "analytics"))]
+pub type FacetSearchAggregator = mock_analytics::FacetSearchAggregator;
 
 // if the feature analytics is enabled we use the real analytics
+#[cfg(feature = "analytics")]
 pub type SegmentAnalytics = segment_analytics::SegmentAnalytics;
-
-use crate::Opt;
-
-/// A macro used to quickly define events that don't aggregate or send anything besides an empty event with its name.
-#[macro_export]
-macro_rules! empty_analytics {
-    ($struct_name:ident, $event_name:literal) => {
-        #[derive(Default)]
-        struct $struct_name {}
-
-        impl $crate::analytics::Aggregate for $struct_name {
-            fn event_name(&self) -> &'static str {
-                $event_name
-            }
-
-            fn aggregate(self: Box<Self>, _other: Box<Self>) -> Box<Self> {
-                self
-            }
-
-            fn into_event(self: Box<Self>) -> serde_json::Value {
-                serde_json::json!({})
-            }
-        }
-    };
-}
+#[cfg(feature = "analytics")]
+pub type SearchAggregator = segment_analytics::SearchAggregator;
+#[cfg(feature = "analytics")]
+pub type SimilarAggregator = segment_analytics::SimilarAggregator;
+#[cfg(feature = "analytics")]
+pub type MultiSearchAggregator = segment_analytics::MultiSearchAggregator;
+#[cfg(feature = "analytics")]
+pub type FacetSearchAggregator = segment_analytics::FacetSearchAggregator;
 
 /// The Meilisearch config dir:
 /// `~/.config/Meilisearch` on *NIX or *BSD.
@@ -79,88 +78,52 @@ pub enum DocumentFetchKind {
     Normal { with_filter: bool, limit: usize, offset: usize, retrieve_vectors: bool },
 }
 
-/// To send an event to segment, your event must be able to aggregate itself with another event of the same type.
-pub trait Aggregate: 'static + mopa::Any + Send {
-    /// The name of the event that will be sent to segment.
-    fn event_name(&self) -> &'static str;
-
-    /// Will be called every time an event has been used twice before segment flushed its buffer.
-    fn aggregate(self: Box<Self>, new: Box<Self>) -> Box<Self>
-    where
-        Self: Sized;
-
-    /// Converts your structure to the final event that'll be sent to segment.
-    fn into_event(self: Box<Self>) -> serde_json::Value;
-}
-
-mopafy!(Aggregate);
-
-/// Helper trait to define multiple aggregates with the same content but a different name.
-/// Commonly used when you must aggregate a search with POST or with GET, for example.
-pub trait AggregateMethod: 'static + Default + Send {
-    fn event_name() -> &'static str;
-}
-
-/// A macro used to quickly define multiple aggregate method with their name
-/// Usage:
-/// ```rust
-/// use meilisearch::aggregate_methods;
-///
-/// aggregate_methods!(
-///     SearchGET => "Documents Searched GET",
-///     SearchPOST => "Documents Searched POST",
-/// );
-/// ```
-#[macro_export]
-macro_rules! aggregate_methods {
-    ($method:ident => $event_name:literal) => {
-        #[derive(Default)]
-        pub struct $method {}
-
-        impl $crate::analytics::AggregateMethod for $method {
-            fn event_name() -> &'static str {
-                $event_name
-            }
-        }
-    };
-    ($($method:ident => $event_name:literal,)+) => {
-        $(
-            aggregate_methods!($method => $event_name);
-        )+
-
-    };
-}
-
-#[derive(Clone)]
-pub struct Analytics {
-    segment: Option<Arc<SegmentAnalytics>>,
-}
-
-impl Analytics {
-    pub async fn new(
-        opt: &Opt,
-        index_scheduler: Arc<IndexScheduler>,
-        auth_controller: Arc<AuthController>,
-    ) -> Self {
-        if opt.no_analytics {
-            Self { segment: None }
-        } else {
-            Self { segment: SegmentAnalytics::new(opt, index_scheduler, auth_controller).await }
-        }
-    }
-
-    pub fn no_analytics() -> Self {
-        Self { segment: None }
-    }
-
-    pub fn instance_uid(&self) -> Option<&InstanceUid> {
-        self.segment.as_ref().map(|segment| segment.instance_uid.as_ref())
-    }
+pub trait Analytics: Sync + Send {
+    fn instance_uid(&self) -> Option<&InstanceUid>;
 
     /// The method used to publish most analytics that do not need to be batched every hours
-    pub fn publish<T: Aggregate>(&self, event: T, request: &HttpRequest) {
-        if let Some(ref segment) = self.segment {
-            let _ = segment.sender.try_send(segment_analytics::Message::new(event, request));
-        }
-    }
+    fn publish(&self, event_name: String, send: Value, request: Option<&HttpRequest>);
+
+    /// This method should be called to aggregate a get search
+    fn get_search(&self, aggregate: SearchAggregator);
+
+    /// This method should be called to aggregate a post search
+    fn post_search(&self, aggregate: SearchAggregator);
+
+    /// This method should be called to aggregate a get similar request
+    fn get_similar(&self, aggregate: SimilarAggregator);
+
+    /// This method should be called to aggregate a post similar request
+    fn post_similar(&self, aggregate: SimilarAggregator);
+
+    /// This method should be called to aggregate a post array of searches
+    fn post_multi_search(&self, aggregate: MultiSearchAggregator);
+
+    /// This method should be called to aggregate post facet values searches
+    fn post_facet_search(&self, aggregate: FacetSearchAggregator);
+
+    // this method should be called to aggregate a add documents request
+    fn add_documents(
+        &self,
+        documents_query: &UpdateDocumentsQuery,
+        index_creation: bool,
+        request: &HttpRequest,
+    );
+
+    // this method should be called to aggregate a fetch documents request
+    fn get_fetch_documents(&self, documents_query: &DocumentFetchKind, request: &HttpRequest);
+
+    // this method should be called to aggregate a fetch documents request
+    fn post_fetch_documents(&self, documents_query: &DocumentFetchKind, request: &HttpRequest);
+
+    // this method should be called to aggregate a add documents request
+    fn delete_documents(&self, kind: DocumentDeletionKind, request: &HttpRequest);
+
+    // this method should be called to batch a update documents request
+    fn update_documents(
+        &self,
+        documents_query: &UpdateDocumentsQuery,
+        index_creation: bool,
+        request: &HttpRequest,
+    );
 }

meilisearch/src/error.rs

@@ -4,7 +4,6 @@ use byte_unit::{Byte, UnitType};
 use meilisearch_types::document_formats::{DocumentFormatError, PayloadType};
 use meilisearch_types::error::{Code, ErrorCode, ResponseError};
 use meilisearch_types::index_uid::{IndexUid, IndexUidFormatError};
-use meilisearch_types::milli::OrderBy;
 use serde_json::Value;
 use tokio::task::JoinError;
 
@@ -28,20 +27,6 @@ pub enum MeilisearchHttpError {
     EmptyFilter,
     #[error("Invalid syntax for the filter parameter: `expected {}, found: {1}`.", .0.join(", "))]
     InvalidExpression(&'static [&'static str], Value),
-    #[error("Using `federationOptions` is not allowed in a non-federated search.\n - Hint: remove `federationOptions` from query #{0} or add `federation` to the request.")]
-    FederationOptionsInNonFederatedRequest(usize),
-    #[error("Inside `.queries[{0}]`: Using pagination options is not allowed in federated queries.\n - Hint: remove `{1}` from query #{0} or remove `federation` from the request\n - Hint: pass `federation.limit` and `federation.offset` for pagination in federated search")]
-    PaginationInFederatedQuery(usize, &'static str),
-    #[error("Inside `.queries[{0}]`: Using facet options is not allowed in federated queries.\n - Hint: remove `facets` from query #{0} or remove `federation` from the request\n - Hint: pass `federation.facetsByIndex.{1}: {2:?}` for facets in federated search")]
-    FacetsInFederatedQuery(usize, String, Vec<String>),
-    #[error("Inconsistent order for values in facet `{facet}`: index `{previous_uid}` orders {previous_facet_order}, but index `{current_uid}` orders {index_facet_order}.\n - Hint: Remove `federation.mergeFacets` or change `faceting.sortFacetValuesBy` to be consistent in settings.")]
-    InconsistentFacetOrder {
-        facet: String,
-        previous_facet_order: OrderBy,
-        previous_uid: String,
-        index_facet_order: OrderBy,
-        current_uid: String,
-    },
     #[error("A {0} payload is missing.")]
     MissingPayload(PayloadType),
     #[error("Too many search requests running at the same time: {0}. Retry after 10s.")]
@@ -72,7 +57,7 @@ pub enum MeilisearchHttpError {
     DocumentFormat(#[from] DocumentFormatError),
     #[error(transparent)]
     Join(#[from] JoinError),
-    #[error("Invalid request: missing `hybrid` parameter when `vector` is present.")]
+    #[error("Invalid request: missing `hybrid` parameter when both `q` and `vector` are present.")]
     MissingSearchHybrid,
 }
 
@@ -101,16 +86,6 @@ impl ErrorCode for MeilisearchHttpError {
             MeilisearchHttpError::DocumentFormat(e) => e.error_code(),
             MeilisearchHttpError::Join(_) => Code::Internal,
             MeilisearchHttpError::MissingSearchHybrid => Code::MissingSearchHybrid,
-            MeilisearchHttpError::FederationOptionsInNonFederatedRequest(_) => {
-                Code::InvalidMultiSearchFederationOptions
-            }
-            MeilisearchHttpError::PaginationInFederatedQuery(_, _) => {
-                Code::InvalidMultiSearchQueryPagination
-            }
-            MeilisearchHttpError::FacetsInFederatedQuery(..) => Code::InvalidMultiSearchQueryFacets,
-            MeilisearchHttpError::InconsistentFacetOrder { .. } => {
-                Code::InvalidMultiSearchFacetOrder
-            }
         }
     }
 }

meilisearch/src/lib.rs

@@ -13,10 +13,11 @@ pub mod search_queue;
 
 use std::fs::File;
 use std::io::{BufReader, BufWriter};
+use std::num::NonZeroUsize;
 use std::path::Path;
 use std::str::FromStr;
 use std::sync::Arc;
-use std::thread;
+use std::thread::{self, available_parallelism};
 use std::time::Duration;
 
 use actix_cors::Cors;
@@ -36,7 +37,7 @@ use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchR
 use meilisearch_types::milli::update::{IndexDocumentsConfig, IndexDocumentsMethod};
 use meilisearch_types::settings::apply_settings_to_builder;
 use meilisearch_types::tasks::KindWithContent;
-use meilisearch_types::versioning::{check_version_file, create_current_version_file};
+use meilisearch_types::versioning::{check_version_file, create_version_file};
 use meilisearch_types::{compression, milli, VERSION_FILE_NAME};
 pub use option::Opt;
 use option::ScheduleSnapshot;
@@ -117,10 +118,9 @@ pub type LogStderrType = tracing_subscriber::filter::Filtered<
 pub fn create_app(
     index_scheduler: Data<IndexScheduler>,
     auth_controller: Data<AuthController>,
-    search_queue: Data<SearchQueue>,
     opt: Opt,
     logs: (LogRouteHandle, LogStderrHandle),
-    analytics: Data<Analytics>,
+    analytics: Arc<dyn Analytics>,
     enable_dashboard: bool,
 ) -> actix_web::App<
     impl ServiceFactory<
@@ -137,7 +137,6 @@ pub fn create_app(
                 s,
                 index_scheduler.clone(),
                 auth_controller.clone(),
-                search_queue.clone(),
                 &opt,
                 logs,
                 analytics.clone(),
@@ -319,7 +318,7 @@ fn open_or_create_database_unchecked(
     match (
         index_scheduler_builder(),
         auth_controller.map_err(anyhow::Error::from),
-        create_current_version_file(&opt.db_path).map_err(anyhow::Error::from),
+        create_version_file(&opt.db_path).map_err(anyhow::Error::from),
     ) {
         (Ok(i), Ok(a), Ok(())) => Ok((i, a)),
         (Err(e), _, _) | (_, Err(e), _) | (_, _, Err(e)) => {
@@ -470,17 +469,20 @@ pub fn configure_data(
     config: &mut web::ServiceConfig,
     index_scheduler: Data<IndexScheduler>,
     auth: Data<AuthController>,
-    search_queue: Data<SearchQueue>,
     opt: &Opt,
     (logs_route, logs_stderr): (LogRouteHandle, LogStderrHandle),
-    analytics: Data<Analytics>,
+    analytics: Arc<dyn Analytics>,
 ) {
+    let search_queue = SearchQueue::new(
+        opt.experimental_search_queue_size,
+        available_parallelism().unwrap_or(NonZeroUsize::new(2).unwrap()),
+    );
     let http_payload_size_limit = opt.http_payload_size_limit.as_u64() as usize;
     config
         .app_data(index_scheduler)
         .app_data(auth)
-        .app_data(search_queue)
-        .app_data(analytics)
+        .app_data(web::Data::new(search_queue))
+        .app_data(web::Data::from(analytics))
         .app_data(web::Data::new(logs_route))
         .app_data(web::Data::new(logs_stderr))
         .app_data(web::Data::new(opt.clone()))

meilisearch/src/main.rs

@@ -1,11 +1,8 @@
 use std::env;
 use std::io::{stderr, LineWriter, Write};
-use std::num::NonZeroUsize;
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
-use std::thread::available_parallelism;
-use std::time::Duration;
 
 use actix_web::http::KeepAlive;
 use actix_web::web::Data;
@@ -14,7 +11,6 @@ use index_scheduler::IndexScheduler;
 use is_terminal::IsTerminal;
 use meilisearch::analytics::Analytics;
 use meilisearch::option::LogMode;
-use meilisearch::search_queue::SearchQueue;
 use meilisearch::{
     analytics, create_app, setup_meilisearch, LogRouteHandle, LogRouteType, LogStderrHandle,
     LogStderrType, Opt, SubscriberForSecondLayer,
@@ -76,19 +72,6 @@ fn on_panic(info: &std::panic::PanicInfo) {
 
 #[actix_web::main]
 async fn main() -> anyhow::Result<()> {
-    try_main().await.inspect_err(|error| {
-        tracing::error!(%error);
-        let mut current = error.source();
-        let mut depth = 0;
-        while let Some(source) = current {
-            tracing::info!(%source, depth, "Error caused by");
-            current = source.source();
-            depth += 1;
-        }
-    })
-}
-
-async fn try_main() -> anyhow::Result<()> {
     let (opt, config_read_from) = Opt::try_build()?;
 
     std::panic::set_hook(Box::new(on_panic));
@@ -124,12 +107,19 @@ async fn try_main() -> anyhow::Result<()> {
 
     let (index_scheduler, auth_controller) = setup_meilisearch(&opt)?;
 
-    let analytics =
-        analytics::Analytics::new(&opt, index_scheduler.clone(), auth_controller.clone()).await;
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
+    let analytics = if !opt.no_analytics {
+        analytics::SegmentAnalytics::new(&opt, index_scheduler.clone(), auth_controller.clone())
+            .await
+    } else {
+        analytics::MockAnalytics::new(&opt)
+    };
+    #[cfg(any(debug_assertions, not(feature = "analytics")))]
+    let analytics = analytics::MockAnalytics::new(&opt);
 
     print_launch_resume(&opt, analytics.clone(), config_read_from);
 
-    run_http(index_scheduler, auth_controller, opt, log_handle, Arc::new(analytics)).await?;
+    run_http(index_scheduler, auth_controller, opt, log_handle, analytics).await?;
 
     Ok(())
 }
@@ -139,30 +129,17 @@ async fn run_http(
     auth_controller: Arc<AuthController>,
     opt: Opt,
     logs: (LogRouteHandle, LogStderrHandle),
-    analytics: Arc<Analytics>,
+    analytics: Arc<dyn Analytics>,
 ) -> anyhow::Result<()> {
     let enable_dashboard = &opt.env == "development";
     let opt_clone = opt.clone();
     let index_scheduler = Data::from(index_scheduler);
     let auth_controller = Data::from(auth_controller);
-    let analytics = Data::from(analytics);
-    let search_queue = SearchQueue::new(
-        opt.experimental_search_queue_size,
-        available_parallelism()
-            .unwrap_or(NonZeroUsize::new(2).unwrap())
-            .checked_mul(opt.experimental_nb_searches_per_core)
-            .unwrap_or(NonZeroUsize::MAX),
-    )
-    .with_time_to_abort(Duration::from_secs(
-        usize::from(opt.experimental_drop_search_after) as u64
-    ));
-    let search_queue = Data::new(search_queue);
 
     let http_server = HttpServer::new(move || {
         create_app(
             index_scheduler.clone(),
             auth_controller.clone(),
-            search_queue.clone(),
             opt.clone(),
             logs.clone(),
             analytics.clone(),
@@ -174,14 +151,18 @@ async fn run_http(
     .keep_alive(KeepAlive::Os);
 
     if let Some(config) = opt_clone.get_ssl_config()? {
-        http_server.bind_rustls_0_23(opt_clone.http_addr, config)?.run().await?;
+        http_server.bind_rustls_021(opt_clone.http_addr, config)?.run().await?;
     } else {
         http_server.bind(&opt_clone.http_addr)?.run().await?;
     }
     Ok(())
 }
 
-pub fn print_launch_resume(opt: &Opt, analytics: Analytics, config_read_from: Option<PathBuf>) {
+pub fn print_launch_resume(
+    opt: &Opt,
+    analytics: Arc<dyn Analytics>,
+    config_read_from: Option<PathBuf>,
+) {
     let build_info = build_info::BuildInfo::from_build();
 
     let protocol =
@@ -223,6 +204,7 @@ pub fn print_launch_resume(opt: &Opt, analytics: Analytics, config_read_from: Op
         eprintln!("Prototype:\t\t{:?}", prototype);
     }
 
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
     {
         if !opt.no_analytics {
             eprintln!(

meilisearch/src/middleware.rs

@@ -55,17 +55,16 @@ where
         let index_scheduler = req.app_data::<Data<IndexScheduler>>().unwrap();
         let features = index_scheduler.features();
 
-        let request_path = req.path();
-        let request_pattern = req.match_pattern();
-        let metric_path = request_pattern.as_ref().map_or(request_path, String::as_str).to_string();
-        let request_method = req.method().to_string();
-
         if features.check_metrics().is_ok() {
+            let request_path = req.path();
             let is_registered_resource = req.resource_map().has_resource(request_path);
             if is_registered_resource {
+                let request_pattern = req.match_pattern();
+                let metric_path = request_pattern.as_ref().map_or(request_path, String::as_str);
+                let request_method = req.method().to_string();
                 histogram_timer = Some(
                     crate::metrics::MEILISEARCH_HTTP_RESPONSE_TIME_SECONDS
-                        .with_label_values(&[&request_method, &metric_path])
+                        .with_label_values(&[&request_method, metric_path])
                         .start_timer(),
                 );
             }
@@ -77,7 +76,11 @@ where
             let res = fut.await?;
 
             crate::metrics::MEILISEARCH_HTTP_REQUESTS_TOTAL
-                .with_label_values(&[&request_method, &metric_path, res.status().as_str()])
+                .with_label_values(&[
+                    res.request().method().as_str(),
+                    res.request().path(),
+                    res.status().as_str(),
+                ])
                 .inc();
 
             if let Some(histogram_timer) = histogram_timer {

meilisearch/src/option.rs

@@ -2,7 +2,7 @@ use std::env::VarError;
 use std::ffi::OsStr;
 use std::fmt::Display;
 use std::io::{BufReader, Read};
-use std::num::{NonZeroUsize, ParseIntError};
+use std::num::ParseIntError;
 use std::ops::Deref;
 use std::path::PathBuf;
 use std::str::FromStr;
@@ -14,9 +14,11 @@ use clap::Parser;
 use meilisearch_types::features::InstanceTogglableFeatures;
 use meilisearch_types::milli::update::IndexerConfig;
 use meilisearch_types::milli::ThreadPoolNoAbortBuilder;
-use rustls::server::{ServerSessionMemoryCache, WebPkiClientVerifier};
+use rustls::server::{
+    AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, ServerSessionMemoryCache,
+};
 use rustls::RootCertStore;
-use rustls_pemfile::{certs, rsa_private_keys};
+use rustls_pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
 use serde::{Deserialize, Serialize};
 use sysinfo::{MemoryRefreshKind, RefreshKind, System};
 use url::Url;
@@ -29,6 +31,7 @@ const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY";
 const MEILI_ENV: &str = "MEILI_ENV";
 const MEILI_TASK_WEBHOOK_URL: &str = "MEILI_TASK_WEBHOOK_URL";
 const MEILI_TASK_WEBHOOK_AUTHORIZATION_HEADER: &str = "MEILI_TASK_WEBHOOK_AUTHORIZATION_HEADER";
+#[cfg(feature = "analytics")]
 const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS";
 const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT";
 const MEILI_SSL_CERT_PATH: &str = "MEILI_SSL_CERT_PATH";
@@ -51,11 +54,8 @@ const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL";
 const MEILI_EXPERIMENTAL_LOGS_MODE: &str = "MEILI_EXPERIMENTAL_LOGS_MODE";
 const MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS: &str = "MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS";
 const MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE: &str = "MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE";
-const MEILI_EXPERIMENTAL_CONTAINS_FILTER: &str = "MEILI_EXPERIMENTAL_CONTAINS_FILTER";
 const MEILI_EXPERIMENTAL_ENABLE_METRICS: &str = "MEILI_EXPERIMENTAL_ENABLE_METRICS";
 const MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE: &str = "MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE";
-const MEILI_EXPERIMENTAL_DROP_SEARCH_AFTER: &str = "MEILI_EXPERIMENTAL_DROP_SEARCH_AFTER";
-const MEILI_EXPERIMENTAL_NB_SEARCHES_PER_CORE: &str = "MEILI_EXPERIMENTAL_NB_SEARCHES_PER_CORE";
 const MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: &str =
     "MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE";
 const MEILI_EXPERIMENTAL_MAX_NUMBER_OF_BATCHED_TASKS: &str =
@@ -209,6 +209,7 @@ pub struct Opt {
     /// Meilisearch automatically collects data from all instances that do not opt out using this flag.
     /// All gathered data is used solely for the purpose of improving Meilisearch, and can be deleted
     /// at any time.
+    #[cfg(feature = "analytics")]
     #[serde(default)] // we can't send true
     #[clap(long, env = MEILI_NO_ANALYTICS)]
     pub no_analytics: bool,
@@ -338,13 +339,6 @@ pub struct Opt {
     #[serde(default)]
     pub log_level: LogLevel,
 
-    /// Experimental contains filter feature. For more information, see: <https://github.com/orgs/meilisearch/discussions/763>
-    ///
-    /// Enables the experimental contains filter operator.
-    #[clap(long, env = MEILI_EXPERIMENTAL_CONTAINS_FILTER)]
-    #[serde(default)]
-    pub experimental_contains_filter: bool,
-
     /// Experimental metrics feature. For more information, see: <https://github.com/meilisearch/meilisearch/discussions/3518>
     ///
     /// Enables the Prometheus metrics on the `GET /metrics` endpoint.
@@ -357,26 +351,10 @@ pub struct Opt {
     /// Lets you customize the size of the search queue. Meilisearch processes your search requests as fast as possible but once the
     /// queue is full it starts returning HTTP 503, Service Unavailable.
     /// The default value is 1000.
-    #[clap(long, env = MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE, default_value_t = default_experimental_search_queue_size())]
-    #[serde(default = "default_experimental_search_queue_size")]
+    #[clap(long, env = MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE, default_value_t = 1000)]
+    #[serde(default)]
     pub experimental_search_queue_size: usize,
 
-    /// Experimental drop search after. For more information, see: <https://github.com/orgs/meilisearch/discussions/783>
-    ///
-    /// Let you customize after how many seconds Meilisearch should consider a search request irrelevant and drop it.
-    /// The default value is 60.
-    #[clap(long, env = MEILI_EXPERIMENTAL_DROP_SEARCH_AFTER, default_value_t = default_drop_search_after())]
-    #[serde(default = "default_drop_search_after")]
-    pub experimental_drop_search_after: NonZeroUsize,
-
-    /// Experimental number of searches per core. For more information, see: <https://github.com/orgs/meilisearch/discussions/784>
-    ///
-    /// Lets you customize how many search requests can run on each core concurrently.
-    /// The default value is 4.
-    #[clap(long, env = MEILI_EXPERIMENTAL_NB_SEARCHES_PER_CORE, default_value_t = default_nb_searches_per_core())]
-    #[serde(default = "default_nb_searches_per_core")]
-    pub experimental_nb_searches_per_core: NonZeroUsize,
-
     /// Experimental logs mode feature. For more information, see: <https://github.com/orgs/meilisearch/discussions/723>
     ///
     /// Change the mode of the logs on the console.
@@ -423,6 +401,7 @@ pub struct Opt {
 
 impl Opt {
     /// Whether analytics should be enabled or not.
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
     pub fn analytics(&self) -> bool {
         !self.no_analytics
     }
@@ -502,12 +481,10 @@ impl Opt {
             ignore_missing_dump: _,
             ignore_dump_if_db_exists: _,
             config_file_path: _,
+            #[cfg(feature = "analytics")]
             no_analytics,
-            experimental_contains_filter,
             experimental_enable_metrics,
             experimental_search_queue_size,
-            experimental_drop_search_after,
-            experimental_nb_searches_per_core,
             experimental_logs_mode,
             experimental_enable_logs_route,
             experimental_replication_parameters,
@@ -529,7 +506,10 @@ impl Opt {
             );
         }
 
+        #[cfg(feature = "analytics")]
+        {
             export_to_env_if_not_present(MEILI_NO_ANALYTICS, no_analytics.to_string());
+        }
         export_to_env_if_not_present(
             MEILI_HTTP_PAYLOAD_SIZE_LIMIT,
             http_payload_size_limit.to_string(),
@@ -560,10 +540,6 @@ impl Opt {
 
         export_to_env_if_not_present(MEILI_DUMP_DIR, dump_dir);
         export_to_env_if_not_present(MEILI_LOG_LEVEL, log_level.to_string());
-        export_to_env_if_not_present(
-            MEILI_EXPERIMENTAL_CONTAINS_FILTER,
-            experimental_contains_filter.to_string(),
-        );
         export_to_env_if_not_present(
             MEILI_EXPERIMENTAL_ENABLE_METRICS,
             experimental_enable_metrics.to_string(),
@@ -572,14 +548,6 @@ impl Opt {
             MEILI_EXPERIMENTAL_SEARCH_QUEUE_SIZE,
             experimental_search_queue_size.to_string(),
         );
-        export_to_env_if_not_present(
-            MEILI_EXPERIMENTAL_DROP_SEARCH_AFTER,
-            experimental_drop_search_after.to_string(),
-        );
-        export_to_env_if_not_present(
-            MEILI_EXPERIMENTAL_NB_SEARCHES_PER_CORE,
-            experimental_nb_searches_per_core.to_string(),
-        );
         export_to_env_if_not_present(
             MEILI_EXPERIMENTAL_LOGS_MODE,
             experimental_logs_mode.to_string(),
@@ -601,21 +569,23 @@ impl Opt {
 
     pub fn get_ssl_config(&self) -> anyhow::Result<Option<rustls::ServerConfig>> {
         if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) {
-            let config = rustls::ServerConfig::builder();
+            let config = rustls::ServerConfig::builder().with_safe_defaults();
 
             let config = match &self.ssl_auth_path {
                 Some(auth_path) => {
                     let roots = load_certs(auth_path.to_path_buf())?;
                     let mut client_auth_roots = RootCertStore::empty();
                     for root in roots {
-                        client_auth_roots.add(root).unwrap();
+                        client_auth_roots.add(&root).unwrap();
                     }
-                    let mut client_verifier =
-                        WebPkiClientVerifier::builder(client_auth_roots.into());
-                    if !self.ssl_require_auth {
-                        client_verifier = client_verifier.allow_unauthenticated();
+                    if self.ssl_require_auth {
+                        let verifier = AllowAnyAuthenticatedClient::new(client_auth_roots);
+                        config.with_client_cert_verifier(Arc::from(verifier))
+                    } else {
+                        let verifier =
+                            AllowAnyAnonymousOrAuthenticatedClient::new(client_auth_roots);
+                        config.with_client_cert_verifier(Arc::from(verifier))
                     }
-                    config.with_client_cert_verifier(client_verifier.build()?)
                 }
                 None => config.with_no_client_auth(),
             };
@@ -624,7 +594,7 @@ impl Opt {
             let privkey = load_private_key(key_path.to_path_buf())?;
             let ocsp = load_ocsp(&self.ssl_ocsp_path)?;
             let mut config = config
-                .with_single_cert_with_ocsp(certs, privkey, ocsp)
+                .with_single_cert_with_ocsp_and_sct(certs, privkey, ocsp, vec![])
                 .map_err(|_| anyhow::anyhow!("bad certificates/private key"))?;
 
             config.key_log = Arc::new(rustls::KeyLogFile::new());
@@ -634,7 +604,7 @@ impl Opt {
             }
 
             if self.ssl_tickets {
-                config.ticketer = rustls::crypto::ring::Ticketer::new().unwrap();
+                config.ticketer = rustls::Ticketer::new().unwrap();
             }
 
             Ok(Some(config))
@@ -647,7 +617,6 @@ impl Opt {
         InstanceTogglableFeatures {
             metrics: self.experimental_enable_metrics,
             logs_route: self.experimental_enable_logs_route,
-            contains_filter: self.experimental_contains_filter,
         }
     }
 }
@@ -800,26 +769,21 @@ impl Deref for MaxThreads {
     }
 }
 
-fn load_certs(
-    filename: PathBuf,
-) -> anyhow::Result<Vec<rustls::pki_types::CertificateDer<'static>>> {
+fn load_certs(filename: PathBuf) -> anyhow::Result<Vec<rustls::Certificate>> {
     let certfile =
         fs::File::open(filename).map_err(|_| anyhow::anyhow!("cannot open certificate file"))?;
     let mut reader = BufReader::new(certfile);
     certs(&mut reader)
-        .collect::<Result<Vec<_>, _>>()
+        .map(|certs| certs.into_iter().map(rustls::Certificate).collect())
         .map_err(|_| anyhow::anyhow!("cannot read certificate file"))
 }
 
-fn load_private_key(
-    filename: PathBuf,
-) -> anyhow::Result<rustls::pki_types::PrivateKeyDer<'static>> {
+fn load_private_key(filename: PathBuf) -> anyhow::Result<rustls::PrivateKey> {
     let rsa_keys = {
         let keyfile = fs::File::open(filename.clone())
             .map_err(|_| anyhow::anyhow!("cannot open private key file"))?;
         let mut reader = BufReader::new(keyfile);
         rsa_private_keys(&mut reader)
-            .collect::<Result<Vec<_>, _>>()
             .map_err(|_| anyhow::anyhow!("file contains invalid rsa private key"))?
     };
 
@@ -827,21 +791,19 @@ fn load_private_key(
         let keyfile = fs::File::open(filename)
             .map_err(|_| anyhow::anyhow!("cannot open private key file"))?;
         let mut reader = BufReader::new(keyfile);
-        rustls_pemfile::pkcs8_private_keys(&mut reader).collect::<Result<Vec<_>, _>>().map_err(
-            |_| {
+        pkcs8_private_keys(&mut reader).map_err(|_| {
             anyhow::anyhow!(
                 "file contains invalid pkcs8 private key (encrypted keys not supported)"
             )
-            },
-        )?
+        })?
     };
 
     // prefer to load pkcs8 keys
     if !pkcs8_keys.is_empty() {
-        Ok(rustls::pki_types::PrivateKeyDer::Pkcs8(pkcs8_keys[0].clone_key()))
+        Ok(rustls::PrivateKey(pkcs8_keys[0].clone()))
     } else {
         assert!(!rsa_keys.is_empty());
-        Ok(rustls::pki_types::PrivateKeyDer::Pkcs1(rsa_keys[0].clone_key()))
+        Ok(rustls::PrivateKey(rsa_keys[0].clone()))
     }
 }
 
@@ -911,18 +873,6 @@ fn default_dump_dir() -> PathBuf {
     PathBuf::from(DEFAULT_DUMP_DIR)
 }
 
-fn default_experimental_search_queue_size() -> usize {
-    1000
-}
-
-fn default_drop_search_after() -> NonZeroUsize {
-    NonZeroUsize::new(60).unwrap()
-}
-
-fn default_nb_searches_per_core() -> NonZeroUsize {
-    NonZeroUsize::new(4).unwrap()
-}
-
 /// Indicates if a snapshot was scheduled, and if yes with which interval.
 #[derive(Debug, Default, Copy, Clone, Deserialize, Serialize)]
 pub enum ScheduleSnapshot {

meilisearch/src/routes/dump.rs

@@ -4,6 +4,7 @@ use index_scheduler::IndexScheduler;
 use meilisearch_auth::AuthController;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::tasks::KindWithContent;
+use serde_json::json;
 use tracing::debug;
 
 use crate::analytics::Analytics;
@@ -17,16 +18,14 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(web::resource("").route(web::post().to(SeqHandler(create_dump))));
 }
 
-crate::empty_analytics!(DumpAnalytics, "Dump Created");
-
 pub async fn create_dump(
     index_scheduler: GuardedData<ActionPolicy<{ actions::DUMPS_CREATE }>, Data<IndexScheduler>>,
     auth_controller: GuardedData<ActionPolicy<{ actions::DUMPS_CREATE }>, Data<AuthController>>,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    analytics.publish(DumpAnalytics::default(), &req);
+    analytics.publish("Dump Created".to_string(), json!({}), Some(&req));
 
     let task = KindWithContent::DumpCreation {
         keys: auth_controller.list_keys()?,

meilisearch/src/routes/features.rs

@@ -6,10 +6,10 @@ use index_scheduler::IndexScheduler;
 use meilisearch_types::deserr::DeserrJsonError;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::keys::actions;
-use serde::Serialize;
+use serde_json::json;
 use tracing::debug;
 
-use crate::analytics::{Aggregate, Analytics};
+use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::ActionPolicy;
 use crate::extractors::authentication::GuardedData;
 use crate::extractors::sequential_extractor::SeqHandler;
@@ -17,7 +17,7 @@ use crate::extractors::sequential_extractor::SeqHandler;
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(
         web::resource("")
-            .route(web::get().to(get_features))
+            .route(web::get().to(SeqHandler(get_features)))
             .route(web::patch().to(SeqHandler(patch_features))),
     );
 }
@@ -27,9 +27,12 @@ async fn get_features(
         ActionPolicy<{ actions::EXPERIMENTAL_FEATURES_GET }>,
         Data<IndexScheduler>,
     >,
+    req: HttpRequest,
+    analytics: Data<dyn Analytics>,
 ) -> HttpResponse {
     let features = index_scheduler.features();
 
+    analytics.publish("Experimental features Seen".to_string(), json!(null), Some(&req));
     let features = features.runtime_features();
     debug!(returns = ?features, "Get features");
     HttpResponse::Ok().json(features)
@@ -44,39 +47,6 @@ pub struct RuntimeTogglableFeatures {
     pub metrics: Option<bool>,
     #[deserr(default)]
     pub logs_route: Option<bool>,
-    #[deserr(default)]
-    pub edit_documents_by_function: Option<bool>,
-    #[deserr(default)]
-    pub contains_filter: Option<bool>,
-}
-
-#[derive(Serialize)]
-pub struct PatchExperimentalFeatureAnalytics {
-    vector_store: bool,
-    metrics: bool,
-    logs_route: bool,
-    edit_documents_by_function: bool,
-    contains_filter: bool,
-}
-
-impl Aggregate for PatchExperimentalFeatureAnalytics {
-    fn event_name(&self) -> &'static str {
-        "Experimental features Updated"
-    }
-
-    fn aggregate(self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        Box::new(Self {
-            vector_store: new.vector_store,
-            metrics: new.metrics,
-            logs_route: new.logs_route,
-            edit_documents_by_function: new.edit_documents_by_function,
-            contains_filter: new.contains_filter,
-        })
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        serde_json::to_value(*self).unwrap_or_default()
-    }
 }
 
 async fn patch_features(
@@ -86,7 +56,7 @@ async fn patch_features(
     >,
     new_features: AwebJson<RuntimeTogglableFeatures, DeserrJsonError>,
     req: HttpRequest,
-    analytics: Data<Analytics>,
+    analytics: Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let features = index_scheduler.features();
     debug!(parameters = ?new_features, "Patch features");
@@ -96,33 +66,22 @@ async fn patch_features(
         vector_store: new_features.0.vector_store.unwrap_or(old_features.vector_store),
         metrics: new_features.0.metrics.unwrap_or(old_features.metrics),
         logs_route: new_features.0.logs_route.unwrap_or(old_features.logs_route),
-        edit_documents_by_function: new_features
-            .0
-            .edit_documents_by_function
-            .unwrap_or(old_features.edit_documents_by_function),
-        contains_filter: new_features.0.contains_filter.unwrap_or(old_features.contains_filter),
     };
 
     // explicitly destructure for analytics rather than using the `Serialize` implementation, because
     // the it renames to camelCase, which we don't want for analytics.
     // **Do not** ignore fields with `..` or `_` here, because we want to add them in the future.
-    let meilisearch_types::features::RuntimeTogglableFeatures {
-        vector_store,
-        metrics,
-        logs_route,
-        edit_documents_by_function,
-        contains_filter,
-    } = new_features;
+    let meilisearch_types::features::RuntimeTogglableFeatures { vector_store, metrics, logs_route } =
+        new_features;
 
     analytics.publish(
-        PatchExperimentalFeatureAnalytics {
-            vector_store,
-            metrics,
-            logs_route,
-            edit_documents_by_function,
-            contains_filter,
-        },
-        &req,
+        "Experimental features Updated".to_string(),
+        json!({
+            "vector_store": vector_store,
+            "metrics": metrics,
+            "logs_route": logs_route,
+        }),
+        Some(&req),
     );
     index_scheduler.put_runtime_features(new_features)?;
     debug!(returns = ?new_features, "Patch features");

meilisearch/src/routes/indexes/documents.rs

@@ -1,6 +1,4 @@
-use std::collections::HashSet;
 use std::io::ErrorKind;
-use std::marker::PhantomData;
 
 use actix_web::http::header::CONTENT_TYPE;
 use actix_web::web::Data;
@@ -9,7 +7,7 @@ use bstr::ByteSlice as _;
 use deserr::actix_web::{AwebJson, AwebQueryParameter};
 use deserr::Deserr;
 use futures::StreamExt;
-use index_scheduler::{IndexScheduler, RoFeatures, TaskId};
+use index_scheduler::{IndexScheduler, TaskId};
 use meilisearch_types::deserr::query_params::Param;
 use meilisearch_types::deserr::{DeserrJsonError, DeserrQueryParamError};
 use meilisearch_types::document_formats::{read_csv, read_json, read_ndjson, PayloadType};
@@ -25,14 +23,14 @@ use meilisearch_types::tasks::KindWithContent;
 use meilisearch_types::{milli, Document, Index};
 use mime::Mime;
 use once_cell::sync::Lazy;
-use serde::{Deserialize, Serialize};
+use serde::Deserialize;
 use serde_json::Value;
 use tempfile::tempfile;
 use tokio::fs::File;
 use tokio::io::{AsyncSeekExt, AsyncWriteExt, BufWriter};
 use tracing::debug;
 
-use crate::analytics::{Aggregate, AggregateMethod, Analytics};
+use crate::analytics::{Analytics, DocumentDeletionKind, DocumentFetchKind};
 use crate::error::MeilisearchHttpError;
 use crate::error::PayloadError::ReceivePayload;
 use crate::extractors::authentication::policies::*;
@@ -43,7 +41,7 @@ use crate::routes::{
     get_task_id, is_dry_run, PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT,
 };
 use crate::search::{parse_filter, RetrieveVectors};
-use crate::{aggregate_methods, Opt};
+use crate::Opt;
 
 static ACCEPTED_CONTENT_TYPE: Lazy<Vec<String>> = Lazy::new(|| {
     vec!["application/json".to_string(), "application/x-ndjson".to_string(), "text/csv".to_string()]
@@ -84,7 +82,6 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         web::resource("/delete-batch").route(web::post().to(SeqHandler(delete_documents_batch))),
     )
     .service(web::resource("/delete").route(web::post().to(SeqHandler(delete_documents_by_filter))))
-    .service(web::resource("/edit").route(web::post().to(SeqHandler(edit_documents_by_function))))
     .service(web::resource("/fetch").route(web::post().to(SeqHandler(documents_by_query_post))))
     .service(
         web::resource("/{document_id}")
@@ -102,84 +99,12 @@ pub struct GetDocument {
     retrieve_vectors: Param<bool>,
 }
 
-aggregate_methods!(
-    DocumentsGET => "Documents Fetched GET",
-    DocumentsPOST => "Documents Fetched POST",
-);
-
-#[derive(Serialize)]
-pub struct DocumentsFetchAggregator<Method: AggregateMethod> {
-    // a call on ../documents/:doc_id
-    per_document_id: bool,
-    // if a filter was used
-    per_filter: bool,
-
-    #[serde(rename = "vector.retrieve_vectors")]
-    retrieve_vectors: bool,
-
-    // pagination
-    #[serde(rename = "pagination.max_limit")]
-    max_limit: usize,
-    #[serde(rename = "pagination.max_offset")]
-    max_offset: usize,
-
-    marker: std::marker::PhantomData<Method>,
-}
-
-#[derive(Copy, Clone, Debug, PartialEq, Eq)]
-pub enum DocumentFetchKind {
-    PerDocumentId { retrieve_vectors: bool },
-    Normal { with_filter: bool, limit: usize, offset: usize, retrieve_vectors: bool },
-}
-
-impl<Method: AggregateMethod> DocumentsFetchAggregator<Method> {
-    pub fn from_query(query: &DocumentFetchKind) -> Self {
-        let (limit, offset, retrieve_vectors) = match query {
-            DocumentFetchKind::PerDocumentId { retrieve_vectors } => (1, 0, *retrieve_vectors),
-            DocumentFetchKind::Normal { limit, offset, retrieve_vectors, .. } => {
-                (*limit, *offset, *retrieve_vectors)
-            }
-        };
-
-        Self {
-            per_document_id: matches!(query, DocumentFetchKind::PerDocumentId { .. }),
-            per_filter: matches!(query, DocumentFetchKind::Normal { with_filter, .. } if *with_filter),
-            max_limit: limit,
-            max_offset: offset,
-            retrieve_vectors,
-
-            marker: PhantomData,
-        }
-    }
-}
-
-impl<Method: AggregateMethod> Aggregate for DocumentsFetchAggregator<Method> {
-    fn event_name(&self) -> &'static str {
-        Method::event_name()
-    }
-
-    fn aggregate(self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        Box::new(Self {
-            per_document_id: self.per_document_id | new.per_document_id,
-            per_filter: self.per_filter | new.per_filter,
-            retrieve_vectors: self.retrieve_vectors | new.retrieve_vectors,
-            max_limit: self.max_limit.max(new.max_limit),
-            max_offset: self.max_offset.max(new.max_offset),
-            marker: PhantomData,
-        })
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        serde_json::to_value(*self).unwrap_or_default()
-    }
-}
-
 pub async fn get_document(
     index_scheduler: GuardedData<ActionPolicy<{ actions::DOCUMENTS_GET }>, Data<IndexScheduler>>,
     document_param: web::Path<DocumentParam>,
     params: AwebQueryParameter<GetDocument, DeserrQueryParamError>,
     req: HttpRequest,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let DocumentParam { index_uid, document_id } = document_param.into_inner();
     debug!(parameters = ?params, "Get document");
@@ -191,15 +116,8 @@ pub async fn get_document(
     let features = index_scheduler.features();
     let retrieve_vectors = RetrieveVectors::new(param_retrieve_vectors.0, features)?;
 
-    analytics.publish(
-        DocumentsFetchAggregator::<DocumentsGET> {
-            retrieve_vectors: param_retrieve_vectors.0,
-            per_document_id: true,
-            per_filter: false,
-            max_limit: 0,
-            max_offset: 0,
-            marker: PhantomData,
-        },
+    analytics.get_fetch_documents(
+        &DocumentFetchKind::PerDocumentId { retrieve_vectors: param_retrieve_vectors.0 },
         &req,
     );
 
@@ -210,52 +128,17 @@ pub async fn get_document(
     Ok(HttpResponse::Ok().json(document))
 }
 
-#[derive(Serialize)]
-pub struct DocumentsDeletionAggregator {
-    per_document_id: bool,
-    clear_all: bool,
-    per_batch: bool,
-    per_filter: bool,
-}
-
-impl Aggregate for DocumentsDeletionAggregator {
-    fn event_name(&self) -> &'static str {
-        "Documents Deleted"
-    }
-
-    fn aggregate(self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        Box::new(Self {
-            per_document_id: self.per_document_id | new.per_document_id,
-            clear_all: self.clear_all | new.clear_all,
-            per_batch: self.per_batch | new.per_batch,
-            per_filter: self.per_filter | new.per_filter,
-        })
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        serde_json::to_value(*self).unwrap_or_default()
-    }
-}
-
 pub async fn delete_document(
     index_scheduler: GuardedData<ActionPolicy<{ actions::DOCUMENTS_DELETE }>, Data<IndexScheduler>>,
     path: web::Path<DocumentParam>,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let DocumentParam { index_uid, document_id } = path.into_inner();
     let index_uid = IndexUid::try_from(index_uid)?;
 
-    analytics.publish(
-        DocumentsDeletionAggregator {
-            per_document_id: true,
-            clear_all: false,
-            per_batch: false,
-            per_filter: false,
-        },
-        &req,
-    );
+    analytics.delete_documents(DocumentDeletionKind::PerDocumentId, &req);
 
     let task = KindWithContent::DocumentDeletion {
         index_uid: index_uid.to_string(),
@@ -306,19 +189,17 @@ pub async fn documents_by_query_post(
     index_uid: web::Path<String>,
     body: AwebJson<BrowseQuery, DeserrJsonError>,
     req: HttpRequest,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let body = body.into_inner();
     debug!(parameters = ?body, "Get documents POST");
 
-    analytics.publish(
-        DocumentsFetchAggregator::<DocumentsPOST> {
-            per_filter: body.filter.is_some(),
+    analytics.post_fetch_documents(
+        &DocumentFetchKind::Normal {
+            with_filter: body.filter.is_some(),
+            limit: body.limit,
+            offset: body.offset,
             retrieve_vectors: body.retrieve_vectors,
-            max_limit: body.limit,
-            max_offset: body.offset,
-            per_document_id: false,
-            marker: PhantomData,
         },
         &req,
     );
@@ -331,7 +212,7 @@ pub async fn get_documents(
     index_uid: web::Path<String>,
     params: AwebQueryParameter<BrowseQueryGet, DeserrQueryParamError>,
     req: HttpRequest,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!(parameters = ?params, "Get documents GET");
 
@@ -353,14 +234,12 @@ pub async fn get_documents(
         filter,
     };
 
-    analytics.publish(
-        DocumentsFetchAggregator::<DocumentsGET> {
-            per_filter: query.filter.is_some(),
+    analytics.get_fetch_documents(
+        &DocumentFetchKind::Normal {
+            with_filter: query.filter.is_some(),
+            limit: query.limit,
+            offset: query.offset,
             retrieve_vectors: query.retrieve_vectors,
-            max_limit: query.limit,
-            max_offset: query.offset,
-            per_document_id: false,
-            marker: PhantomData,
         },
         &req,
     );
@@ -380,15 +259,8 @@ fn documents_by_query(
     let retrieve_vectors = RetrieveVectors::new(retrieve_vectors, features)?;
 
     let index = index_scheduler.index(&index_uid)?;
-    let (total, documents) = retrieve_documents(
-        &index,
-        offset,
-        limit,
-        filter,
-        fields,
-        retrieve_vectors,
-        index_scheduler.features(),
-    )?;
+    let (total, documents) =
+        retrieve_documents(&index, offset, limit, filter, fields, retrieve_vectors)?;
 
     let ret = PaginationView::new(offset, limit, total as usize, documents);
 
@@ -418,39 +290,6 @@ fn from_char_csv_delimiter(
     }
 }
 
-aggregate_methods!(
-    Replaced => "Documents Added",
-    Updated => "Documents Updated",
-);
-
-#[derive(Serialize)]
-pub struct DocumentsAggregator<T: AggregateMethod> {
-    payload_types: HashSet<String>,
-    primary_key: HashSet<String>,
-    index_creation: bool,
-    #[serde(skip)]
-    method: PhantomData<T>,
-}
-
-impl<Method: AggregateMethod> Aggregate for DocumentsAggregator<Method> {
-    fn event_name(&self) -> &'static str {
-        Method::event_name()
-    }
-
-    fn aggregate(self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        Box::new(Self {
-            payload_types: self.payload_types.union(&new.payload_types).cloned().collect(),
-            primary_key: self.primary_key.union(&new.primary_key).cloned().collect(),
-            index_creation: self.index_creation | new.index_creation,
-            method: PhantomData,
-        })
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        serde_json::to_value(self).unwrap_or_default()
-    }
-}
-
 pub async fn replace_documents(
     index_scheduler: GuardedData<ActionPolicy<{ actions::DOCUMENTS_ADD }>, Data<IndexScheduler>>,
     index_uid: web::Path<String>,
@@ -458,34 +297,14 @@ pub async fn replace_documents(
     body: Payload,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
     debug!(parameters = ?params, "Replace documents");
     let params = params.into_inner();
 
-    let mut content_types = HashSet::new();
-    let content_type = req
-        .headers()
-        .get(CONTENT_TYPE)
-        .and_then(|s| s.to_str().ok())
-        .unwrap_or("unknown")
-        .to_string();
-    content_types.insert(content_type);
-    let mut primary_keys = HashSet::new();
-    if let Some(primary_key) = params.primary_key.clone() {
-        primary_keys.insert(primary_key);
-    }
-    analytics.publish(
-        DocumentsAggregator::<Replaced> {
-            payload_types: content_types,
-            primary_key: primary_keys,
-            index_creation: index_scheduler.index_exists(&index_uid).map_or(true, |x| !x),
-            method: PhantomData,
-        },
-        &req,
-    );
+    analytics.add_documents(&params, index_scheduler.index(&index_uid).is_err(), &req);
 
     let allow_index_creation = index_scheduler.filters().allow_index_creation(&index_uid);
     let uid = get_task_id(&req, &opt)?;
@@ -515,34 +334,14 @@ pub async fn update_documents(
     body: Payload,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
     let params = params.into_inner();
     debug!(parameters = ?params, "Update documents");
 
-    let mut content_types = HashSet::new();
-    let content_type = req
-        .headers()
-        .get(CONTENT_TYPE)
-        .and_then(|s| s.to_str().ok())
-        .unwrap_or("unknown")
-        .to_string();
-    content_types.insert(content_type);
-    let mut primary_keys = HashSet::new();
-    if let Some(primary_key) = params.primary_key.clone() {
-        primary_keys.insert(primary_key);
-    }
-    analytics.publish(
-        DocumentsAggregator::<Updated> {
-            payload_types: content_types,
-            primary_key: primary_keys,
-            index_creation: index_scheduler.index_exists(&index_uid).map_or(true, |x| !x),
-            method: PhantomData,
-        },
-        &req,
-    );
+    analytics.update_documents(&params, index_scheduler.index(&index_uid).is_err(), &req);
 
     let allow_index_creation = index_scheduler.filters().allow_index_creation(&index_uid);
     let uid = get_task_id(&req, &opt)?;
@@ -709,20 +508,12 @@ pub async fn delete_documents_batch(
     body: web::Json<Vec<Value>>,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!(parameters = ?body, "Delete documents by batch");
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
-    analytics.publish(
-        DocumentsDeletionAggregator {
-            per_batch: true,
-            per_document_id: false,
-            clear_all: false,
-            per_filter: false,
-        },
-        &req,
-    );
+    analytics.delete_documents(DocumentDeletionKind::PerBatch, &req);
 
     let ids = body
         .iter()
@@ -755,27 +546,21 @@ pub async fn delete_documents_by_filter(
     body: AwebJson<DocumentDeletionByFilter, DeserrJsonError>,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!(parameters = ?body, "Delete documents by filter");
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
     let index_uid = index_uid.into_inner();
     let filter = body.into_inner().filter;
 
-    analytics.publish(
-        DocumentsDeletionAggregator {
-            per_filter: true,
-            per_document_id: false,
-            clear_all: false,
-            per_batch: false,
-        },
-        &req,
-    );
+    analytics.delete_documents(DocumentDeletionKind::PerFilter, &req);
 
     // we ensure the filter is well formed before enqueuing it
-    crate::search::parse_filter(&filter, Code::InvalidDocumentFilter, index_scheduler.features())?
-        .ok_or(MeilisearchHttpError::EmptyFilter)?;
-
+    || -> Result<_, ResponseError> {
+        Ok(crate::search::parse_filter(&filter)?.ok_or(MeilisearchHttpError::EmptyFilter)?)
+    }()
+    // and whatever was the error, the error code should always be an InvalidDocumentFilter
+    .map_err(|err| ResponseError::from_msg(err.message, Code::InvalidDocumentFilter))?;
     let task = KindWithContent::DocumentDeletionByFilter { index_uid, filter_expr: filter };
 
     let uid = get_task_id(&req, &opt)?;
@@ -789,131 +574,15 @@ pub async fn delete_documents_by_filter(
     Ok(HttpResponse::Accepted().json(task))
 }
 
-#[derive(Debug, Deserr)]
-#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
-pub struct DocumentEditionByFunction {
-    #[deserr(default, error = DeserrJsonError<InvalidDocumentFilter>)]
-    pub filter: Option<Value>,
-    #[deserr(default, error = DeserrJsonError<InvalidDocumentEditionContext>)]
-    pub context: Option<Value>,
-    #[deserr(error = DeserrJsonError<InvalidDocumentEditionFunctionFilter>, missing_field_error = DeserrJsonError::missing_document_edition_function)]
-    pub function: String,
-}
-
-#[derive(Serialize)]
-struct EditDocumentsByFunctionAggregator {
-    // Set to true if at least one request was filtered
-    filtered: bool,
-    // Set to true if at least one request contained a context
-    with_context: bool,
-
-    index_creation: bool,
-}
-
-impl Aggregate for EditDocumentsByFunctionAggregator {
-    fn event_name(&self) -> &'static str {
-        "Documents Edited By Function"
-    }
-
-    fn aggregate(self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        Box::new(Self {
-            filtered: self.filtered | new.filtered,
-            with_context: self.with_context | new.with_context,
-            index_creation: self.index_creation | new.index_creation,
-        })
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        serde_json::to_value(*self).unwrap_or_default()
-    }
-}
-
-pub async fn edit_documents_by_function(
-    index_scheduler: GuardedData<ActionPolicy<{ actions::DOCUMENTS_ALL }>, Data<IndexScheduler>>,
-    index_uid: web::Path<String>,
-    params: AwebJson<DocumentEditionByFunction, DeserrJsonError>,
-    req: HttpRequest,
-    opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
-) -> Result<HttpResponse, ResponseError> {
-    debug!(parameters = ?params, "Edit documents by function");
-
-    index_scheduler
-        .features()
-        .check_edit_documents_by_function("Using the documents edit route")?;
-
-    let index_uid = IndexUid::try_from(index_uid.into_inner())?;
-    let index_uid = index_uid.into_inner();
-    let params = params.into_inner();
-
-    analytics.publish(
-        EditDocumentsByFunctionAggregator {
-            filtered: params.filter.is_some(),
-            with_context: params.context.is_some(),
-            index_creation: index_scheduler.index(&index_uid).is_err(),
-        },
-        &req,
-    );
-
-    let DocumentEditionByFunction { filter, context, function } = params;
-    let engine = milli::rhai::Engine::new();
-    if let Err(e) = engine.compile(&function) {
-        return Err(ResponseError::from_msg(e.to_string(), Code::BadRequest));
-    }
-
-    if let Some(ref filter) = filter {
-        // we ensure the filter is well formed before enqueuing it
-        crate::search::parse_filter(
-            filter,
-            Code::InvalidDocumentFilter,
-            index_scheduler.features(),
-        )?
-        .ok_or(MeilisearchHttpError::EmptyFilter)?;
-    }
-    let task = KindWithContent::DocumentEdition {
-        index_uid,
-        filter_expr: filter,
-        context: match context {
-            Some(Value::Object(m)) => Some(m),
-            None => None,
-            _ => {
-                return Err(ResponseError::from_msg(
-                    "The context must be an object".to_string(),
-                    Code::InvalidDocumentEditionContext,
-                ))
-            }
-        },
-        function,
-    };
-
-    let uid = get_task_id(&req, &opt)?;
-    let dry_run = is_dry_run(&req, &opt)?;
-    let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
-            .await??
-            .into();
-
-    debug!(returns = ?task, "Edit documents by function");
-    Ok(HttpResponse::Accepted().json(task))
-}
-
 pub async fn clear_all_documents(
     index_scheduler: GuardedData<ActionPolicy<{ actions::DOCUMENTS_DELETE }>, Data<IndexScheduler>>,
     index_uid: web::Path<String>,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
-    analytics.publish(
-        DocumentsDeletionAggregator {
-            clear_all: true,
-            per_document_id: false,
-            per_batch: false,
-            per_filter: false,
-        },
-        &req,
-    );
+    analytics.delete_documents(DocumentDeletionKind::ClearAll, &req);
 
     let task = KindWithContent::DocumentClear { index_uid: index_uid.to_string() };
     let uid = get_task_id(&req, &opt)?;
@@ -934,11 +603,16 @@ fn some_documents<'a, 't: 'a>(
     retrieve_vectors: RetrieveVectors,
 ) -> Result<impl Iterator<Item = Result<Document, ResponseError>> + 'a, ResponseError> {
     let fields_ids_map = index.fields_ids_map(rtxn)?;
+    let dictionary = index.document_decompression_dictionary(rtxn)?;
     let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
     let embedding_configs = index.embedding_configs(rtxn)?;
+    let mut buffer = Vec::new();
 
-    Ok(index.iter_documents(rtxn, doc_ids)?.map(move |ret| {
-        ret.map_err(ResponseError::from).and_then(|(key, document)| -> Result<_, ResponseError> {
+    Ok(index.iter_compressed_documents(rtxn, doc_ids)?.map(move |ret| {
+        ret.map_err(ResponseError::from).and_then(
+            |(key, compressed_document)| -> Result<_, ResponseError> {
+                let document = compressed_document
+                    .decompress_with_optional_dictionary(&mut buffer, dictionary.as_ref())?;
                 let mut document = milli::obkv_to_json(&all_fields, &fields_ids_map, document)?;
                 match retrieve_vectors {
                     RetrieveVectors::Ignore => {}
@@ -963,7 +637,8 @@ fn some_documents<'a, 't: 'a>(
                             };
                             vectors.insert(
                                 name,
-                            serde_json::to_value(embeddings).map_err(MeilisearchHttpError::from)?,
+                                serde_json::to_value(embeddings)
+                                    .map_err(MeilisearchHttpError::from)?,
                             );
                         }
                         document.insert("_vectors".into(), vectors.into());
@@ -971,7 +646,8 @@ fn some_documents<'a, 't: 'a>(
                 }
 
                 Ok(document)
-        })
+            },
+        )
     }))
 }
 
@@ -982,12 +658,12 @@ fn retrieve_documents<S: AsRef<str>>(
     filter: Option<Value>,
     attributes_to_retrieve: Option<Vec<S>>,
     retrieve_vectors: RetrieveVectors,
-    features: RoFeatures,
 ) -> Result<(u64, Vec<Document>), ResponseError> {
     let rtxn = index.read_txn()?;
     let filter = &filter;
     let filter = if let Some(filter) = filter {
-        parse_filter(filter, Code::InvalidDocumentFilter, features)?
+        parse_filter(filter)
+            .map_err(|err| ResponseError::from_msg(err.to_string(), Code::InvalidDocumentFilter))?
     } else {
         None
     };

meilisearch/src/routes/indexes/facet_search.rs

@@ -1,5 +1,3 @@
-use std::collections::{BinaryHeap, HashSet};
-
 use actix_web::web::Data;
 use actix_web::{web, HttpRequest, HttpResponse};
 use deserr::actix_web::AwebJson;
@@ -8,19 +6,17 @@ use meilisearch_types::deserr::DeserrJsonError;
 use meilisearch_types::error::deserr_codes::*;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::index_uid::IndexUid;
-use meilisearch_types::locales::Locale;
 use serde_json::Value;
 use tracing::debug;
 
-use crate::analytics::{Aggregate, Analytics};
+use crate::analytics::{Analytics, FacetSearchAggregator};
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::routes::indexes::search::search_kind;
 use crate::search::{
-    add_search_rules, perform_facet_search, FacetSearchResult, HybridQuery, MatchingStrategy,
-    RankingScoreThreshold, SearchQuery, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
-    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
-    DEFAULT_SEARCH_OFFSET,
+    add_search_rules, perform_facet_search, HybridQuery, MatchingStrategy, RankingScoreThreshold,
+    SearchQuery, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
+    DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET,
 };
 use crate::search_queue::SearchQueue;
 
@@ -52,110 +48,6 @@ pub struct FacetSearchQuery {
     pub attributes_to_search_on: Option<Vec<String>>,
     #[deserr(default, error = DeserrJsonError<InvalidSearchRankingScoreThreshold>, default)]
     pub ranking_score_threshold: Option<RankingScoreThreshold>,
-    #[deserr(default, error = DeserrJsonError<InvalidSearchLocales>, default)]
-    pub locales: Option<Vec<Locale>>,
-}
-
-#[derive(Default)]
-pub struct FacetSearchAggregator {
-    // requests
-    total_received: usize,
-    total_succeeded: usize,
-    time_spent: BinaryHeap<usize>,
-
-    // The set of all facetNames that were used
-    facet_names: HashSet<String>,
-
-    // As there been any other parameter than the facetName or facetQuery ones?
-    additional_search_parameters_provided: bool,
-}
-
-impl FacetSearchAggregator {
-    #[allow(clippy::field_reassign_with_default)]
-    pub fn from_query(query: &FacetSearchQuery) -> Self {
-        let FacetSearchQuery {
-            facet_query: _,
-            facet_name,
-            vector,
-            q,
-            filter,
-            matching_strategy,
-            attributes_to_search_on,
-            hybrid,
-            ranking_score_threshold,
-            locales,
-        } = query;
-
-        Self {
-            total_received: 1,
-            facet_names: Some(facet_name.clone()).into_iter().collect(),
-            additional_search_parameters_provided: q.is_some()
-                || vector.is_some()
-                || filter.is_some()
-                || *matching_strategy != MatchingStrategy::default()
-                || attributes_to_search_on.is_some()
-                || hybrid.is_some()
-                || ranking_score_threshold.is_some()
-                || locales.is_some(),
-            ..Default::default()
-        }
-    }
-
-    pub fn succeed(&mut self, result: &FacetSearchResult) {
-        let FacetSearchResult { facet_hits: _, facet_query: _, processing_time_ms } = result;
-        self.total_succeeded = 1;
-        self.time_spent.push(*processing_time_ms as usize);
-    }
-}
-
-impl Aggregate for FacetSearchAggregator {
-    fn event_name(&self) -> &'static str {
-        "Facet Searched POST"
-    }
-
-    fn aggregate(mut self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        for time in new.time_spent {
-            self.time_spent.push(time);
-        }
-
-        Box::new(Self {
-            total_received: self.total_received.saturating_add(new.total_received),
-            total_succeeded: self.total_succeeded.saturating_add(new.total_succeeded),
-            time_spent: self.time_spent,
-            facet_names: self.facet_names.union(&new.facet_names).cloned().collect(),
-            additional_search_parameters_provided: self.additional_search_parameters_provided
-                | new.additional_search_parameters_provided,
-        })
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        let Self {
-            total_received,
-            total_succeeded,
-            time_spent,
-            facet_names,
-            additional_search_parameters_provided,
-        } = *self;
-        // the index of the 99th percentage of value
-        let percentile_99th = 0.99 * (total_succeeded as f64 - 1.) + 1.;
-        // we get all the values in a sorted manner
-        let time_spent = time_spent.into_sorted_vec();
-        // We are only interested by the slowest value of the 99th fastest results
-        let time_spent = time_spent.get(percentile_99th as usize);
-
-        serde_json::json!({
-            "requests": {
-                "99th_response_time":  time_spent.map(|t| format!("{:.2}", t)),
-                "total_succeeded": total_succeeded,
-                "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics
-                "total_received": total_received,
-            },
-            "facets": {
-                "total_distinct_facet_count": facet_names.len(),
-                "additional_search_parameters_provided": additional_search_parameters_provided,
-            },
-        })
-    }
 }
 
 pub async fn search(
@@ -164,18 +56,17 @@ pub async fn search(
     index_uid: web::Path<String>,
     params: AwebJson<FacetSearchQuery, DeserrJsonError>,
     req: HttpRequest,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
     let query = params.into_inner();
     debug!(parameters = ?query, "Facet search");
 
-    let mut aggregate = FacetSearchAggregator::from_query(&query);
+    let mut aggregate = FacetSearchAggregator::from_query(&query, &req);
 
     let facet_query = query.facet_query.clone();
     let facet_name = query.facet_name.clone();
-    let locales = query.locales.clone().map(|l| l.into_iter().map(Into::into).collect());
     let mut search_query = SearchQuery::from(query);
 
     // Tenant token search_rules.
@@ -186,26 +77,16 @@ pub async fn search(
     let index = index_scheduler.index(&index_uid)?;
     let features = index_scheduler.features();
     let search_kind = search_kind(&search_query, &index_scheduler, &index, features)?;
-    let permit = search_queue.try_get_search_permit().await?;
+    let _permit = search_queue.try_get_search_permit().await?;
     let search_result = tokio::task::spawn_blocking(move || {
-        perform_facet_search(
-            &index,
-            search_query,
-            facet_query,
-            facet_name,
-            search_kind,
-            index_scheduler.features(),
-            locales,
-        )
+        perform_facet_search(&index, search_query, facet_query, facet_name, search_kind)
     })
-    .await;
-    permit.drop().await;
-    let search_result = search_result?;
+    .await?;
 
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
     }
-    analytics.publish(aggregate, &req);
+    analytics.post_facet_search(aggregate);
 
     let search_result = search_result?;
 
@@ -225,7 +106,6 @@ impl From<FacetSearchQuery> for SearchQuery {
             attributes_to_search_on,
             hybrid,
             ranking_score_threshold,
-            locales,
         } = value;
 
         SearchQuery {
@@ -254,7 +134,6 @@ impl From<FacetSearchQuery> for SearchQuery {
             attributes_to_search_on,
             hybrid,
             ranking_score_threshold,
-            locales,
         }
     }
 }

meilisearch/src/routes/indexes/mod.rs

@@ -1,4 +1,3 @@
-use std::collections::BTreeSet;
 use std::convert::Infallible;
 
 use actix_web::web::Data;
@@ -14,11 +13,12 @@ use meilisearch_types::index_uid::IndexUid;
 use meilisearch_types::milli::{self, FieldDistribution, Index};
 use meilisearch_types::tasks::KindWithContent;
 use serde::Serialize;
+use serde_json::json;
 use time::OffsetDateTime;
 use tracing::debug;
 
 use super::{get_task_id, Pagination, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT};
-use crate::analytics::{Aggregate, Analytics};
+use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::{AuthenticationError, GuardedData};
 use crate::extractors::sequential_extractor::SeqHandler;
@@ -28,11 +28,8 @@ use crate::Opt;
 pub mod documents;
 pub mod facet_search;
 pub mod search;
-mod search_analytics;
 pub mod settings;
-mod settings_analytics;
 pub mod similar;
-mod similar_analytics;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(
@@ -126,31 +123,12 @@ pub struct IndexCreateRequest {
     primary_key: Option<String>,
 }
 
-#[derive(Serialize)]
-struct IndexCreatedAggregate {
-    primary_key: BTreeSet<String>,
-}
-
-impl Aggregate for IndexCreatedAggregate {
-    fn event_name(&self) -> &'static str {
-        "Index Created"
-    }
-
-    fn aggregate(self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        Box::new(Self { primary_key: self.primary_key.union(&new.primary_key).cloned().collect() })
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        serde_json::to_value(*self).unwrap_or_default()
-    }
-}
-
 pub async fn create_index(
     index_scheduler: GuardedData<ActionPolicy<{ actions::INDEXES_CREATE }>, Data<IndexScheduler>>,
     body: AwebJson<IndexCreateRequest, DeserrJsonError>,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!(parameters = ?body, "Create index");
     let IndexCreateRequest { primary_key, uid } = body.into_inner();
@@ -158,8 +136,9 @@ pub async fn create_index(
     let allow_index_creation = index_scheduler.filters().allow_index_creation(&uid);
     if allow_index_creation {
         analytics.publish(
-            IndexCreatedAggregate { primary_key: primary_key.iter().cloned().collect() },
-            &req,
+            "Index Created".to_string(),
+            json!({ "primary_key": primary_key }),
+            Some(&req),
         );
 
         let task = KindWithContent::IndexCreation { index_uid: uid.to_string(), primary_key };
@@ -215,38 +194,21 @@ pub async fn get_index(
     Ok(HttpResponse::Ok().json(index_view))
 }
 
-#[derive(Serialize)]
-struct IndexUpdatedAggregate {
-    primary_key: BTreeSet<String>,
-}
-
-impl Aggregate for IndexUpdatedAggregate {
-    fn event_name(&self) -> &'static str {
-        "Index Updated"
-    }
-
-    fn aggregate(self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        Box::new(Self { primary_key: self.primary_key.union(&new.primary_key).cloned().collect() })
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        serde_json::to_value(*self).unwrap_or_default()
-    }
-}
 pub async fn update_index(
     index_scheduler: GuardedData<ActionPolicy<{ actions::INDEXES_UPDATE }>, Data<IndexScheduler>>,
     index_uid: web::Path<String>,
     body: AwebJson<UpdateIndexRequest, DeserrJsonError>,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!(parameters = ?body, "Update index");
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
     let body = body.into_inner();
     analytics.publish(
-        IndexUpdatedAggregate { primary_key: body.primary_key.iter().cloned().collect() },
-        &req,
+        "Index Updated".to_string(),
+        json!({ "primary_key": body.primary_key }),
+        Some(&req),
     );
 
     let task = KindWithContent::IndexUpdate {

meilisearch/src/routes/indexes/search.rs

@@ -7,19 +7,17 @@ use meilisearch_types::deserr::{DeserrJsonError, DeserrQueryParamError};
 use meilisearch_types::error::deserr_codes::*;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::index_uid::IndexUid;
-use meilisearch_types::locales::Locale;
 use meilisearch_types::milli;
 use meilisearch_types::serde_cs::vec::CS;
 use serde_json::Value;
 use tracing::debug;
 
-use crate::analytics::Analytics;
+use crate::analytics::{Analytics, SearchAggregator};
 use crate::error::MeilisearchHttpError;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::extractors::sequential_extractor::SeqHandler;
 use crate::metrics::MEILISEARCH_DEGRADED_SEARCH_REQUESTS;
-use crate::routes::indexes::search_analytics::{SearchAggregator, SearchGET, SearchPOST};
 use crate::search::{
     add_search_rules, perform_search, HybridQuery, MatchingStrategy, RankingScoreThreshold,
     RetrieveVectors, SearchKind, SearchQuery, SemanticRatio, DEFAULT_CROP_LENGTH,
@@ -91,8 +89,6 @@ pub struct SearchQueryGet {
     pub hybrid_semantic_ratio: Option<SemanticRatioGet>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSearchRankingScoreThreshold>)]
     pub ranking_score_threshold: Option<RankingScoreThresholdGet>,
-    #[deserr(default, error = DeserrQueryParamError<InvalidSearchLocales>)]
-    pub locales: Option<CS<Locale>>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, deserr::Deserr)]
@@ -129,10 +125,8 @@ impl std::ops::Deref for SemanticRatioGet {
     }
 }
 
-impl TryFrom<SearchQueryGet> for SearchQuery {
-    type Error = ResponseError;
-
-    fn try_from(other: SearchQueryGet) -> Result<Self, Self::Error> {
+impl From<SearchQueryGet> for SearchQuery {
+    fn from(other: SearchQueryGet) -> Self {
         let filter = match other.filter {
             Some(f) => match serde_json::from_str(&f) {
                 Ok(v) => Some(v),
@@ -143,28 +137,19 @@ impl TryFrom<SearchQueryGet> for SearchQuery {
 
         let hybrid = match (other.hybrid_embedder, other.hybrid_semantic_ratio) {
             (None, None) => None,
-            (None, Some(_)) => {
-                return Err(ResponseError::from_msg(
-                    "`hybridEmbedder` is mandatory when `hybridSemanticRatio` is present".into(),
-                    meilisearch_types::error::Code::InvalidHybridQuery,
-                ));
-            }
-            (Some(embedder), None) => {
-                Some(HybridQuery { semantic_ratio: DEFAULT_SEMANTIC_RATIO(), embedder })
+            (None, Some(semantic_ratio)) => {
+                Some(HybridQuery { semantic_ratio: *semantic_ratio, embedder: None })
             }
+            (Some(embedder), None) => Some(HybridQuery {
+                semantic_ratio: DEFAULT_SEMANTIC_RATIO(),
+                embedder: Some(embedder),
+            }),
             (Some(embedder), Some(semantic_ratio)) => {
-                Some(HybridQuery { semantic_ratio: *semantic_ratio, embedder })
+                Some(HybridQuery { semantic_ratio: *semantic_ratio, embedder: Some(embedder) })
             }
         };
 
-        if other.vector.is_some() && hybrid.is_none() {
-            return Err(ResponseError::from_msg(
-                "`hybridEmbedder` is mandatory when `vector` is present".into(),
-                meilisearch_types::error::Code::MissingSearchHybrid,
-            ));
-        }
-
-        Ok(Self {
+        Self {
             q: other.q,
             vector: other.vector.map(CS::into_inner),
             offset: other.offset.0,
@@ -190,8 +175,7 @@ impl TryFrom<SearchQueryGet> for SearchQuery {
             attributes_to_search_on: other.attributes_to_search_on.map(|o| o.into_iter().collect()),
             hybrid,
             ranking_score_threshold: other.ranking_score_threshold.map(|o| o.0),
-            locales: other.locales.map(|o| o.into_iter().collect()),
-        })
+        }
     }
 }
 
@@ -226,36 +210,34 @@ pub async fn search_with_url_query(
     index_uid: web::Path<String>,
     params: AwebQueryParameter<SearchQueryGet, DeserrQueryParamError>,
     req: HttpRequest,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!(parameters = ?params, "Search get");
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
-    let mut query: SearchQuery = params.into_inner().try_into()?;
+    let mut query: SearchQuery = params.into_inner().into();
 
     // Tenant token search_rules.
     if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(&index_uid) {
         add_search_rules(&mut query.filter, search_rules);
     }
 
-    let mut aggregate = SearchAggregator::<SearchGET>::from_query(&query);
+    let mut aggregate = SearchAggregator::from_query(&query, &req);
 
     let index = index_scheduler.index(&index_uid)?;
     let features = index_scheduler.features();
 
     let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)?;
     let retrieve_vector = RetrieveVectors::new(query.retrieve_vectors, features)?;
-    let permit = search_queue.try_get_search_permit().await?;
+    let _permit = search_queue.try_get_search_permit().await?;
     let search_result = tokio::task::spawn_blocking(move || {
-        perform_search(&index, query, search_kind, retrieve_vector, index_scheduler.features())
+        perform_search(&index, query, search_kind, retrieve_vector)
     })
-    .await;
-    permit.drop().await;
-    let search_result = search_result?;
+    .await?;
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
     }
-    analytics.publish(aggregate, &req);
+    analytics.get_search(aggregate);
 
     let search_result = search_result?;
 
@@ -269,7 +251,7 @@ pub async fn search_with_post(
     index_uid: web::Path<String>,
     params: AwebJson<SearchQuery, DeserrJsonError>,
     req: HttpRequest,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
@@ -281,7 +263,7 @@ pub async fn search_with_post(
         add_search_rules(&mut query.filter, search_rules);
     }
 
-    let mut aggregate = SearchAggregator::<SearchPOST>::from_query(&query);
+    let mut aggregate = SearchAggregator::from_query(&query, &req);
 
     let index = index_scheduler.index(&index_uid)?;
 
@@ -290,20 +272,18 @@ pub async fn search_with_post(
     let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)?;
     let retrieve_vectors = RetrieveVectors::new(query.retrieve_vectors, features)?;
 
-    let permit = search_queue.try_get_search_permit().await?;
+    let _permit = search_queue.try_get_search_permit().await?;
     let search_result = tokio::task::spawn_blocking(move || {
-        perform_search(&index, query, search_kind, retrieve_vectors, index_scheduler.features())
+        perform_search(&index, query, search_kind, retrieve_vectors)
     })
-    .await;
-    permit.drop().await;
-    let search_result = search_result?;
+    .await?;
     if let Ok(ref search_result) = search_result {
         aggregate.succeed(search_result);
         if search_result.degraded {
             MEILISEARCH_DEGRADED_SEARCH_REQUESTS.inc();
         }
     }
-    analytics.publish(aggregate, &req);
+    analytics.post_search(aggregate);
 
     let search_result = search_result?;
 
@@ -324,36 +304,44 @@ pub fn search_kind(
         features.check_vector("Passing `hybrid` as a parameter")?;
     }
 
-    // handle with care, the order of cases matters, the semantics is subtle
-    match (query.q.as_deref(), &query.hybrid, query.vector.as_deref()) {
-        // empty query, no vector => placeholder search
-        (Some(q), _, None) if q.trim().is_empty() => Ok(SearchKind::KeywordOnly),
-        // no query, no vector => placeholder search
-        (None, _, None) => Ok(SearchKind::KeywordOnly),
-        // hybrid.semantic_ratio == 1.0 => vector
-        (_, Some(HybridQuery { semantic_ratio, embedder }), v) if **semantic_ratio == 1.0 => {
-            SearchKind::semantic(index_scheduler, index, embedder, v.map(|v| v.len()))
+    // regardless of anything, always do a keyword search when we don't have a vector and the query is whitespace or missing
+    if query.vector.is_none() {
+        match &query.q {
+            Some(q) if q.trim().is_empty() => return Ok(SearchKind::KeywordOnly),
+            None => return Ok(SearchKind::KeywordOnly),
+            _ => {}
         }
-        // hybrid.semantic_ratio == 0.0 => keyword
-        (_, Some(HybridQuery { semantic_ratio, embedder: _ }), _) if **semantic_ratio == 0.0 => {
-            Ok(SearchKind::KeywordOnly)
     }
-        // no query, hybrid, vector => semantic
-        (None, Some(HybridQuery { semantic_ratio: _, embedder }), Some(v)) => {
-            SearchKind::semantic(index_scheduler, index, embedder, Some(v.len()))
-        }
-        // query, no hybrid, no vector => keyword
-        (Some(_), None, None) => Ok(SearchKind::KeywordOnly),
-        // query, hybrid, maybe vector => hybrid
-        (Some(_), Some(HybridQuery { semantic_ratio, embedder }), v) => SearchKind::hybrid(
+
+    match &query.hybrid {
+        Some(HybridQuery { semantic_ratio, embedder }) if **semantic_ratio == 1.0 => {
+            Ok(SearchKind::semantic(
                 index_scheduler,
                 index,
-            embedder,
+                embedder.as_deref(),
+                query.vector.as_ref().map(Vec::len),
+            )?)
+        }
+        Some(HybridQuery { semantic_ratio, embedder: _ }) if **semantic_ratio == 0.0 => {
+            Ok(SearchKind::KeywordOnly)
+        }
+        Some(HybridQuery { semantic_ratio, embedder }) => Ok(SearchKind::hybrid(
+            index_scheduler,
+            index,
+            embedder.as_deref(),
             **semantic_ratio,
-            v.map(|v| v.len()),
-        ),
-
-        (_, None, Some(_)) => Err(MeilisearchHttpError::MissingSearchHybrid.into()),
+            query.vector.as_ref().map(Vec::len),
+        )?),
+        None => match (query.q.as_deref(), query.vector.as_deref()) {
+            (_query, None) => Ok(SearchKind::KeywordOnly),
+            (None, Some(_vector)) => Ok(SearchKind::semantic(
+                index_scheduler,
+                index,
+                None,
+                query.vector.as_ref().map(Vec::len),
+            )?),
+            (Some(_), Some(_)) => Err(MeilisearchHttpError::MissingSearchHybrid.into()),
+        },
     }
 }
 

meilisearch/src/routes/indexes/search_analytics.rs

@@ -1,485 +0,0 @@
-use once_cell::sync::Lazy;
-use regex::Regex;
-use serde_json::{json, Value};
-use std::collections::{BTreeSet, BinaryHeap, HashMap};
-
-use meilisearch_types::locales::Locale;
-
-use crate::{
-    aggregate_methods,
-    analytics::{Aggregate, AggregateMethod},
-    search::{
-        SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
-        DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
-        DEFAULT_SEMANTIC_RATIO,
-    },
-};
-
-aggregate_methods!(
-    SearchGET => "Documents Searched GET",
-    SearchPOST => "Documents Searched POST",
-);
-
-#[derive(Default)]
-pub struct SearchAggregator<Method: AggregateMethod> {
-    // requests
-    total_received: usize,
-    total_succeeded: usize,
-    total_degraded: usize,
-    total_used_negative_operator: usize,
-    time_spent: BinaryHeap<usize>,
-
-    // sort
-    sort_with_geo_point: bool,
-    // every time a request has a filter, this field must be incremented by the number of terms it contains
-    sort_sum_of_criteria_terms: usize,
-    // every time a request has a filter, this field must be incremented by one
-    sort_total_number_of_criteria: usize,
-
-    // distinct
-    distinct: bool,
-
-    // filter
-    filter_with_geo_radius: bool,
-    filter_with_geo_bounding_box: bool,
-    // every time a request has a filter, this field must be incremented by the number of terms it contains
-    filter_sum_of_criteria_terms: usize,
-    // every time a request has a filter, this field must be incremented by one
-    filter_total_number_of_criteria: usize,
-    used_syntax: HashMap<String, usize>,
-
-    // attributes_to_search_on
-    // every time a search is done using attributes_to_search_on
-    attributes_to_search_on_total_number_of_uses: usize,
-
-    // q
-    // The maximum number of terms in a q request
-    max_terms_number: usize,
-
-    // vector
-    // The maximum number of floats in a vector request
-    max_vector_size: usize,
-    // Whether the semantic ratio passed to a hybrid search equals the default ratio.
-    semantic_ratio: bool,
-    hybrid: bool,
-    retrieve_vectors: bool,
-
-    // every time a search is done, we increment the counter linked to the used settings
-    matching_strategy: HashMap<String, usize>,
-
-    // List of the unique Locales passed as parameter
-    locales: BTreeSet<Locale>,
-
-    // pagination
-    max_limit: usize,
-    max_offset: usize,
-    finite_pagination: usize,
-
-    // formatting
-    max_attributes_to_retrieve: usize,
-    max_attributes_to_highlight: usize,
-    highlight_pre_tag: bool,
-    highlight_post_tag: bool,
-    max_attributes_to_crop: usize,
-    crop_marker: bool,
-    show_matches_position: bool,
-    crop_length: bool,
-
-    // facets
-    facets_sum_of_terms: usize,
-    facets_total_number_of_facets: usize,
-
-    // scoring
-    show_ranking_score: bool,
-    show_ranking_score_details: bool,
-    ranking_score_threshold: bool,
-
-    marker: std::marker::PhantomData<Method>,
-}
-
-impl<Method: AggregateMethod> SearchAggregator<Method> {
-    #[allow(clippy::field_reassign_with_default)]
-    pub fn from_query(query: &SearchQuery) -> Self {
-        let SearchQuery {
-            q,
-            vector,
-            offset,
-            limit,
-            page,
-            hits_per_page,
-            attributes_to_retrieve: _,
-            retrieve_vectors,
-            attributes_to_crop: _,
-            crop_length,
-            attributes_to_highlight: _,
-            show_matches_position,
-            show_ranking_score,
-            show_ranking_score_details,
-            filter,
-            sort,
-            distinct,
-            facets: _,
-            highlight_pre_tag,
-            highlight_post_tag,
-            crop_marker,
-            matching_strategy,
-            attributes_to_search_on,
-            hybrid,
-            ranking_score_threshold,
-            locales,
-        } = query;
-
-        let mut ret = Self::default();
-
-        ret.total_received = 1;
-
-        if let Some(ref sort) = sort {
-            ret.sort_total_number_of_criteria = 1;
-            ret.sort_with_geo_point = sort.iter().any(|s| s.contains("_geoPoint("));
-            ret.sort_sum_of_criteria_terms = sort.len();
-        }
-
-        ret.distinct = distinct.is_some();
-
-        if let Some(ref filter) = filter {
-            static RE: Lazy<Regex> = Lazy::new(|| Regex::new("AND | OR").unwrap());
-            ret.filter_total_number_of_criteria = 1;
-
-            let syntax = match filter {
-                Value::String(_) => "string".to_string(),
-                Value::Array(values) => {
-                    if values.iter().map(|v| v.to_string()).any(|s| RE.is_match(&s)) {
-                        "mixed".to_string()
-                    } else {
-                        "array".to_string()
-                    }
-                }
-                _ => "none".to_string(),
-            };
-            // convert the string to a HashMap
-            ret.used_syntax.insert(syntax, 1);
-
-            let stringified_filters = filter.to_string();
-            ret.filter_with_geo_radius = stringified_filters.contains("_geoRadius(");
-            ret.filter_with_geo_bounding_box = stringified_filters.contains("_geoBoundingBox(");
-            ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count();
-        }
-
-        // attributes_to_search_on
-        if attributes_to_search_on.is_some() {
-            ret.attributes_to_search_on_total_number_of_uses = 1;
-        }
-
-        if let Some(ref q) = q {
-            ret.max_terms_number = q.split_whitespace().count();
-        }
-
-        if let Some(ref vector) = vector {
-            ret.max_vector_size = vector.len();
-        }
-        ret.retrieve_vectors |= retrieve_vectors;
-
-        if query.is_finite_pagination() {
-            let limit = hits_per_page.unwrap_or_else(DEFAULT_SEARCH_LIMIT);
-            ret.max_limit = limit;
-            ret.max_offset = page.unwrap_or(1).saturating_sub(1) * limit;
-            ret.finite_pagination = 1;
-        } else {
-            ret.max_limit = *limit;
-            ret.max_offset = *offset;
-            ret.finite_pagination = 0;
-        }
-
-        ret.matching_strategy.insert(format!("{:?}", matching_strategy), 1);
-
-        if let Some(locales) = locales {
-            ret.locales = locales.iter().copied().collect();
-        }
-
-        ret.highlight_pre_tag = *highlight_pre_tag != DEFAULT_HIGHLIGHT_PRE_TAG();
-        ret.highlight_post_tag = *highlight_post_tag != DEFAULT_HIGHLIGHT_POST_TAG();
-        ret.crop_marker = *crop_marker != DEFAULT_CROP_MARKER();
-        ret.crop_length = *crop_length != DEFAULT_CROP_LENGTH();
-        ret.show_matches_position = *show_matches_position;
-
-        ret.show_ranking_score = *show_ranking_score;
-        ret.show_ranking_score_details = *show_ranking_score_details;
-        ret.ranking_score_threshold = ranking_score_threshold.is_some();
-
-        if let Some(hybrid) = hybrid {
-            ret.semantic_ratio = hybrid.semantic_ratio != DEFAULT_SEMANTIC_RATIO();
-            ret.hybrid = true;
-        }
-
-        ret
-    }
-
-    pub fn succeed(&mut self, result: &SearchResult) {
-        let SearchResult {
-            hits: _,
-            query: _,
-            processing_time_ms,
-            hits_info: _,
-            semantic_hit_count: _,
-            facet_distribution: _,
-            facet_stats: _,
-            degraded,
-            used_negative_operator,
-        } = result;
-
-        self.total_succeeded = self.total_succeeded.saturating_add(1);
-        if *degraded {
-            self.total_degraded = self.total_degraded.saturating_add(1);
-        }
-        if *used_negative_operator {
-            self.total_used_negative_operator = self.total_used_negative_operator.saturating_add(1);
-        }
-        self.time_spent.push(*processing_time_ms as usize);
-    }
-}
-
-impl<Method: AggregateMethod> Aggregate for SearchAggregator<Method> {
-    fn event_name(&self) -> &'static str {
-        Method::event_name()
-    }
-
-    fn aggregate(mut self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        let Self {
-            total_received,
-            total_succeeded,
-            mut time_spent,
-            sort_with_geo_point,
-            sort_sum_of_criteria_terms,
-            sort_total_number_of_criteria,
-            distinct,
-            filter_with_geo_radius,
-            filter_with_geo_bounding_box,
-            filter_sum_of_criteria_terms,
-            filter_total_number_of_criteria,
-            used_syntax,
-            attributes_to_search_on_total_number_of_uses,
-            max_terms_number,
-            max_vector_size,
-            retrieve_vectors,
-            matching_strategy,
-            max_limit,
-            max_offset,
-            finite_pagination,
-            max_attributes_to_retrieve,
-            max_attributes_to_highlight,
-            highlight_pre_tag,
-            highlight_post_tag,
-            max_attributes_to_crop,
-            crop_marker,
-            show_matches_position,
-            crop_length,
-            facets_sum_of_terms,
-            facets_total_number_of_facets,
-            show_ranking_score,
-            show_ranking_score_details,
-            semantic_ratio,
-            hybrid,
-            total_degraded,
-            total_used_negative_operator,
-            ranking_score_threshold,
-            mut locales,
-            marker: _,
-        } = *new;
-
-        // request
-        self.total_received = self.total_received.saturating_add(total_received);
-        self.total_succeeded = self.total_succeeded.saturating_add(total_succeeded);
-        self.total_degraded = self.total_degraded.saturating_add(total_degraded);
-        self.total_used_negative_operator =
-            self.total_used_negative_operator.saturating_add(total_used_negative_operator);
-        self.time_spent.append(&mut time_spent);
-
-        // sort
-        self.sort_with_geo_point |= sort_with_geo_point;
-        self.sort_sum_of_criteria_terms =
-            self.sort_sum_of_criteria_terms.saturating_add(sort_sum_of_criteria_terms);
-        self.sort_total_number_of_criteria =
-            self.sort_total_number_of_criteria.saturating_add(sort_total_number_of_criteria);
-
-        // distinct
-        self.distinct |= distinct;
-
-        // filter
-        self.filter_with_geo_radius |= filter_with_geo_radius;
-        self.filter_with_geo_bounding_box |= filter_with_geo_bounding_box;
-        self.filter_sum_of_criteria_terms =
-            self.filter_sum_of_criteria_terms.saturating_add(filter_sum_of_criteria_terms);
-        self.filter_total_number_of_criteria =
-            self.filter_total_number_of_criteria.saturating_add(filter_total_number_of_criteria);
-        for (key, value) in used_syntax.into_iter() {
-            let used_syntax = self.used_syntax.entry(key).or_insert(0);
-            *used_syntax = used_syntax.saturating_add(value);
-        }
-
-        // attributes_to_search_on
-        self.attributes_to_search_on_total_number_of_uses = self
-            .attributes_to_search_on_total_number_of_uses
-            .saturating_add(attributes_to_search_on_total_number_of_uses);
-
-        // q
-        self.max_terms_number = self.max_terms_number.max(max_terms_number);
-
-        // vector
-        self.max_vector_size = self.max_vector_size.max(max_vector_size);
-        self.retrieve_vectors |= retrieve_vectors;
-        self.semantic_ratio |= semantic_ratio;
-        self.hybrid |= hybrid;
-
-        // pagination
-        self.max_limit = self.max_limit.max(max_limit);
-        self.max_offset = self.max_offset.max(max_offset);
-        self.finite_pagination += finite_pagination;
-
-        // formatting
-        self.max_attributes_to_retrieve =
-            self.max_attributes_to_retrieve.max(max_attributes_to_retrieve);
-        self.max_attributes_to_highlight =
-            self.max_attributes_to_highlight.max(max_attributes_to_highlight);
-        self.highlight_pre_tag |= highlight_pre_tag;
-        self.highlight_post_tag |= highlight_post_tag;
-        self.max_attributes_to_crop = self.max_attributes_to_crop.max(max_attributes_to_crop);
-        self.crop_marker |= crop_marker;
-        self.show_matches_position |= show_matches_position;
-        self.crop_length |= crop_length;
-
-        // facets
-        self.facets_sum_of_terms = self.facets_sum_of_terms.saturating_add(facets_sum_of_terms);
-        self.facets_total_number_of_facets =
-            self.facets_total_number_of_facets.saturating_add(facets_total_number_of_facets);
-
-        // matching strategy
-        for (key, value) in matching_strategy.into_iter() {
-            let matching_strategy = self.matching_strategy.entry(key).or_insert(0);
-            *matching_strategy = matching_strategy.saturating_add(value);
-        }
-
-        // scoring
-        self.show_ranking_score |= show_ranking_score;
-        self.show_ranking_score_details |= show_ranking_score_details;
-        self.ranking_score_threshold |= ranking_score_threshold;
-
-        // locales
-        self.locales.append(&mut locales);
-
-        self
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        let Self {
-            total_received,
-            total_succeeded,
-            time_spent,
-            sort_with_geo_point,
-            sort_sum_of_criteria_terms,
-            sort_total_number_of_criteria,
-            distinct,
-            filter_with_geo_radius,
-            filter_with_geo_bounding_box,
-            filter_sum_of_criteria_terms,
-            filter_total_number_of_criteria,
-            used_syntax,
-            attributes_to_search_on_total_number_of_uses,
-            max_terms_number,
-            max_vector_size,
-            retrieve_vectors,
-            matching_strategy,
-            max_limit,
-            max_offset,
-            finite_pagination,
-            max_attributes_to_retrieve,
-            max_attributes_to_highlight,
-            highlight_pre_tag,
-            highlight_post_tag,
-            max_attributes_to_crop,
-            crop_marker,
-            show_matches_position,
-            crop_length,
-            facets_sum_of_terms,
-            facets_total_number_of_facets,
-            show_ranking_score,
-            show_ranking_score_details,
-            semantic_ratio,
-            hybrid,
-            total_degraded,
-            total_used_negative_operator,
-            ranking_score_threshold,
-            locales,
-            marker: _,
-        } = *self;
-
-        // we get all the values in a sorted manner
-        let time_spent = time_spent.into_sorted_vec();
-        // the index of the 99th percentage of value
-        let percentile_99th = time_spent.len() * 99 / 100;
-        // We are only interested by the slowest value of the 99th fastest results
-        let time_spent = time_spent.get(percentile_99th);
-
-        json!({
-            "requests": {
-                "99th_response_time": time_spent.map(|t| format!("{:.2}", t)),
-                "total_succeeded": total_succeeded,
-                "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics
-                "total_received": total_received,
-                "total_degraded": total_degraded,
-                "total_used_negative_operator": total_used_negative_operator,
-            },
-            "sort": {
-                "with_geoPoint": sort_with_geo_point,
-                "avg_criteria_number": format!("{:.2}", sort_sum_of_criteria_terms as f64 / sort_total_number_of_criteria as f64),
-            },
-            "distinct": distinct,
-            "filter": {
-               "with_geoRadius": filter_with_geo_radius,
-               "with_geoBoundingBox": filter_with_geo_bounding_box,
-               "avg_criteria_number": format!("{:.2}", filter_sum_of_criteria_terms as f64 / filter_total_number_of_criteria as f64),
-               "most_used_syntax": used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)),
-            },
-            "attributes_to_search_on": {
-               "total_number_of_uses": attributes_to_search_on_total_number_of_uses,
-            },
-            "q": {
-               "max_terms_number": max_terms_number,
-            },
-            "vector": {
-                "max_vector_size": max_vector_size,
-                "retrieve_vectors": retrieve_vectors,
-            },
-            "hybrid": {
-                "enabled": hybrid,
-                "semantic_ratio": semantic_ratio,
-            },
-            "pagination": {
-               "max_limit": max_limit,
-               "max_offset": max_offset,
-               "most_used_navigation": if finite_pagination > (total_received / 2) { "exhaustive" } else { "estimated" },
-            },
-            "formatting": {
-                "max_attributes_to_retrieve": max_attributes_to_retrieve,
-                "max_attributes_to_highlight": max_attributes_to_highlight,
-                "highlight_pre_tag": highlight_pre_tag,
-                "highlight_post_tag": highlight_post_tag,
-                "max_attributes_to_crop": max_attributes_to_crop,
-                "crop_marker": crop_marker,
-                "show_matches_position": show_matches_position,
-                "crop_length": crop_length,
-            },
-            "facets": {
-                "avg_facets_number": format!("{:.2}", facets_sum_of_terms as f64 / facets_total_number_of_facets as f64),
-            },
-            "matching_strategy": {
-                "most_used_strategy": matching_strategy.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)),
-            },
-            "locales": locales,
-            "scoring": {
-                "show_ranking_score": show_ranking_score,
-                "show_ranking_score_details": show_ranking_score_details,
-                "ranking_score_threshold": ranking_score_threshold,
-            },
-        })
-    }
-}

meilisearch/src/routes/indexes/settings.rs

@@ -1,14 +1,15 @@
-use super::settings_analytics::*;
 use actix_web::web::Data;
 use actix_web::{web, HttpRequest, HttpResponse};
 use deserr::actix_web::AwebJson;
 use index_scheduler::IndexScheduler;
 use meilisearch_types::deserr::DeserrJsonError;
 use meilisearch_types::error::ResponseError;
+use meilisearch_types::facet_values_sort::FacetValuesSort;
 use meilisearch_types::index_uid::IndexUid;
 use meilisearch_types::milli::update::Setting;
-use meilisearch_types::settings::{settings, SecretPolicy, Settings, Unchecked};
+use meilisearch_types::settings::{settings, RankingRuleView, SecretPolicy, Settings, Unchecked};
 use meilisearch_types::tasks::KindWithContent;
+use serde_json::json;
 use tracing::debug;
 
 use crate::analytics::Analytics;
@@ -19,7 +20,7 @@ use crate::Opt;
 
 #[macro_export]
 macro_rules! make_setting_route {
-    ($route:literal, $update_verb:ident, $type:ty, $err_ty:ty, $attr:ident, $camelcase_attr:literal, $analytics:ident) => {
+    ($route:literal, $update_verb:ident, $type:ty, $err_ty:ty, $attr:ident, $camelcase_attr:literal, $analytics_var:ident, $analytics:expr) => {
         pub mod $attr {
             use actix_web::web::Data;
             use actix_web::{web, HttpRequest, HttpResponse, Resource};
@@ -79,7 +80,7 @@ macro_rules! make_setting_route {
                 body: deserr::actix_web::AwebJson<Option<$type>, $err_ty>,
                 req: HttpRequest,
                 opt: web::Data<Opt>,
-                analytics: web::Data<Analytics>,
+                $analytics_var: web::Data<dyn Analytics>,
             ) -> std::result::Result<HttpResponse, ResponseError> {
                 let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
@@ -87,10 +88,7 @@ macro_rules! make_setting_route {
                 debug!(parameters = ?body, "Update settings");
 
                 #[allow(clippy::redundant_closure_call)]
-                analytics.publish(
-                    $crate::routes::indexes::settings_analytics::$analytics::new(body.as_ref()).into_settings(),
-                    &req,
-                );
+                $analytics(&body, &req);
 
                 let new_settings = Settings {
                     $attr: match body {
@@ -162,7 +160,21 @@ make_setting_route!(
     >,
     filterable_attributes,
     "filterableAttributes",
-    FilterableAttributesAnalytics
+    analytics,
+    |setting: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "FilterableAttributes Updated".to_string(),
+            json!({
+                "filterable_attributes": {
+                    "total": setting.as_ref().map(|filter| filter.len()).unwrap_or(0),
+                    "has_geo": setting.as_ref().map(|filter| filter.contains("_geo")).unwrap_or(false),
+                }
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -174,7 +186,21 @@ make_setting_route!(
     >,
     sortable_attributes,
     "sortableAttributes",
-    SortableAttributesAnalytics
+    analytics,
+    |setting: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "SortableAttributes Updated".to_string(),
+            json!({
+                "sortable_attributes": {
+                    "total": setting.as_ref().map(|sort| sort.len()),
+                    "has_geo": setting.as_ref().map(|sort| sort.contains("_geo")),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -186,7 +212,21 @@ make_setting_route!(
     >,
     displayed_attributes,
     "displayedAttributes",
-    DisplayedAttributesAnalytics
+    analytics,
+    |displayed: &Option<Vec<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "DisplayedAttributes Updated".to_string(),
+            json!({
+                "displayed_attributes": {
+                    "total": displayed.as_ref().map(|displayed| displayed.len()),
+                    "with_wildcard": displayed.as_ref().map(|displayed| displayed.iter().any(|displayed| displayed == "*")),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -198,7 +238,40 @@ make_setting_route!(
     >,
     typo_tolerance,
     "typoTolerance",
-    TypoToleranceAnalytics
+    analytics,
+    |setting: &Option<meilisearch_types::settings::TypoSettings>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "TypoTolerance Updated".to_string(),
+            json!({
+                "typo_tolerance": {
+                    "enabled": setting.as_ref().map(|s| !matches!(s.enabled, Setting::Set(false))),
+                    "disable_on_attributes": setting
+                        .as_ref()
+                        .and_then(|s| s.disable_on_attributes.as_ref().set().map(|m| !m.is_empty())),
+                    "disable_on_words": setting
+                        .as_ref()
+                        .and_then(|s| s.disable_on_words.as_ref().set().map(|m| !m.is_empty())),
+                    "min_word_size_for_one_typo": setting
+                        .as_ref()
+                        .and_then(|s| s.min_word_size_for_typos
+                            .as_ref()
+                            .set()
+                            .map(|s| s.one_typo.set()))
+                        .flatten(),
+                    "min_word_size_for_two_typos": setting
+                        .as_ref()
+                        .and_then(|s| s.min_word_size_for_typos
+                            .as_ref()
+                            .set()
+                            .map(|s| s.two_typos.set()))
+                        .flatten(),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -210,7 +283,21 @@ make_setting_route!(
     >,
     searchable_attributes,
     "searchableAttributes",
-    SearchableAttributesAnalytics
+    analytics,
+    |setting: &Option<Vec<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "SearchableAttributes Updated".to_string(),
+            json!({
+                "searchable_attributes": {
+                    "total": setting.as_ref().map(|searchable| searchable.len()),
+                    "with_wildcard": setting.as_ref().map(|searchable| searchable.iter().any(|searchable| searchable == "*")),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -222,7 +309,20 @@ make_setting_route!(
     >,
     stop_words,
     "stopWords",
-    StopWordsAnalytics
+    analytics,
+    |stop_words: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "StopWords Updated".to_string(),
+            json!({
+                "stop_words": {
+                    "total": stop_words.as_ref().map(|stop_words| stop_words.len()),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -234,7 +334,20 @@ make_setting_route!(
     >,
     non_separator_tokens,
     "nonSeparatorTokens",
-    NonSeparatorTokensAnalytics
+    analytics,
+    |non_separator_tokens: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "nonSeparatorTokens Updated".to_string(),
+            json!({
+                "non_separator_tokens": {
+                    "total": non_separator_tokens.as_ref().map(|non_separator_tokens| non_separator_tokens.len()),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -246,7 +359,20 @@ make_setting_route!(
     >,
     separator_tokens,
     "separatorTokens",
-    SeparatorTokensAnalytics
+    analytics,
+    |separator_tokens: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "separatorTokens Updated".to_string(),
+            json!({
+                "separator_tokens": {
+                    "total": separator_tokens.as_ref().map(|separator_tokens| separator_tokens.len()),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -258,7 +384,20 @@ make_setting_route!(
     >,
     dictionary,
     "dictionary",
-    DictionaryAnalytics
+    analytics,
+    |dictionary: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "dictionary Updated".to_string(),
+            json!({
+                "dictionary": {
+                    "total": dictionary.as_ref().map(|dictionary| dictionary.len()),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -270,7 +409,20 @@ make_setting_route!(
     >,
     synonyms,
     "synonyms",
-    SynonymsAnalytics
+    analytics,
+    |synonyms: &Option<std::collections::BTreeMap<String, Vec<String>>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "Synonyms Updated".to_string(),
+            json!({
+                "synonyms": {
+                    "total": synonyms.as_ref().map(|synonyms| synonyms.len()),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -282,7 +434,19 @@ make_setting_route!(
     >,
     distinct_attribute,
     "distinctAttribute",
-    DistinctAttributeAnalytics
+    analytics,
+    |distinct: &Option<String>, req: &HttpRequest| {
+        use serde_json::json;
+        analytics.publish(
+            "DistinctAttribute Updated".to_string(),
+            json!({
+                "distinct_attribute": {
+                    "set": distinct.is_some(),
+                }
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -294,19 +458,20 @@ make_setting_route!(
     >,
     proximity_precision,
     "proximityPrecision",
-    ProximityPrecisionAnalytics
+    analytics,
+    |precision: &Option<meilisearch_types::settings::ProximityPrecisionView>, req: &HttpRequest| {
+        use serde_json::json;
+        analytics.publish(
+            "ProximityPrecision Updated".to_string(),
+            json!({
+                "proximity_precision": {
+                    "set": precision.is_some(),
+                    "value": precision.unwrap_or_default(),
+                }
+            }),
+            Some(req),
         );
-
-make_setting_route!(
-    "/localized-attributes",
-    put,
-    Vec<meilisearch_types::locales::LocalizedAttributesRuleView>,
-    meilisearch_types::deserr::DeserrJsonError<
-        meilisearch_types::error::deserr_codes::InvalidSettingsLocalizedAttributes,
-    >,
-    localized_attributes,
-    "localizedAttributes",
-    LocalesAnalytics
+    }
 );
 
 make_setting_route!(
@@ -318,7 +483,26 @@ make_setting_route!(
     >,
     ranking_rules,
     "rankingRules",
-    RankingRulesAnalytics
+    analytics,
+    |setting: &Option<Vec<meilisearch_types::settings::RankingRuleView>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "RankingRules Updated".to_string(),
+            json!({
+                "ranking_rules": {
+                    "words_position": setting.as_ref().map(|rr| rr.iter().position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Words))),
+                    "typo_position": setting.as_ref().map(|rr| rr.iter().position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Typo))),
+                    "proximity_position": setting.as_ref().map(|rr| rr.iter().position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Proximity))),
+                    "attribute_position": setting.as_ref().map(|rr| rr.iter().position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Attribute))),
+                    "sort_position": setting.as_ref().map(|rr| rr.iter().position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Sort))),
+                    "exactness_position": setting.as_ref().map(|rr| rr.iter().position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Exactness))),
+                    "values": setting.as_ref().map(|rr| rr.iter().filter(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Asc(_) | meilisearch_types::settings::RankingRuleView::Desc(_)) ).map(|x| x.to_string()).collect::<Vec<_>>().join(", ")),
+                }
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -330,7 +514,25 @@ make_setting_route!(
     >,
     faceting,
     "faceting",
-    FacetingAnalytics
+    analytics,
+    |setting: &Option<meilisearch_types::settings::FacetingSettings>, req: &HttpRequest| {
+        use serde_json::json;
+        use meilisearch_types::facet_values_sort::FacetValuesSort;
+
+        analytics.publish(
+            "Faceting Updated".to_string(),
+            json!({
+                "faceting": {
+                    "max_values_per_facet": setting.as_ref().and_then(|s| s.max_values_per_facet.set()),
+                    "sort_facet_values_by_star_count": setting.as_ref().and_then(|s| {
+                        s.sort_facet_values_by.as_ref().set().map(|s| s.iter().any(|(k, v)| k == "*" && v == &FacetValuesSort::Count))
+                    }),
+                    "sort_facet_values_by_total": setting.as_ref().and_then(|s| s.sort_facet_values_by.as_ref().set().map(|s| s.len())),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -342,7 +544,20 @@ make_setting_route!(
     >,
     pagination,
     "pagination",
-    PaginationAnalytics
+    analytics,
+    |setting: &Option<meilisearch_types::settings::PaginationSettings>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "Pagination Updated".to_string(),
+            json!({
+                "pagination": {
+                    "max_total_hits": setting.as_ref().and_then(|s| s.max_total_hits.set()),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -354,8 +569,59 @@ make_setting_route!(
     >,
     embedders,
     "embedders",
-    EmbeddersAnalytics
+    analytics,
+    |setting: &Option<std::collections::BTreeMap<String, Setting<meilisearch_types::milli::vector::settings::EmbeddingSettings>>>, req: &HttpRequest| {
+
+
+        analytics.publish(
+            "Embedders Updated".to_string(),
+            serde_json::json!({"embedders": crate::routes::indexes::settings::embedder_analytics(setting.as_ref())}),
+            Some(req),
         );
+    }
+);
+
+fn embedder_analytics(
+    setting: Option<
+        &std::collections::BTreeMap<
+            String,
+            Setting<meilisearch_types::milli::vector::settings::EmbeddingSettings>,
+        >,
+    >,
+) -> serde_json::Value {
+    let mut sources = std::collections::HashSet::new();
+
+    if let Some(s) = &setting {
+        for source in s
+            .values()
+            .filter_map(|config| config.clone().set())
+            .filter_map(|config| config.source.set())
+        {
+            use meilisearch_types::milli::vector::settings::EmbedderSource;
+            match source {
+                EmbedderSource::OpenAi => sources.insert("openAi"),
+                EmbedderSource::HuggingFace => sources.insert("huggingFace"),
+                EmbedderSource::UserProvided => sources.insert("userProvided"),
+                EmbedderSource::Ollama => sources.insert("ollama"),
+                EmbedderSource::Rest => sources.insert("rest"),
+            };
+        }
+    };
+
+    let document_template_used = setting.as_ref().map(|map| {
+        map.values()
+            .filter_map(|config| config.clone().set())
+            .any(|config| config.document_template.set().is_some())
+    });
+
+    json!(
+        {
+            "total": setting.as_ref().map(|s| s.len()),
+            "sources": sources,
+            "document_template_used": document_template_used,
+        }
+    )
+}
 
 make_setting_route!(
     "/search-cutoff-ms",
@@ -366,7 +632,14 @@ make_setting_route!(
     >,
     search_cutoff_ms,
     "searchCutoffMs",
-    SearchCutoffMsAnalytics
+    analytics,
+    |setting: &Option<u64>, req: &HttpRequest| {
+        analytics.publish(
+            "Search Cutoff Updated".to_string(),
+            serde_json::json!({"search_cutoff_ms": setting }),
+            Some(req),
+        );
+    }
 );
 
 macro_rules! generate_configure {
@@ -387,7 +660,6 @@ generate_configure!(
     filterable_attributes,
     sortable_attributes,
     displayed_attributes,
-    localized_attributes,
     searchable_attributes,
     distinct_attribute,
     proximity_precision,
@@ -410,7 +682,7 @@ pub async fn update_all(
     body: AwebJson<Settings<Unchecked>, DeserrJsonError>,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
@@ -419,45 +691,103 @@ pub async fn update_all(
     let new_settings = validate_settings(new_settings, &index_scheduler)?;
 
     analytics.publish(
-        SettingsAnalytics {
-            ranking_rules: RankingRulesAnalytics::new(new_settings.ranking_rules.as_ref().set()),
-            searchable_attributes: SearchableAttributesAnalytics::new(
-                new_settings.searchable_attributes.as_ref().set(),
-            ),
-            displayed_attributes: DisplayedAttributesAnalytics::new(
-                new_settings.displayed_attributes.as_ref().set(),
-            ),
-            sortable_attributes: SortableAttributesAnalytics::new(
-                new_settings.sortable_attributes.as_ref().set(),
-            ),
-            filterable_attributes: FilterableAttributesAnalytics::new(
-                new_settings.filterable_attributes.as_ref().set(),
-            ),
-            distinct_attribute: DistinctAttributeAnalytics::new(
-                new_settings.distinct_attribute.as_ref().set(),
-            ),
-            proximity_precision: ProximityPrecisionAnalytics::new(
-                new_settings.proximity_precision.as_ref().set(),
-            ),
-            typo_tolerance: TypoToleranceAnalytics::new(new_settings.typo_tolerance.as_ref().set()),
-            faceting: FacetingAnalytics::new(new_settings.faceting.as_ref().set()),
-            pagination: PaginationAnalytics::new(new_settings.pagination.as_ref().set()),
-            stop_words: StopWordsAnalytics::new(new_settings.stop_words.as_ref().set()),
-            synonyms: SynonymsAnalytics::new(new_settings.synonyms.as_ref().set()),
-            embedders: EmbeddersAnalytics::new(new_settings.embedders.as_ref().set()),
-            search_cutoff_ms: SearchCutoffMsAnalytics::new(
-                new_settings.search_cutoff_ms.as_ref().set(),
-            ),
-            locales: LocalesAnalytics::new(new_settings.localized_attributes.as_ref().set()),
-            dictionary: DictionaryAnalytics::new(new_settings.dictionary.as_ref().set()),
-            separator_tokens: SeparatorTokensAnalytics::new(
-                new_settings.separator_tokens.as_ref().set(),
-            ),
-            non_separator_tokens: NonSeparatorTokensAnalytics::new(
-                new_settings.non_separator_tokens.as_ref().set(),
-            ),
+        "Settings Updated".to_string(),
+        json!({
+           "ranking_rules": {
+                "words_position": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().position(|s| matches!(s, RankingRuleView::Words))),
+                "typo_position": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().position(|s| matches!(s, RankingRuleView::Typo))),
+                "proximity_position": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().position(|s| matches!(s, RankingRuleView::Proximity))),
+                "attribute_position": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().position(|s| matches!(s, RankingRuleView::Attribute))),
+                "sort_position": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().position(|s| matches!(s, RankingRuleView::Sort))),
+                "exactness_position": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().position(|s| matches!(s, RankingRuleView::Exactness))),
+                "values": new_settings.ranking_rules.as_ref().set().map(|rr| rr.iter().filter(|s| !matches!(s, RankingRuleView::Asc(_) | RankingRuleView::Desc(_)) ).map(|x| x.to_string()).collect::<Vec<_>>().join(", ")),
             },
-        &req,
+            "searchable_attributes": {
+                "total": new_settings.searchable_attributes.as_ref().set().map(|searchable| searchable.len()),
+                "with_wildcard": new_settings.searchable_attributes.as_ref().set().map(|searchable| searchable.iter().any(|searchable| searchable == "*")),
+            },
+            "displayed_attributes": {
+                "total": new_settings.displayed_attributes.as_ref().set().map(|displayed| displayed.len()),
+                "with_wildcard": new_settings.displayed_attributes.as_ref().set().map(|displayed| displayed.iter().any(|displayed| displayed == "*")),
+            },
+           "sortable_attributes": {
+                "total": new_settings.sortable_attributes.as_ref().set().map(|sort| sort.len()),
+                "has_geo": new_settings.sortable_attributes.as_ref().set().map(|sort| sort.iter().any(|s| s == "_geo")),
+            },
+           "filterable_attributes": {
+                "total": new_settings.filterable_attributes.as_ref().set().map(|filter| filter.len()),
+                "has_geo": new_settings.filterable_attributes.as_ref().set().map(|filter| filter.iter().any(|s| s == "_geo")),
+            },
+            "distinct_attribute": {
+                "set": new_settings.distinct_attribute.as_ref().set().is_some()
+            },
+            "proximity_precision": {
+                "set": new_settings.proximity_precision.as_ref().set().is_some(),
+                "value": new_settings.proximity_precision.as_ref().set().copied().unwrap_or_default()
+            },
+            "typo_tolerance": {
+                "enabled": new_settings.typo_tolerance
+                    .as_ref()
+                    .set()
+                    .and_then(|s| s.enabled.as_ref().set())
+                    .copied(),
+                "disable_on_attributes": new_settings.typo_tolerance
+                    .as_ref()
+                    .set()
+                    .and_then(|s| s.disable_on_attributes.as_ref().set().map(|m| !m.is_empty())),
+                "disable_on_words": new_settings.typo_tolerance
+                    .as_ref()
+                    .set()
+                    .and_then(|s| s.disable_on_words.as_ref().set().map(|m| !m.is_empty())),
+                "min_word_size_for_one_typo": new_settings.typo_tolerance
+                    .as_ref()
+                    .set()
+                    .and_then(|s| s.min_word_size_for_typos
+                        .as_ref()
+                        .set()
+                        .map(|s| s.one_typo.set()))
+                    .flatten(),
+                "min_word_size_for_two_typos": new_settings.typo_tolerance
+                    .as_ref()
+                    .set()
+                    .and_then(|s| s.min_word_size_for_typos
+                        .as_ref()
+                        .set()
+                        .map(|s| s.two_typos.set()))
+                    .flatten(),
+            },
+            "faceting": {
+                "max_values_per_facet": new_settings.faceting
+                    .as_ref()
+                    .set()
+                    .and_then(|s| s.max_values_per_facet.as_ref().set()),
+                "sort_facet_values_by_star_count": new_settings.faceting
+                    .as_ref()
+                    .set()
+                    .and_then(|s| {
+                        s.sort_facet_values_by.as_ref().set().map(|s| s.iter().any(|(k, v)| k == "*" && v == &FacetValuesSort::Count))
+                    }),
+                "sort_facet_values_by_total": new_settings.faceting
+                    .as_ref()
+                    .set()
+                    .and_then(|s| s.sort_facet_values_by.as_ref().set().map(|s| s.len())),
+            },
+            "pagination": {
+                "max_total_hits": new_settings.pagination
+                    .as_ref()
+                    .set()
+                    .and_then(|s| s.max_total_hits.as_ref().set()),
+            },
+            "stop_words": {
+                "total": new_settings.stop_words.as_ref().set().map(|stop_words| stop_words.len()),
+            },
+            "synonyms": {
+                "total": new_settings.synonyms.as_ref().set().map(|synonyms| synonyms.len()),
+            },
+            "embedders": crate::routes::indexes::settings::embedder_analytics(new_settings.embedders.as_ref().set()),
+            "search_cutoff_ms": new_settings.search_cutoff_ms.as_ref().set(),
+        }),
+        Some(&req),
     );
 
     let allow_index_creation = index_scheduler.filters().allow_index_creation(&index_uid);

meilisearch/src/routes/indexes/settings_analytics.rs

@@ -1,621 +0,0 @@
-//! All the structures used to make the analytics on the settings works.
-//! The signatures of the `new` functions are not very rust idiomatic because they must match the types received
-//! through the sub-settings route directly without any manipulation.
-//! This is why we often use a `Option<&Vec<_>>` instead of a `Option<&[_]>`.
-
-use meilisearch_types::locales::{Locale, LocalizedAttributesRuleView};
-use meilisearch_types::milli::update::Setting;
-use meilisearch_types::milli::vector::settings::EmbeddingSettings;
-use meilisearch_types::settings::{
-    FacetingSettings, PaginationSettings, ProximityPrecisionView, TypoSettings,
-};
-use meilisearch_types::{facet_values_sort::FacetValuesSort, settings::RankingRuleView};
-use serde::Serialize;
-use std::collections::{BTreeMap, BTreeSet, HashSet};
-
-use crate::analytics::Aggregate;
-
-#[derive(Serialize, Default)]
-pub struct SettingsAnalytics {
-    pub ranking_rules: RankingRulesAnalytics,
-    pub searchable_attributes: SearchableAttributesAnalytics,
-    pub displayed_attributes: DisplayedAttributesAnalytics,
-    pub sortable_attributes: SortableAttributesAnalytics,
-    pub filterable_attributes: FilterableAttributesAnalytics,
-    pub distinct_attribute: DistinctAttributeAnalytics,
-    pub proximity_precision: ProximityPrecisionAnalytics,
-    pub typo_tolerance: TypoToleranceAnalytics,
-    pub faceting: FacetingAnalytics,
-    pub pagination: PaginationAnalytics,
-    pub stop_words: StopWordsAnalytics,
-    pub synonyms: SynonymsAnalytics,
-    pub embedders: EmbeddersAnalytics,
-    pub search_cutoff_ms: SearchCutoffMsAnalytics,
-    pub locales: LocalesAnalytics,
-    pub dictionary: DictionaryAnalytics,
-    pub separator_tokens: SeparatorTokensAnalytics,
-    pub non_separator_tokens: NonSeparatorTokensAnalytics,
-}
-
-impl Aggregate for SettingsAnalytics {
-    fn event_name(&self) -> &'static str {
-        "Settings Updated"
-    }
-
-    fn aggregate(self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        Box::new(Self {
-            ranking_rules: RankingRulesAnalytics {
-                words_position: new
-                    .ranking_rules
-                    .words_position
-                    .or(self.ranking_rules.words_position),
-                typo_position: new.ranking_rules.typo_position.or(self.ranking_rules.typo_position),
-                proximity_position: new
-                    .ranking_rules
-                    .proximity_position
-                    .or(self.ranking_rules.proximity_position),
-                attribute_position: new
-                    .ranking_rules
-                    .attribute_position
-                    .or(self.ranking_rules.attribute_position),
-                sort_position: new.ranking_rules.sort_position.or(self.ranking_rules.sort_position),
-                exactness_position: new
-                    .ranking_rules
-                    .exactness_position
-                    .or(self.ranking_rules.exactness_position),
-                values: new.ranking_rules.values.or(self.ranking_rules.values),
-            },
-            searchable_attributes: SearchableAttributesAnalytics {
-                total: new.searchable_attributes.total.or(self.searchable_attributes.total),
-                with_wildcard: new
-                    .searchable_attributes
-                    .with_wildcard
-                    .or(self.searchable_attributes.with_wildcard),
-            },
-            displayed_attributes: DisplayedAttributesAnalytics {
-                total: new.displayed_attributes.total.or(self.displayed_attributes.total),
-                with_wildcard: new
-                    .displayed_attributes
-                    .with_wildcard
-                    .or(self.displayed_attributes.with_wildcard),
-            },
-            sortable_attributes: SortableAttributesAnalytics {
-                total: new.sortable_attributes.total.or(self.sortable_attributes.total),
-                has_geo: new.sortable_attributes.has_geo.or(self.sortable_attributes.has_geo),
-            },
-            filterable_attributes: FilterableAttributesAnalytics {
-                total: new.filterable_attributes.total.or(self.filterable_attributes.total),
-                has_geo: new.filterable_attributes.has_geo.or(self.filterable_attributes.has_geo),
-            },
-            distinct_attribute: DistinctAttributeAnalytics {
-                set: self.distinct_attribute.set | new.distinct_attribute.set,
-            },
-            proximity_precision: ProximityPrecisionAnalytics {
-                set: self.proximity_precision.set | new.proximity_precision.set,
-                value: new.proximity_precision.value.or(self.proximity_precision.value),
-            },
-            typo_tolerance: TypoToleranceAnalytics {
-                enabled: new.typo_tolerance.enabled.or(self.typo_tolerance.enabled),
-                disable_on_attributes: new
-                    .typo_tolerance
-                    .disable_on_attributes
-                    .or(self.typo_tolerance.disable_on_attributes),
-                disable_on_words: new
-                    .typo_tolerance
-                    .disable_on_words
-                    .or(self.typo_tolerance.disable_on_words),
-                min_word_size_for_one_typo: new
-                    .typo_tolerance
-                    .min_word_size_for_one_typo
-                    .or(self.typo_tolerance.min_word_size_for_one_typo),
-                min_word_size_for_two_typos: new
-                    .typo_tolerance
-                    .min_word_size_for_two_typos
-                    .or(self.typo_tolerance.min_word_size_for_two_typos),
-            },
-            faceting: FacetingAnalytics {
-                max_values_per_facet: new
-                    .faceting
-                    .max_values_per_facet
-                    .or(self.faceting.max_values_per_facet),
-                sort_facet_values_by_star_count: new
-                    .faceting
-                    .sort_facet_values_by_star_count
-                    .or(self.faceting.sort_facet_values_by_star_count),
-                sort_facet_values_by_total: new
-                    .faceting
-                    .sort_facet_values_by_total
-                    .or(self.faceting.sort_facet_values_by_total),
-            },
-            pagination: PaginationAnalytics {
-                max_total_hits: new.pagination.max_total_hits.or(self.pagination.max_total_hits),
-            },
-            stop_words: StopWordsAnalytics {
-                total: new.stop_words.total.or(self.stop_words.total),
-            },
-            synonyms: SynonymsAnalytics { total: new.synonyms.total.or(self.synonyms.total) },
-            embedders: EmbeddersAnalytics {
-                total: new.embedders.total.or(self.embedders.total),
-                sources: match (self.embedders.sources, new.embedders.sources) {
-                    (None, None) => None,
-                    (Some(sources), None) | (None, Some(sources)) => Some(sources),
-                    (Some(this), Some(other)) => Some(this.union(&other).cloned().collect()),
-                },
-                document_template_used: match (
-                    self.embedders.document_template_used,
-                    new.embedders.document_template_used,
-                ) {
-                    (None, None) => None,
-                    (Some(used), None) | (None, Some(used)) => Some(used),
-                    (Some(this), Some(other)) => Some(this | other),
-                },
-                document_template_max_bytes: match (
-                    self.embedders.document_template_max_bytes,
-                    new.embedders.document_template_max_bytes,
-                ) {
-                    (None, None) => None,
-                    (Some(bytes), None) | (None, Some(bytes)) => Some(bytes),
-                    (Some(this), Some(other)) => Some(this.max(other)),
-                },
-                binary_quantization_used: match (
-                    self.embedders.binary_quantization_used,
-                    new.embedders.binary_quantization_used,
-                ) {
-                    (None, None) => None,
-                    (Some(bq), None) | (None, Some(bq)) => Some(bq),
-                    (Some(this), Some(other)) => Some(this | other),
-                },
-            },
-            search_cutoff_ms: SearchCutoffMsAnalytics {
-                search_cutoff_ms: new
-                    .search_cutoff_ms
-                    .search_cutoff_ms
-                    .or(self.search_cutoff_ms.search_cutoff_ms),
-            },
-            locales: LocalesAnalytics { locales: new.locales.locales.or(self.locales.locales) },
-            dictionary: DictionaryAnalytics {
-                total: new.dictionary.total.or(self.dictionary.total),
-            },
-            separator_tokens: SeparatorTokensAnalytics {
-                total: new.non_separator_tokens.total.or(self.separator_tokens.total),
-            },
-            non_separator_tokens: NonSeparatorTokensAnalytics {
-                total: new.non_separator_tokens.total.or(self.non_separator_tokens.total),
-            },
-        })
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        serde_json::to_value(*self).unwrap_or_default()
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct RankingRulesAnalytics {
-    pub words_position: Option<usize>,
-    pub typo_position: Option<usize>,
-    pub proximity_position: Option<usize>,
-    pub attribute_position: Option<usize>,
-    pub sort_position: Option<usize>,
-    pub exactness_position: Option<usize>,
-    pub values: Option<String>,
-}
-
-impl RankingRulesAnalytics {
-    pub fn new(rr: Option<&Vec<RankingRuleView>>) -> Self {
-        RankingRulesAnalytics {
-            words_position: rr.as_ref().and_then(|rr| {
-                rr.iter()
-                    .position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Words))
-            }),
-            typo_position: rr.as_ref().and_then(|rr| {
-                rr.iter()
-                    .position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Typo))
-            }),
-            proximity_position: rr.as_ref().and_then(|rr| {
-                rr.iter().position(|s| {
-                    matches!(s, meilisearch_types::settings::RankingRuleView::Proximity)
-                })
-            }),
-            attribute_position: rr.as_ref().and_then(|rr| {
-                rr.iter().position(|s| {
-                    matches!(s, meilisearch_types::settings::RankingRuleView::Attribute)
-                })
-            }),
-            sort_position: rr.as_ref().and_then(|rr| {
-                rr.iter()
-                    .position(|s| matches!(s, meilisearch_types::settings::RankingRuleView::Sort))
-            }),
-            exactness_position: rr.as_ref().and_then(|rr| {
-                rr.iter().position(|s| {
-                    matches!(s, meilisearch_types::settings::RankingRuleView::Exactness)
-                })
-            }),
-            values: rr.as_ref().map(|rr| {
-                rr.iter()
-                    .filter(|s| {
-                        matches!(
-                            s,
-                            meilisearch_types::settings::RankingRuleView::Asc(_)
-                                | meilisearch_types::settings::RankingRuleView::Desc(_)
-                        )
-                    })
-                    .map(|x| x.to_string())
-                    .collect::<Vec<_>>()
-                    .join(", ")
-            }),
-        }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { ranking_rules: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct SearchableAttributesAnalytics {
-    pub total: Option<usize>,
-    pub with_wildcard: Option<bool>,
-}
-
-impl SearchableAttributesAnalytics {
-    pub fn new(setting: Option<&Vec<String>>) -> Self {
-        Self {
-            total: setting.as_ref().map(|searchable| searchable.len()),
-            with_wildcard: setting
-                .as_ref()
-                .map(|searchable| searchable.iter().any(|searchable| searchable == "*")),
-        }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { searchable_attributes: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct DisplayedAttributesAnalytics {
-    pub total: Option<usize>,
-    pub with_wildcard: Option<bool>,
-}
-
-impl DisplayedAttributesAnalytics {
-    pub fn new(displayed: Option<&Vec<String>>) -> Self {
-        Self {
-            total: displayed.as_ref().map(|displayed| displayed.len()),
-            with_wildcard: displayed
-                .as_ref()
-                .map(|displayed| displayed.iter().any(|displayed| displayed == "*")),
-        }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { displayed_attributes: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct SortableAttributesAnalytics {
-    pub total: Option<usize>,
-    pub has_geo: Option<bool>,
-}
-
-impl SortableAttributesAnalytics {
-    pub fn new(setting: Option<&BTreeSet<String>>) -> Self {
-        Self {
-            total: setting.as_ref().map(|sort| sort.len()),
-            has_geo: setting.as_ref().map(|sort| sort.contains("_geo")),
-        }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { sortable_attributes: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct FilterableAttributesAnalytics {
-    pub total: Option<usize>,
-    pub has_geo: Option<bool>,
-}
-
-impl FilterableAttributesAnalytics {
-    pub fn new(setting: Option<&BTreeSet<String>>) -> Self {
-        Self {
-            total: setting.as_ref().map(|filter| filter.len()),
-            has_geo: setting.as_ref().map(|filter| filter.contains("_geo")),
-        }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { filterable_attributes: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct DistinctAttributeAnalytics {
-    pub set: bool,
-}
-
-impl DistinctAttributeAnalytics {
-    pub fn new(distinct: Option<&String>) -> Self {
-        Self { set: distinct.is_some() }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { distinct_attribute: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct ProximityPrecisionAnalytics {
-    pub set: bool,
-    pub value: Option<ProximityPrecisionView>,
-}
-
-impl ProximityPrecisionAnalytics {
-    pub fn new(precision: Option<&ProximityPrecisionView>) -> Self {
-        Self { set: precision.is_some(), value: precision.cloned() }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { proximity_precision: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct TypoToleranceAnalytics {
-    pub enabled: Option<bool>,
-    pub disable_on_attributes: Option<bool>,
-    pub disable_on_words: Option<bool>,
-    pub min_word_size_for_one_typo: Option<u8>,
-    pub min_word_size_for_two_typos: Option<u8>,
-}
-
-impl TypoToleranceAnalytics {
-    pub fn new(setting: Option<&TypoSettings>) -> Self {
-        Self {
-            enabled: setting.as_ref().map(|s| !matches!(s.enabled, Setting::Set(false))),
-            disable_on_attributes: setting
-                .as_ref()
-                .and_then(|s| s.disable_on_attributes.as_ref().set().map(|m| !m.is_empty())),
-            disable_on_words: setting
-                .as_ref()
-                .and_then(|s| s.disable_on_words.as_ref().set().map(|m| !m.is_empty())),
-            min_word_size_for_one_typo: setting
-                .as_ref()
-                .and_then(|s| s.min_word_size_for_typos.as_ref().set().map(|s| s.one_typo.set()))
-                .flatten(),
-            min_word_size_for_two_typos: setting
-                .as_ref()
-                .and_then(|s| s.min_word_size_for_typos.as_ref().set().map(|s| s.two_typos.set()))
-                .flatten(),
-        }
-    }
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { typo_tolerance: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct FacetingAnalytics {
-    pub max_values_per_facet: Option<usize>,
-    pub sort_facet_values_by_star_count: Option<bool>,
-    pub sort_facet_values_by_total: Option<usize>,
-}
-
-impl FacetingAnalytics {
-    pub fn new(setting: Option<&FacetingSettings>) -> Self {
-        Self {
-            max_values_per_facet: setting.as_ref().and_then(|s| s.max_values_per_facet.set()),
-            sort_facet_values_by_star_count: setting.as_ref().and_then(|s| {
-                s.sort_facet_values_by
-                    .as_ref()
-                    .set()
-                    .map(|s| s.iter().any(|(k, v)| k == "*" && v == &FacetValuesSort::Count))
-            }),
-            sort_facet_values_by_total: setting
-                .as_ref()
-                .and_then(|s| s.sort_facet_values_by.as_ref().set().map(|s| s.len())),
-        }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { faceting: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct PaginationAnalytics {
-    pub max_total_hits: Option<usize>,
-}
-
-impl PaginationAnalytics {
-    pub fn new(setting: Option<&PaginationSettings>) -> Self {
-        Self { max_total_hits: setting.as_ref().and_then(|s| s.max_total_hits.set()) }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { pagination: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct StopWordsAnalytics {
-    pub total: Option<usize>,
-}
-
-impl StopWordsAnalytics {
-    pub fn new(stop_words: Option<&BTreeSet<String>>) -> Self {
-        Self { total: stop_words.as_ref().map(|stop_words| stop_words.len()) }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { stop_words: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct SynonymsAnalytics {
-    pub total: Option<usize>,
-}
-
-impl SynonymsAnalytics {
-    pub fn new(synonyms: Option<&BTreeMap<String, Vec<String>>>) -> Self {
-        Self { total: synonyms.as_ref().map(|synonyms| synonyms.len()) }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { synonyms: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct EmbeddersAnalytics {
-    // last
-    pub total: Option<usize>,
-    // Merge the sources
-    pub sources: Option<HashSet<String>>,
-    // |=
-    pub document_template_used: Option<bool>,
-    // max
-    pub document_template_max_bytes: Option<usize>,
-    // |=
-    pub binary_quantization_used: Option<bool>,
-}
-
-impl EmbeddersAnalytics {
-    pub fn new(setting: Option<&BTreeMap<String, Setting<EmbeddingSettings>>>) -> Self {
-        let mut sources = std::collections::HashSet::new();
-
-        if let Some(s) = &setting {
-            for source in s
-                .values()
-                .filter_map(|config| config.clone().set())
-                .filter_map(|config| config.source.set())
-            {
-                use meilisearch_types::milli::vector::settings::EmbedderSource;
-                match source {
-                    EmbedderSource::OpenAi => sources.insert("openAi".to_string()),
-                    EmbedderSource::HuggingFace => sources.insert("huggingFace".to_string()),
-                    EmbedderSource::UserProvided => sources.insert("userProvided".to_string()),
-                    EmbedderSource::Ollama => sources.insert("ollama".to_string()),
-                    EmbedderSource::Rest => sources.insert("rest".to_string()),
-                };
-            }
-        };
-
-        Self {
-            total: setting.as_ref().map(|s| s.len()),
-            sources: Some(sources),
-            document_template_used: setting.as_ref().map(|map| {
-                map.values()
-                    .filter_map(|config| config.clone().set())
-                    .any(|config| config.document_template.set().is_some())
-            }),
-            document_template_max_bytes: setting.as_ref().and_then(|map| {
-                map.values()
-                    .filter_map(|config| config.clone().set())
-                    .filter_map(|config| config.document_template_max_bytes.set())
-                    .max()
-            }),
-            binary_quantization_used: setting.as_ref().map(|map| {
-                map.values()
-                    .filter_map(|config| config.clone().set())
-                    .any(|config| config.binary_quantized.set().is_some())
-            }),
-        }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { embedders: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-#[serde(transparent)]
-pub struct SearchCutoffMsAnalytics {
-    pub search_cutoff_ms: Option<u64>,
-}
-
-impl SearchCutoffMsAnalytics {
-    pub fn new(setting: Option<&u64>) -> Self {
-        Self { search_cutoff_ms: setting.copied() }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { search_cutoff_ms: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-#[serde(transparent)]
-pub struct LocalesAnalytics {
-    pub locales: Option<BTreeSet<Locale>>,
-}
-
-impl LocalesAnalytics {
-    pub fn new(rules: Option<&Vec<LocalizedAttributesRuleView>>) -> Self {
-        LocalesAnalytics {
-            locales: rules.as_ref().map(|rules| {
-                rules
-                    .iter()
-                    .flat_map(|rule| rule.locales.iter().cloned())
-                    .collect::<std::collections::BTreeSet<_>>()
-            }),
-        }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { locales: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct DictionaryAnalytics {
-    pub total: Option<usize>,
-}
-
-impl DictionaryAnalytics {
-    pub fn new(dictionary: Option<&BTreeSet<String>>) -> Self {
-        Self { total: dictionary.as_ref().map(|dictionary| dictionary.len()) }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { dictionary: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct SeparatorTokensAnalytics {
-    pub total: Option<usize>,
-}
-
-impl SeparatorTokensAnalytics {
-    pub fn new(separator_tokens: Option<&BTreeSet<String>>) -> Self {
-        Self { total: separator_tokens.as_ref().map(|separator_tokens| separator_tokens.len()) }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { separator_tokens: self, ..Default::default() }
-    }
-}
-
-#[derive(Serialize, Default)]
-pub struct NonSeparatorTokensAnalytics {
-    pub total: Option<usize>,
-}
-
-impl NonSeparatorTokensAnalytics {
-    pub fn new(non_separator_tokens: Option<&BTreeSet<String>>) -> Self {
-        Self {
-            total: non_separator_tokens
-                .as_ref()
-                .map(|non_separator_tokens| non_separator_tokens.len()),
-        }
-    }
-
-    pub fn into_settings(self) -> SettingsAnalytics {
-        SettingsAnalytics { non_separator_tokens: self, ..Default::default() }
-    }
-}

meilisearch/src/routes/indexes/similar.rs

@@ -13,10 +13,9 @@ use serde_json::Value;
 use tracing::debug;
 
 use super::ActionPolicy;
-use crate::analytics::Analytics;
+use crate::analytics::{Analytics, SimilarAggregator};
 use crate::extractors::authentication::GuardedData;
 use crate::extractors::sequential_extractor::SeqHandler;
-use crate::routes::indexes::similar_analytics::{SimilarAggregator, SimilarGET, SimilarPOST};
 use crate::search::{
     add_search_rules, perform_similar, RankingScoreThresholdSimilar, RetrieveVectors, SearchKind,
     SimilarQuery, SimilarResult, DEFAULT_SEARCH_LIMIT, DEFAULT_SEARCH_OFFSET,
@@ -35,13 +34,13 @@ pub async fn similar_get(
     index_uid: web::Path<String>,
     params: AwebQueryParameter<SimilarQueryGet, DeserrQueryParamError>,
     req: HttpRequest,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
     let query = params.0.try_into()?;
 
-    let mut aggregate = SimilarAggregator::<SimilarGET>::from_query(&query);
+    let mut aggregate = SimilarAggregator::from_query(&query, &req);
 
     debug!(parameters = ?query, "Similar get");
 
@@ -50,7 +49,7 @@ pub async fn similar_get(
     if let Ok(similar) = &similar {
         aggregate.succeed(similar);
     }
-    analytics.publish(aggregate, &req);
+    analytics.get_similar(aggregate);
 
     let similar = similar?;
 
@@ -63,21 +62,21 @@ pub async fn similar_post(
     index_uid: web::Path<String>,
     params: AwebJson<SimilarQuery, DeserrJsonError>,
     req: HttpRequest,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
     let query = params.into_inner();
     debug!(parameters = ?query, "Similar post");
 
-    let mut aggregate = SimilarAggregator::<SimilarPOST>::from_query(&query);
+    let mut aggregate = SimilarAggregator::from_query(&query, &req);
 
     let similar = similar(index_scheduler, index_uid, query).await;
 
     if let Ok(similar) = &similar {
         aggregate.succeed(similar);
     }
-    analytics.publish(aggregate, &req);
+    analytics.post_similar(aggregate);
 
     let similar = similar?;
 
@@ -103,19 +102,11 @@ async fn similar(
 
     let index = index_scheduler.index(&index_uid)?;
 
-    let (embedder_name, embedder, quantized) =
-        SearchKind::embedder(&index_scheduler, &index, &query.embedder, None)?;
+    let (embedder_name, embedder) =
+        SearchKind::embedder(&index_scheduler, &index, query.embedder.as_deref(), None)?;
 
     tokio::task::spawn_blocking(move || {
-        perform_similar(
-            &index,
-            query,
-            embedder_name,
-            embedder,
-            quantized,
-            retrieve_vectors,
-            index_scheduler.features(),
-        )
+        perform_similar(&index, query, embedder_name, embedder, retrieve_vectors)
     })
     .await?
 }
@@ -141,8 +132,8 @@ pub struct SimilarQueryGet {
     show_ranking_score_details: Param<bool>,
     #[deserr(default, error = DeserrQueryParamError<InvalidSimilarRankingScoreThreshold>, default)]
     pub ranking_score_threshold: Option<RankingScoreThresholdGet>,
-    #[deserr(error = DeserrQueryParamError<InvalidEmbedder>)]
-    pub embedder: String,
+    #[deserr(default, error = DeserrQueryParamError<InvalidEmbedder>)]
+    pub embedder: Option<String>,
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, deserr::Deserr)]

meilisearch/src/routes/indexes/similar_analytics.rs

@@ -1,235 +0,0 @@
-use std::collections::{BinaryHeap, HashMap};
-
-use once_cell::sync::Lazy;
-use regex::Regex;
-use serde_json::{json, Value};
-
-use crate::{
-    aggregate_methods,
-    analytics::{Aggregate, AggregateMethod},
-    search::{SimilarQuery, SimilarResult},
-};
-
-aggregate_methods!(
-    SimilarPOST => "Similar POST",
-    SimilarGET => "Similar GET",
-);
-
-#[derive(Default)]
-pub struct SimilarAggregator<Method: AggregateMethod> {
-    // requests
-    total_received: usize,
-    total_succeeded: usize,
-    time_spent: BinaryHeap<usize>,
-
-    // filter
-    filter_with_geo_radius: bool,
-    filter_with_geo_bounding_box: bool,
-    // every time a request has a filter, this field must be incremented by the number of terms it contains
-    filter_sum_of_criteria_terms: usize,
-    // every time a request has a filter, this field must be incremented by one
-    filter_total_number_of_criteria: usize,
-    used_syntax: HashMap<String, usize>,
-
-    // Whether a non-default embedder was specified
-    retrieve_vectors: bool,
-
-    // pagination
-    max_limit: usize,
-    max_offset: usize,
-
-    // formatting
-    max_attributes_to_retrieve: usize,
-
-    // scoring
-    show_ranking_score: bool,
-    show_ranking_score_details: bool,
-    ranking_score_threshold: bool,
-
-    marker: std::marker::PhantomData<Method>,
-}
-
-impl<Method: AggregateMethod> SimilarAggregator<Method> {
-    #[allow(clippy::field_reassign_with_default)]
-    pub fn from_query(query: &SimilarQuery) -> Self {
-        let SimilarQuery {
-            id: _,
-            embedder: _,
-            offset,
-            limit,
-            attributes_to_retrieve: _,
-            retrieve_vectors,
-            show_ranking_score,
-            show_ranking_score_details,
-            filter,
-            ranking_score_threshold,
-        } = query;
-
-        let mut ret = Self::default();
-
-        ret.total_received = 1;
-
-        if let Some(ref filter) = filter {
-            static RE: Lazy<Regex> = Lazy::new(|| Regex::new("AND | OR").unwrap());
-            ret.filter_total_number_of_criteria = 1;
-
-            let syntax = match filter {
-                Value::String(_) => "string".to_string(),
-                Value::Array(values) => {
-                    if values.iter().map(|v| v.to_string()).any(|s| RE.is_match(&s)) {
-                        "mixed".to_string()
-                    } else {
-                        "array".to_string()
-                    }
-                }
-                _ => "none".to_string(),
-            };
-            // convert the string to a HashMap
-            ret.used_syntax.insert(syntax, 1);
-
-            let stringified_filters = filter.to_string();
-            ret.filter_with_geo_radius = stringified_filters.contains("_geoRadius(");
-            ret.filter_with_geo_bounding_box = stringified_filters.contains("_geoBoundingBox(");
-            ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count();
-        }
-
-        ret.max_limit = *limit;
-        ret.max_offset = *offset;
-
-        ret.show_ranking_score = *show_ranking_score;
-        ret.show_ranking_score_details = *show_ranking_score_details;
-        ret.ranking_score_threshold = ranking_score_threshold.is_some();
-
-        ret.retrieve_vectors = *retrieve_vectors;
-
-        ret
-    }
-
-    pub fn succeed(&mut self, result: &SimilarResult) {
-        let SimilarResult { id: _, hits: _, processing_time_ms, hits_info: _ } = result;
-
-        self.total_succeeded = self.total_succeeded.saturating_add(1);
-
-        self.time_spent.push(*processing_time_ms as usize);
-    }
-}
-
-impl<Method: AggregateMethod> Aggregate for SimilarAggregator<Method> {
-    fn event_name(&self) -> &'static str {
-        Method::event_name()
-    }
-
-    /// Aggregate one [SimilarAggregator] into another.
-    fn aggregate(mut self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        let Self {
-            total_received,
-            total_succeeded,
-            mut time_spent,
-            filter_with_geo_radius,
-            filter_with_geo_bounding_box,
-            filter_sum_of_criteria_terms,
-            filter_total_number_of_criteria,
-            used_syntax,
-            max_limit,
-            max_offset,
-            max_attributes_to_retrieve,
-            show_ranking_score,
-            show_ranking_score_details,
-            ranking_score_threshold,
-            retrieve_vectors,
-            marker: _,
-        } = *new;
-
-        // request
-        self.total_received = self.total_received.saturating_add(total_received);
-        self.total_succeeded = self.total_succeeded.saturating_add(total_succeeded);
-        self.time_spent.append(&mut time_spent);
-
-        // filter
-        self.filter_with_geo_radius |= filter_with_geo_radius;
-        self.filter_with_geo_bounding_box |= filter_with_geo_bounding_box;
-        self.filter_sum_of_criteria_terms =
-            self.filter_sum_of_criteria_terms.saturating_add(filter_sum_of_criteria_terms);
-        self.filter_total_number_of_criteria =
-            self.filter_total_number_of_criteria.saturating_add(filter_total_number_of_criteria);
-        for (key, value) in used_syntax.into_iter() {
-            let used_syntax = self.used_syntax.entry(key).or_insert(0);
-            *used_syntax = used_syntax.saturating_add(value);
-        }
-
-        self.retrieve_vectors |= retrieve_vectors;
-
-        // pagination
-        self.max_limit = self.max_limit.max(max_limit);
-        self.max_offset = self.max_offset.max(max_offset);
-
-        // formatting
-        self.max_attributes_to_retrieve =
-            self.max_attributes_to_retrieve.max(max_attributes_to_retrieve);
-
-        // scoring
-        self.show_ranking_score |= show_ranking_score;
-        self.show_ranking_score_details |= show_ranking_score_details;
-        self.ranking_score_threshold |= ranking_score_threshold;
-
-        self
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        let Self {
-            total_received,
-            total_succeeded,
-            time_spent,
-            filter_with_geo_radius,
-            filter_with_geo_bounding_box,
-            filter_sum_of_criteria_terms,
-            filter_total_number_of_criteria,
-            used_syntax,
-            max_limit,
-            max_offset,
-            max_attributes_to_retrieve,
-            show_ranking_score,
-            show_ranking_score_details,
-            ranking_score_threshold,
-            retrieve_vectors,
-            marker: _,
-        } = *self;
-
-        // we get all the values in a sorted manner
-        let time_spent = time_spent.into_sorted_vec();
-        // the index of the 99th percentage of value
-        let percentile_99th = time_spent.len() * 99 / 100;
-        // We are only interested by the slowest value of the 99th fastest results
-        let time_spent = time_spent.get(percentile_99th);
-
-        json!({
-            "requests": {
-                "99th_response_time": time_spent.map(|t| format!("{:.2}", t)),
-                "total_succeeded": total_succeeded,
-                "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics
-                "total_received": total_received,
-            },
-            "filter": {
-               "with_geoRadius": filter_with_geo_radius,
-               "with_geoBoundingBox": filter_with_geo_bounding_box,
-               "avg_criteria_number": format!("{:.2}", filter_sum_of_criteria_terms as f64 / filter_total_number_of_criteria as f64),
-               "most_used_syntax": used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)),
-            },
-            "vector": {
-                "retrieve_vectors": retrieve_vectors,
-            },
-            "pagination": {
-               "max_limit": max_limit,
-               "max_offset": max_offset,
-            },
-            "formatting": {
-                "max_attributes_to_retrieve": max_attributes_to_retrieve,
-            },
-            "scoring": {
-                "show_ranking_score": show_ranking_score,
-                "show_ranking_score_details": show_ranking_score_details,
-                "ranking_score_threshold": ranking_score_threshold,
-            }
-        })
-    }
-}

meilisearch/src/routes/mod.rs

@@ -25,7 +25,6 @@ pub mod indexes;
 mod logs;
 mod metrics;
 mod multi_search;
-mod multi_search_analytics;
 mod snapshot;
 mod swap_indexes;
 pub mod tasks;

meilisearch/src/routes/multi_search.rs

@@ -9,20 +9,16 @@ use meilisearch_types::keys::actions;
 use serde::Serialize;
 use tracing::debug;
 
-use crate::analytics::Analytics;
-use crate::error::MeilisearchHttpError;
+use crate::analytics::{Analytics, MultiSearchAggregator};
 use crate::extractors::authentication::policies::ActionPolicy;
 use crate::extractors::authentication::{AuthenticationError, GuardedData};
 use crate::extractors::sequential_extractor::SeqHandler;
 use crate::routes::indexes::search::search_kind;
 use crate::search::{
-    add_search_rules, perform_federated_search, perform_search, FederatedSearch, RetrieveVectors,
-    SearchQueryWithIndex, SearchResultWithIndex,
+    add_search_rules, perform_search, RetrieveVectors, SearchQueryWithIndex, SearchResultWithIndex,
 };
 use crate::search_queue::SearchQueue;
 
-use super::multi_search_analytics::MultiSearchAggregator;
-
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(web::resource("").route(web::post().to(SeqHandler(multi_search_with_post))));
 }
@@ -32,87 +28,46 @@ struct SearchResults {
     results: Vec<SearchResultWithIndex>,
 }
 
+#[derive(Debug, deserr::Deserr)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
+pub struct SearchQueries {
+    queries: Vec<SearchQueryWithIndex>,
+}
+
 pub async fn multi_search_with_post(
     index_scheduler: GuardedData<ActionPolicy<{ actions::SEARCH }>, Data<IndexScheduler>>,
     search_queue: Data<SearchQueue>,
-    params: AwebJson<FederatedSearch, DeserrJsonError>,
+    params: AwebJson<SearchQueries, DeserrJsonError>,
     req: HttpRequest,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    // Since we don't want to process half of the search requests and then get a permit refused
-    // we're going to get one permit for the whole duration of the multi-search request.
-    let permit = search_queue.try_get_search_permit().await?;
-
-    let federated_search = params.into_inner();
-
-    let mut multi_aggregate = MultiSearchAggregator::from_federated_search(&federated_search);
-
-    let FederatedSearch { mut queries, federation } = federated_search;
+    let queries = params.into_inner().queries;
 
+    let mut multi_aggregate = MultiSearchAggregator::from_queries(&queries, &req);
     let features = index_scheduler.features();
 
-    // regardless of federation, check authorization and apply search rules
-    let auth = 'check_authorization: {
-        for (query_index, federated_query) in queries.iter_mut().enumerate() {
-            let index_uid = federated_query.index_uid.as_str();
-            // Check index from API key
-            if !index_scheduler.filters().is_index_authorized(index_uid) {
-                break 'check_authorization Err(AuthenticationError::InvalidToken)
-                    .with_index(query_index);
-            }
-            // Apply search rules from tenant token
-            if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(index_uid)
-            {
-                add_search_rules(&mut federated_query.filter, search_rules);
-            }
-        }
-        Ok(())
-    };
+    // Since we don't want to process half of the search requests and then get a permit refused
+    // we're going to get one permit for the whole duration of the multi-search request.
+    let _permit = search_queue.try_get_search_permit().await?;
 
-    auth.map_err(|(mut err, query_index)| {
-        // Add the query index that failed as context for the error message.
-        // We're doing it only here and not directly in the `WithIndex` trait so that the `with_index` function returns a different type
-        // of result and we can benefit from static typing.
-        err.message = format!("Inside `.queries[{query_index}]`: {}", err.message);
-        err
-    })?;
-
-    let response = match federation {
-        Some(federation) => {
-            let search_result = tokio::task::spawn_blocking(move || {
-                perform_federated_search(&index_scheduler, queries, federation, features)
-            })
-            .await;
-            permit.drop().await;
-
-            if let Ok(Ok(_)) = search_result {
-                multi_aggregate.succeed();
-            }
-
-            analytics.publish(multi_aggregate, &req);
-            HttpResponse::Ok().json(search_result??)
-        }
-        None => {
     // Explicitly expect a `(ResponseError, usize)` for the error type rather than `ResponseError` only,
     // so that `?` doesn't work if it doesn't use `with_index`, ensuring that it is not forgotten in case of code
     // changes.
     let search_results: Result<_, (ResponseError, usize)> = async {
         let mut search_results = Vec::with_capacity(queries.len());
-                for (query_index, (index_uid, query, federation_options)) in queries
-                    .into_iter()
-                    .map(SearchQueryWithIndex::into_index_query_federation)
-                    .enumerate()
+        for (query_index, (index_uid, mut query)) in
+            queries.into_iter().map(SearchQueryWithIndex::into_index_query).enumerate()
         {
             debug!(on_index = query_index, parameters = ?query, "Multi-search");
 
-                    if federation_options.is_some() {
-                        return Err((
-                            MeilisearchHttpError::FederationOptionsInNonFederatedRequest(
-                                query_index,
-                            )
-                            .into(),
-                            query_index,
-                        ));
+            // Check index from API key
+            if !index_scheduler.filters().is_index_authorized(&index_uid) {
+                return Err(AuthenticationError::InvalidToken).with_index(query_index);
+            }
+            // Apply search rules from tenant token
+            if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(&index_uid)
+            {
+                add_search_rules(&mut query.filter, search_rules);
             }
 
             let index = index_scheduler
@@ -126,14 +81,13 @@ pub async fn multi_search_with_post(
                 })
                 .with_index(query_index)?;
 
-                    let search_kind =
-                        search_kind(&query, index_scheduler.get_ref(), &index, features)
-                            .with_index(query_index)?;
-                    let retrieve_vector = RetrieveVectors::new(query.retrieve_vectors, features)
+            let search_kind = search_kind(&query, index_scheduler.get_ref(), &index, features)
                 .with_index(query_index)?;
+            let retrieve_vector =
+                RetrieveVectors::new(query.retrieve_vectors, features).with_index(query_index)?;
 
             let search_result = tokio::task::spawn_blocking(move || {
-                        perform_search(&index, query, search_kind, retrieve_vector, features)
+                perform_search(&index, query, search_kind, retrieve_vector)
             })
             .await
             .with_index(query_index)?;
@@ -146,12 +100,11 @@ pub async fn multi_search_with_post(
         Ok(search_results)
     }
     .await;
-            permit.drop().await;
 
     if search_results.is_ok() {
         multi_aggregate.succeed();
     }
-            analytics.publish(multi_aggregate, &req);
+    analytics.post_multi_search(multi_aggregate);
 
     let search_results = search_results.map_err(|(mut err, query_index)| {
         // Add the query index that failed as context for the error message.
@@ -163,11 +116,7 @@ pub async fn multi_search_with_post(
 
     debug!(returns = ?search_results, "Multi-search");
 
-            HttpResponse::Ok().json(SearchResults { results: search_results })
-        }
-    };
-
-    Ok(response)
+    Ok(HttpResponse::Ok().json(SearchResults { results: search_results }))
 }
 
 /// Local `Result` extension trait to avoid `map_err` boilerplate.

meilisearch/src/routes/multi_search_analytics.rs

@@ -1,170 +0,0 @@
-use std::collections::HashSet;
-
-use serde_json::json;
-
-use crate::{
-    analytics::Aggregate,
-    search::{FederatedSearch, SearchQueryWithIndex},
-};
-
-#[derive(Default)]
-pub struct MultiSearchAggregator {
-    // requests
-    total_received: usize,
-    total_succeeded: usize,
-
-    // sum of the number of distinct indexes in each single request, use with total_received to compute an avg
-    total_distinct_index_count: usize,
-    // number of queries with a single index, use with total_received to compute a proportion
-    total_single_index: usize,
-
-    // sum of the number of search queries in the requests, use with total_received to compute an average
-    total_search_count: usize,
-
-    // scoring
-    show_ranking_score: bool,
-    show_ranking_score_details: bool,
-
-    // federation
-    use_federation: bool,
-}
-
-impl MultiSearchAggregator {
-    pub fn from_federated_search(federated_search: &FederatedSearch) -> Self {
-        let use_federation = federated_search.federation.is_some();
-
-        let distinct_indexes: HashSet<_> = federated_search
-            .queries
-            .iter()
-            .map(|query| {
-                let query = &query;
-                // make sure we get a compilation error if a field gets added to / removed from SearchQueryWithIndex
-                let SearchQueryWithIndex {
-                    index_uid,
-                    federation_options: _,
-                    q: _,
-                    vector: _,
-                    offset: _,
-                    limit: _,
-                    page: _,
-                    hits_per_page: _,
-                    attributes_to_retrieve: _,
-                    retrieve_vectors: _,
-                    attributes_to_crop: _,
-                    crop_length: _,
-                    attributes_to_highlight: _,
-                    show_ranking_score: _,
-                    show_ranking_score_details: _,
-                    show_matches_position: _,
-                    filter: _,
-                    sort: _,
-                    distinct: _,
-                    facets: _,
-                    highlight_pre_tag: _,
-                    highlight_post_tag: _,
-                    crop_marker: _,
-                    matching_strategy: _,
-                    attributes_to_search_on: _,
-                    hybrid: _,
-                    ranking_score_threshold: _,
-                    locales: _,
-                } = query;
-
-                index_uid.as_str()
-            })
-            .collect();
-
-        let show_ranking_score =
-            federated_search.queries.iter().any(|query| query.show_ranking_score);
-        let show_ranking_score_details =
-            federated_search.queries.iter().any(|query| query.show_ranking_score_details);
-
-        Self {
-            total_received: 1,
-            total_succeeded: 0,
-            total_distinct_index_count: distinct_indexes.len(),
-            total_single_index: if distinct_indexes.len() == 1 { 1 } else { 0 },
-            total_search_count: federated_search.queries.len(),
-            show_ranking_score,
-            show_ranking_score_details,
-            use_federation,
-        }
-    }
-
-    pub fn succeed(&mut self) {
-        self.total_succeeded = self.total_succeeded.saturating_add(1);
-    }
-}
-
-impl Aggregate for MultiSearchAggregator {
-    fn event_name(&self) -> &'static str {
-        "Documents Searched by Multi-Search POST"
-    }
-
-    /// Aggregate one [MultiSearchAggregator] into another.
-    fn aggregate(self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        // write the aggregate in a way that will cause a compilation error if a field is added.
-
-        // get ownership of self, replacing it by a default value.
-        let this = *self;
-
-        let total_received = this.total_received.saturating_add(new.total_received);
-        let total_succeeded = this.total_succeeded.saturating_add(new.total_succeeded);
-        let total_distinct_index_count =
-            this.total_distinct_index_count.saturating_add(new.total_distinct_index_count);
-        let total_single_index = this.total_single_index.saturating_add(new.total_single_index);
-        let total_search_count = this.total_search_count.saturating_add(new.total_search_count);
-        let show_ranking_score = this.show_ranking_score || new.show_ranking_score;
-        let show_ranking_score_details =
-            this.show_ranking_score_details || new.show_ranking_score_details;
-        let use_federation = this.use_federation || new.use_federation;
-
-        Box::new(Self {
-            total_received,
-            total_succeeded,
-            total_distinct_index_count,
-            total_single_index,
-            total_search_count,
-            show_ranking_score,
-            show_ranking_score_details,
-            use_federation,
-        })
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        let Self {
-            total_received,
-            total_succeeded,
-            total_distinct_index_count,
-            total_single_index,
-            total_search_count,
-            show_ranking_score,
-            show_ranking_score_details,
-            use_federation,
-        } = *self;
-
-        json!({
-            "requests": {
-                "total_succeeded": total_succeeded,
-                "total_failed": total_received.saturating_sub(total_succeeded), // just to be sure we never panics
-                "total_received": total_received,
-            },
-            "indexes": {
-                "total_single_index": total_single_index,
-                "total_distinct_index_count": total_distinct_index_count,
-                "avg_distinct_index_count": (total_distinct_index_count as f64) / (total_received as f64), // not 0 else returned early
-            },
-            "searches": {
-                "total_search_count": total_search_count,
-                "avg_search_count": (total_search_count as f64) / (total_received as f64),
-            },
-            "scoring": {
-                "show_ranking_score": show_ranking_score,
-                "show_ranking_score_details": show_ranking_score_details,
-            },
-            "federation": {
-                "use_federation": use_federation,
-            }
-        })
-    }
-}

meilisearch/src/routes/snapshot.rs

@@ -3,6 +3,7 @@ use actix_web::{web, HttpRequest, HttpResponse};
 use index_scheduler::IndexScheduler;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::tasks::KindWithContent;
+use serde_json::json;
 use tracing::debug;
 
 use crate::analytics::Analytics;
@@ -16,15 +17,13 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(web::resource("").route(web::post().to(SeqHandler(create_snapshot))));
 }
 
-crate::empty_analytics!(SnapshotAnalytics, "Snapshot Created");
-
 pub async fn create_snapshot(
     index_scheduler: GuardedData<ActionPolicy<{ actions::SNAPSHOTS_CREATE }>, Data<IndexScheduler>>,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    analytics.publish(SnapshotAnalytics::default(), &req);
+    analytics.publish("Snapshot Created".to_string(), json!({}), Some(&req));
 
     let task = KindWithContent::SnapshotCreation;
     let uid = get_task_id(&req, &opt)?;

meilisearch/src/routes/swap_indexes.rs

@@ -8,10 +8,10 @@ use meilisearch_types::error::deserr_codes::InvalidSwapIndexes;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::index_uid::IndexUid;
 use meilisearch_types::tasks::{IndexSwap, KindWithContent};
-use serde::Serialize;
+use serde_json::json;
 
 use super::{get_task_id, is_dry_run, SummarizedTaskView};
-use crate::analytics::{Aggregate, Analytics};
+use crate::analytics::Analytics;
 use crate::error::MeilisearchHttpError;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::{AuthenticationError, GuardedData};
@@ -29,36 +29,21 @@ pub struct SwapIndexesPayload {
     indexes: Vec<IndexUid>,
 }
 
-#[derive(Serialize)]
-struct IndexSwappedAnalytics {
-    swap_operation_number: usize,
-}
-
-impl Aggregate for IndexSwappedAnalytics {
-    fn event_name(&self) -> &'static str {
-        "Indexes Swapped"
-    }
-
-    fn aggregate(self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        Box::new(Self {
-            swap_operation_number: self.swap_operation_number.max(new.swap_operation_number),
-        })
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        serde_json::to_value(*self).unwrap_or_default()
-    }
-}
-
 pub async fn swap_indexes(
     index_scheduler: GuardedData<ActionPolicy<{ actions::INDEXES_SWAP }>, Data<IndexScheduler>>,
     params: AwebJson<Vec<SwapIndexesPayload>, DeserrJsonError>,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let params = params.into_inner();
-    analytics.publish(IndexSwappedAnalytics { swap_operation_number: params.len() }, &req);
+    analytics.publish(
+        "Indexes Swapped".to_string(),
+        json!({
+            "swap_operation_number": params.len(),
+        }),
+        Some(&req),
+    );
     let filters = index_scheduler.filters();
 
     let mut swaps = vec![];

meilisearch/src/routes/tasks.rs

@@ -12,17 +12,18 @@ use meilisearch_types::star_or::{OptionStarOr, OptionStarOrList};
 use meilisearch_types::task_view::TaskView;
 use meilisearch_types::tasks::{Kind, KindWithContent, Status};
 use serde::Serialize;
+use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::macros::format_description;
 use time::{Date, Duration, OffsetDateTime, Time};
 use tokio::task;
 
 use super::{get_task_id, is_dry_run, SummarizedTaskView};
-use crate::analytics::{Aggregate, AggregateMethod, Analytics};
+use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::extractors::sequential_extractor::SeqHandler;
-use crate::{aggregate_methods, Opt};
+use crate::Opt;
 
 const DEFAULT_LIMIT: u32 = 20;
 
@@ -157,69 +158,12 @@ impl TaskDeletionOrCancelationQuery {
     }
 }
 
-aggregate_methods!(
-    CancelTasks => "Tasks Canceled",
-    DeleteTasks => "Tasks Deleted",
-);
-
-#[derive(Serialize)]
-struct TaskFilterAnalytics<Method: AggregateMethod> {
-    filtered_by_uid: bool,
-    filtered_by_index_uid: bool,
-    filtered_by_type: bool,
-    filtered_by_status: bool,
-    filtered_by_canceled_by: bool,
-    filtered_by_before_enqueued_at: bool,
-    filtered_by_after_enqueued_at: bool,
-    filtered_by_before_started_at: bool,
-    filtered_by_after_started_at: bool,
-    filtered_by_before_finished_at: bool,
-    filtered_by_after_finished_at: bool,
-
-    #[serde(skip)]
-    marker: std::marker::PhantomData<Method>,
-}
-
-impl<Method: AggregateMethod + 'static> Aggregate for TaskFilterAnalytics<Method> {
-    fn event_name(&self) -> &'static str {
-        Method::event_name()
-    }
-
-    fn aggregate(self: Box<Self>, new: Box<Self>) -> Box<Self> {
-        Box::new(Self {
-            filtered_by_uid: self.filtered_by_uid | new.filtered_by_uid,
-            filtered_by_index_uid: self.filtered_by_index_uid | new.filtered_by_index_uid,
-            filtered_by_type: self.filtered_by_type | new.filtered_by_type,
-            filtered_by_status: self.filtered_by_status | new.filtered_by_status,
-            filtered_by_canceled_by: self.filtered_by_canceled_by | new.filtered_by_canceled_by,
-            filtered_by_before_enqueued_at: self.filtered_by_before_enqueued_at
-                | new.filtered_by_before_enqueued_at,
-            filtered_by_after_enqueued_at: self.filtered_by_after_enqueued_at
-                | new.filtered_by_after_enqueued_at,
-            filtered_by_before_started_at: self.filtered_by_before_started_at
-                | new.filtered_by_before_started_at,
-            filtered_by_after_started_at: self.filtered_by_after_started_at
-                | new.filtered_by_after_started_at,
-            filtered_by_before_finished_at: self.filtered_by_before_finished_at
-                | new.filtered_by_before_finished_at,
-            filtered_by_after_finished_at: self.filtered_by_after_finished_at
-                | new.filtered_by_after_finished_at,
-
-            marker: std::marker::PhantomData,
-        })
-    }
-
-    fn into_event(self: Box<Self>) -> serde_json::Value {
-        serde_json::to_value(*self).unwrap_or_default()
-    }
-}
-
 async fn cancel_tasks(
     index_scheduler: GuardedData<ActionPolicy<{ actions::TASKS_CANCEL }>, Data<IndexScheduler>>,
     params: AwebQueryParameter<TaskDeletionOrCancelationQuery, DeserrQueryParamError>,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let params = params.into_inner();
 
@@ -228,22 +172,21 @@ async fn cancel_tasks(
     }
 
     analytics.publish(
-        TaskFilterAnalytics::<CancelTasks> {
-            filtered_by_uid: params.uids.is_some(),
-            filtered_by_index_uid: params.index_uids.is_some(),
-            filtered_by_type: params.types.is_some(),
-            filtered_by_status: params.statuses.is_some(),
-            filtered_by_canceled_by: params.canceled_by.is_some(),
-            filtered_by_before_enqueued_at: params.before_enqueued_at.is_some(),
-            filtered_by_after_enqueued_at: params.after_enqueued_at.is_some(),
-            filtered_by_before_started_at: params.before_started_at.is_some(),
-            filtered_by_after_started_at: params.after_started_at.is_some(),
-            filtered_by_before_finished_at: params.before_finished_at.is_some(),
-            filtered_by_after_finished_at: params.after_finished_at.is_some(),
-
-            marker: std::marker::PhantomData,
-        },
-        &req,
+        "Tasks Canceled".to_string(),
+        json!({
+            "filtered_by_uid": params.uids.is_some(),
+            "filtered_by_index_uid": params.index_uids.is_some(),
+            "filtered_by_type": params.types.is_some(),
+            "filtered_by_status": params.statuses.is_some(),
+            "filtered_by_canceled_by": params.canceled_by.is_some(),
+            "filtered_by_before_enqueued_at": params.before_enqueued_at.is_some(),
+            "filtered_by_after_enqueued_at": params.after_enqueued_at.is_some(),
+            "filtered_by_before_started_at": params.before_started_at.is_some(),
+            "filtered_by_after_started_at": params.after_started_at.is_some(),
+            "filtered_by_before_finished_at": params.before_finished_at.is_some(),
+            "filtered_by_after_finished_at": params.after_finished_at.is_some(),
+        }),
+        Some(&req),
     );
 
     let query = params.into_query();
@@ -271,7 +214,7 @@ async fn delete_tasks(
     params: AwebQueryParameter<TaskDeletionOrCancelationQuery, DeserrQueryParamError>,
     req: HttpRequest,
     opt: web::Data<Opt>,
-    analytics: web::Data<Analytics>,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let params = params.into_inner();
 
@@ -280,24 +223,22 @@ async fn delete_tasks(
     }
 
     analytics.publish(
-        TaskFilterAnalytics::<DeleteTasks> {
-            filtered_by_uid: params.uids.is_some(),
-            filtered_by_index_uid: params.index_uids.is_some(),
-            filtered_by_type: params.types.is_some(),
-            filtered_by_status: params.statuses.is_some(),
-            filtered_by_canceled_by: params.canceled_by.is_some(),
-            filtered_by_before_enqueued_at: params.before_enqueued_at.is_some(),
-            filtered_by_after_enqueued_at: params.after_enqueued_at.is_some(),
-            filtered_by_before_started_at: params.before_started_at.is_some(),
-            filtered_by_after_started_at: params.after_started_at.is_some(),
-            filtered_by_before_finished_at: params.before_finished_at.is_some(),
-            filtered_by_after_finished_at: params.after_finished_at.is_some(),
-
-            marker: std::marker::PhantomData,
-        },
-        &req,
+        "Tasks Deleted".to_string(),
+        json!({
+            "filtered_by_uid": params.uids.is_some(),
+            "filtered_by_index_uid": params.index_uids.is_some(),
+            "filtered_by_type": params.types.is_some(),
+            "filtered_by_status": params.statuses.is_some(),
+            "filtered_by_canceled_by": params.canceled_by.is_some(),
+            "filtered_by_before_enqueued_at": params.before_enqueued_at.is_some(),
+            "filtered_by_after_enqueued_at": params.after_enqueued_at.is_some(),
+            "filtered_by_before_started_at": params.before_started_at.is_some(),
+            "filtered_by_after_started_at": params.after_started_at.is_some(),
+            "filtered_by_before_finished_at": params.before_finished_at.is_some(),
+            "filtered_by_after_finished_at": params.after_finished_at.is_some(),
+        }),
+        Some(&req),
     );
-
     let query = params.into_query();
 
     let (tasks, _) = index_scheduler.get_task_ids_from_authorized_indexes(
@@ -650,7 +591,7 @@ mod tests {
             let err = deserr_query_params::<TaskDeletionOrCancelationQuery>(params).unwrap_err();
             snapshot!(meili_snap::json_string!(err), @r###"
             {
-              "message": "Invalid value in parameter `types`: `createIndex` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentEdition`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.",
+              "message": "Invalid value in parameter `types`: `createIndex` is not a valid task type. Available types are `documentAdditionOrUpdate`, `documentDeletion`, `settingsUpdate`, `indexCreation`, `indexDeletion`, `indexUpdate`, `indexSwap`, `taskCancelation`, `taskDeletion`, `dumpCreation`, `snapshotCreation`.",
               "code": "invalid_task_types",
               "type": "invalid_request",
               "link": "https://docs.meilisearch.com/errors#invalid_task_types"