check consistency, create a dump and send push event for failed checks

4479: Skip reindexing when modifying unknown faceted fields r=dureuill a=Kerollmops

This PR improves Meilisearch's decision to reindex when a faceted field is added to the settings, but not a single document contains this field. It is effectively a waste of time to reindex documents when the engine needs to know a field.

This is related to a conversation [we have with our biggest customer (internal link)](https://discord.com/channels/1006923006964154428/1101213808627830794/1217112918857089187). They have 170 million documents, so reindexing this amount would be problematic.

---

The image is available by using the following Docker command. You can see the advancement of the image's build [on the GitHub CI page](https://github.com/meilisearch/meilisearch/actions/runs/8251688778).

```
docker pull getmeili/meilisearch:prototype-no-reindex-unknown-fields-0
```

Here is the hand-made test that shows that when modifying unknown filterable attributes, here `lol`, it doesn't reindex. However, when modifying the known `genre` field, it does reindex. You can see all that by looking at the time spent processing the update.

```json
{
  "uid": 3,
  "indexUid": "movies",
  "status": "succeeded",
  "type": "settingsUpdate",
  "canceledBy": null,
  "details": {
    "filterableAttributes": [
      "genres"
    ]
  },
  "error": null,
  "duration": "PT9.237703S",
  "enqueuedAt": "2024-03-12T15:34:26.836083Z",
  "startedAt": "2024-03-12T15:34:26.836374Z",
  "finishedAt": "2024-03-12T15:34:36.074077Z"
},
{
  "uid": 2,
  "indexUid": "movies",
  "status": "succeeded",
  "type": "settingsUpdate",
  "canceledBy": null,
  "details": {
    "filterableAttributes": [
      "lol"
    ]
  },
  "error": null,
  "duration": "PT0.000751S",
  "enqueuedAt": "2024-03-12T15:33:53.563923Z",
  "startedAt": "2024-03-12T15:33:53.565259Z",
  "finishedAt": "2024-03-12T15:33:53.56601Z"
},
{
  "uid": 0,
  "indexUid": "movies",
  "status": "succeeded",
  "type": "documentAdditionOrUpdate",
  "canceledBy": null,
  "details": {
    "receivedDocuments": 31944,
    "indexedDocuments": 31944
  },
  "error": null,
  "duration": "PT3.120723S",
  "enqueuedAt": "2024-02-17T10:35:55.042864Z",
  "startedAt": "2024-02-17T10:35:55.043505Z",
  "finishedAt": "2024-02-17T10:35:58.164228Z"
}
```

Co-authored-by: Clément Renault <clement@meilisearch.com>

.cargo/config.toml

@@ -0,0 +1,2 @@
+[alias]
+xtask = "run --release --package xtask --"

.github/ISSUE_TEMPLATE/sprint_issue.md

@@ -27,6 +27,23 @@ Related spec: WIP
 - [ ] If prototype validated, merge changes into `main`
 - [ ] Update the spec
 
+### Reminders when modifying the Setting API
+
+<!--- Special steps to remind when adding a new index setting -->
+
+- [ ] Ensure the new setting route is at least tested by the [`test_setting_routes` macro](https://github.com/meilisearch/meilisearch/blob/5204c0b60b384cbc79621b6b2176fca086069e8e/meilisearch/tests/settings/get_settings.rs#L276)
+- [ ] Ensure Analytics are fully implemented
+  - [ ] `/settings/my-new-setting` configurated in the [`make_setting_routes` macro](https://github.com/meilisearch/meilisearch/blob/5204c0b60b384cbc79621b6b2176fca086069e8e/meilisearch/src/routes/indexes/settings.rs#L141-L165)
+  - [ ] global `/settings` route configurated in the [`update_all` function](https://github.com/meilisearch/meilisearch/blob/5204c0b60b384cbc79621b6b2176fca086069e8e/meilisearch/src/routes/indexes/settings.rs#L655-L751)
+- [ ] Ensure the dump serializing is consistent with the `/settings` route serializing, e.g., enums case can be different (`camelCase` in route and `PascalCase` in the dump)
+
+#### Special cases when adding a setting for an experimental feature
+
+- [ ] ⚠️ API stability: The setting does not appear on the main settings route when the feature has never been enabled (e.g. mark it `Unset` when returned from the index in this situation. See [an example](https://github.com/meilisearch/meilisearch/blob/7a89abd2a025606a42f8b219e539117eb2eb029f/meilisearch-types/src/settings.rs#L608))
+- [ ] The setting cannot be set when the feature is disabled, either by the main settings route or the subroute (see [`validate_settings` function](https://github.com/meilisearch/meilisearch/blob/7a89abd2a025606a42f8b219e539117eb2eb029f/meilisearch/src/routes/indexes/settings.rs#L811))
+- [ ] If possible, the setting is reset when the feature is disabled (hard if it requires reindexing)
+
 ## Impacted teams
 
 <!---Ping the related teams. Ask for the engine manager if any hesitation-->
+<!---@meilisearch/docs-team when there is any API change, e.g. settings addition-->

.github/workflows/bench-manual.yml

@@ -0,0 +1,30 @@
+name: Bench (manual)
+
+on:
+    workflow_dispatch:
+        inputs:
+            workload:
+                description: 'The path to the workloads to execute (workloads/...)'
+                required: true
+                default: 'workloads/movies.json'
+
+env:
+    WORKLOAD_NAME: ${{ github.event.inputs.workload }}
+
+jobs:
+    benchmarks:
+        name: Run and upload benchmarks
+        runs-on: benchmarks
+        timeout-minutes: 180 # 3h
+        steps:
+            - uses: actions/checkout@v3
+            - uses: actions-rs/toolchain@v1
+              with:
+                profile: minimal
+                toolchain: stable
+                override: true
+
+            - name: Run benchmarks - workload ${WORKLOAD_NAME} - branch ${{ github.ref }} - commit ${{ github.sha }}
+              run: |
+               cargo xtask bench --api-key "${{ secrets.BENCHMARK_API_KEY }}" --dashboard-url "${{ vars.BENCHMARK_DASHBOARD_URL }}" --reason "Manual [Run #${{ github.run_id }}](https://github.com/meilisearch/meilisearch/actions/runs/${{ github.run_id }})" -- ${WORKLOAD_NAME}
+

.github/workflows/bench-pr.yml

@@ -0,0 +1,46 @@
+name: Bench (PR)
+on:
+    issue_comment:
+        types: [created]
+
+permissions:
+    issues: write
+
+env:
+    GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
+
+jobs:
+    run-benchmarks-on-comment:
+      if: startsWith(github.event.comment.body, '/bench')
+      name: Run and upload benchmarks
+      runs-on: benchmarks
+      timeout-minutes: 180 # 3h
+      steps:
+        - name: Check for Command
+          id: command
+          uses: xt0rted/slash-command-action@v2
+          with:
+              command: bench
+              reaction-type: "rocket"
+              repo-token: ${{ env.GH_TOKEN }}
+
+        - uses: xt0rted/pull-request-comment-branch@v2
+          id: comment-branch
+          with:
+            repo_token: ${{ env.GH_TOKEN }}
+
+        - uses: actions/checkout@v3
+          if: success()
+          with:
+            fetch-depth: 0 # fetch full history to be able to get main commit sha
+            ref: ${{ steps.comment-branch.outputs.head_ref }}
+
+        - uses: actions-rs/toolchain@v1
+          with:
+            profile: minimal
+            toolchain: stable
+            override: true
+
+        - name: Run benchmarks on PR ${{ github.event.issue.id }}
+          run: |
+            cargo xtask bench --api-key "${{ secrets.BENCHMARK_API_KEY }}" --dashboard-url "${{ vars.BENCHMARK_DASHBOARD_URL }}" --reason "[Comment](${{ github.event.comment.url }}) on [#${{github.event.issue.id}}](${{ github.event.issue.url }})" -- ${{ steps.command.outputs.command-arguments }}

.github/workflows/bench-push-indexing.yml

@@ -0,0 +1,25 @@
+name: Indexing bench (push)
+
+on:
+    push:
+        branches:
+            - main
+
+jobs:
+    benchmarks:
+        name: Run and upload benchmarks
+        runs-on: benchmarks
+        timeout-minutes: 180 # 3h
+        steps:
+          - uses: actions/checkout@v3
+          - uses: actions-rs/toolchain@v1
+            with:
+              profile: minimal
+              toolchain: stable
+              override: true
+
+          # Run benchmarks
+          - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch main - Commit ${{ github.sha }}
+            run: |
+              cargo xtask bench --api-key "${{ secrets.BENCHMARK_API_KEY }}" --dashboard-url "${{ vars.BENCHMARK_DASHBOARD_URL }}" --reason "Push on `main` [Run #${{ github.run_id }}](https://github.com/meilisearch/meilisearch/actions/runs/${{ github.run_id }})" -- workloads/*.json
+

.github/workflows/publish-docker-images.yml

@@ -97,7 +97,7 @@ jobs:
       - name: Send CI information to Cloud team
         # Do not send if nightly build (i.e. 'schedule' or 'workflow_dispatch' event)
         if: github.event_name == 'push'
-        uses: peter-evans/repository-dispatch@v2
+        uses: peter-evans/repository-dispatch@v3
         with:
           token: ${{ secrets.MEILI_BOT_GH_PAT }}
           repository: meilisearch/meilisearch-cloud

.github/workflows/sdks-tests.yml

@@ -22,7 +22,7 @@ jobs:
     outputs:
       docker-image: ${{ steps.define-image.outputs.docker-image }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Define the Docker image we need to use
         id: define-image
         run: |
@@ -46,11 +46,11 @@ jobs:
       MEILISEARCH_VERSION: ${{ needs.define-docker-image.outputs.docker-image }}
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-dotnet
       - name: Setup .NET Core
-        uses: actions/setup-dotnet@v3
+        uses: actions/setup-dotnet@v4
         with:
           dotnet-version: "6.0.x"
       - name: Install dependencies
@@ -75,12 +75,12 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-dart
       - uses: dart-lang/setup-dart@v1
         with:
-          sdk: 3.1.1
+          sdk: 'latest'
       - name: Install dependencies
         run: dart pub get
       - name: Run integration tests
@@ -100,10 +100,10 @@ jobs:
           - '7700:7700'
     steps:
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: stable
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-go
       - name: Get dependencies
@@ -129,11 +129,11 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-java
       - name: Set up Java
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: 8
           distribution: 'zulu'
@@ -156,7 +156,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-js
       - name: Setup node
@@ -191,7 +191,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-php
       - name: Install PHP
@@ -220,11 +220,11 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-python
       - name: Set up Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
       - name: Install pipenv
         uses: dschep/install-pipenv-action@v1
       - name: Install dependencies
@@ -245,7 +245,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-ruby
       - name: Set up Ruby 3
@@ -270,7 +270,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-rust
       - name: Build
@@ -291,7 +291,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-swift
       - name: Run tests
@@ -314,7 +314,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-js-plugins
       - name: Setup node
@@ -345,7 +345,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-rails
       - name: Set up Ruby 3
@@ -369,7 +369,7 @@ jobs:
         ports:
           - '7700:7700'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           repository: meilisearch/meilisearch-symfony
       - name: Install PHP

.github/workflows/test-suite.yml

@@ -31,17 +31,10 @@ jobs:
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
       - name: Setup test with Rust stable
-        if: github.event_name != 'schedule'
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
           override: true
-      - name: Setup test with Rust nightly
-        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
       - name: Run cargo check without any default features
@@ -66,6 +59,10 @@ jobs:
       - uses: actions/checkout@v3
       - name: Cache dependencies
         uses: Swatinem/rust-cache@v2.7.1
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -78,7 +75,7 @@ jobs:
           args: --locked --release --all
 
   test-all-features:
-    name: Tests all features
+    name: Tests almost all features
     runs-on: ubuntu-latest
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
@@ -94,16 +91,12 @@ jobs:
         with:
           toolchain: stable
           override: true
-      - name: Run cargo build with all features
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --workspace --locked --release --all-features
-      - name: Run cargo test with all features
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace --locked --release --all-features
+      - name: Run cargo build with almost all features
+        run: |
+          cargo build --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda)"
+      - name: Run cargo test with almost all features
+        run: |
+          cargo test --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda)"
 
   test-disabled-tokenization:
     name: Test disabled tokenization
@@ -164,7 +157,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.71.1
+          toolchain: 1.75.0
           override: true
           components: clippy
       - name: Cache dependencies

.gitignore

@@ -9,6 +9,8 @@
 /data.ms
 /snapshots
 /dumps
+/bench
+/_xtask_benchmark.ms
 
 # Snapshots
 ## ... large

CONTRIBUTING.md

@@ -75,6 +75,12 @@ If you get a "Too many open files" error you might want to increase the open fil
 ulimit -Sn 3000
 ```
 
+#### Build tools
+
+Meilisearch follows the [cargo xtask](https://github.com/matklad/cargo-xtask) workflow to provide some build tools.
+
+Run `cargo xtask --help` from the root of the repository to find out what is available.
+
 ## Git Guidelines
 
 ### Git Branches

Cargo.toml

@@ -16,11 +16,16 @@ members = [
     "json-depth-checker",
     "benchmarks",
     "fuzzers",
+    "tracing-trace",
+    "xtask", "build-info",
 ]
 
 [workspace.package]
-version = "1.6.0"
-authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
+version = "1.7.1"
+authors = [
+    "Quentin de Quelen <quentin@dequelen.me>",
+    "Clément Renault <clement@meilisearch.com>",
+]
 description = "Meilisearch HTTP server"
 homepage = "https://meilisearch.com"
 readme = "README.md"

Dockerfile

@@ -1,5 +1,5 @@
 # Compile
-FROM    rust:1.71.1-alpine3.18 AS compiler
+FROM    rust:1.75.0-alpine3.18 AS compiler
 
 RUN     apk add -q --update-cache --no-cache build-base openssl-dev
 
@@ -8,7 +8,7 @@ WORKDIR /
 ARG     COMMIT_SHA
 ARG     COMMIT_DATE
 ARG     GIT_TAG
-ENV     VERGEN_GIT_SHA=${COMMIT_SHA} VERGEN_GIT_COMMIT_TIMESTAMP=${COMMIT_DATE} VERGEN_GIT_SEMVER_LIGHTWEIGHT=${GIT_TAG}
+ENV     VERGEN_GIT_SHA=${COMMIT_SHA} VERGEN_GIT_COMMIT_TIMESTAMP=${COMMIT_DATE} VERGEN_GIT_DESCRIBE=${GIT_TAG}
 ENV     RUSTFLAGS="-C target-feature=-crt-static"
 
 COPY    . .

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019-2022 Meili SAS
+Copyright (c) 2019-2024 Meili SAS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -41,10 +41,10 @@ Meilisearch helps you shape a delightful search experience in a snap, offering f
 ## ✨ Features
 
 - **Search-as-you-type:** find search results in less than 50 milliseconds
-- **[Typo tolerance](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#typo-tolerance):** get relevant matches even when queries contain typos and misspellings
-- **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code
+- **[Typo tolerance](https://www.meilisearch.com/docs/learn/configuration/typo_tolerance?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** get relevant matches even when queries contain typos and misspellings
+- **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your users' search experience with custom filters and build a faceted search interface in a few lines of code
 - **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** sort results based on price, date, or pretty much anything else your users need
-- **[Synonym support](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#synonyms):** configure synonyms to include more relevant content in your search results
+- **[Synonym support](https://www.meilisearch.com/docs/learn/configuration/synonyms?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** configure synonyms to include more relevant content in your search results
 - **[Geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** filter and sort documents based on geographic data
 - **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
 - **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** control which users can access what data with API keys that allow fine-grained permissions handling
@@ -61,8 +61,6 @@ You can consult Meilisearch's documentation at [https://www.meilisearch.com/docs
 
 For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
 
-You may also want to check out [Meilisearch 101](https://www.meilisearch.com/docs/learn/getting_started/filtering_and_sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) for an introduction to some of Meilisearch's most popular features.
-
 ## ⚡ Supercharge your Meilisearch experience
 
 Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). No credit card required.
@@ -101,7 +99,7 @@ Meilisearch is a search engine created by [Meili](https://www.welcometothejungle
 
 - For feature requests, please visit our [product repository](https://github.com/meilisearch/product/discussions)
 - Found a bug? Open an [issue](https://github.com/meilisearch/meilisearch/issues)!
-- Want to be part of our Discord community? [Join us!](https://discord.gg/meilisearch)
+- Want to be part of our Discord community? [Join us!](https://discord.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
 
 Thank you for your support!
 

assets/grafana-dashboard.json

@@ -106,7 +106,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "meilisearch_index_count{job=\"meilisearch\", instance=\"$instance\"}",
+          "expr": "meilisearch_index_count{job=\"$job\", instance=\"$instance\"}",
           "interval": "",
           "legendFormat": "",
           "range": true,
@@ -165,7 +165,7 @@
             "type": "prometheus"
           },
           "editorMode": "builder",
-          "expr": "meilisearch_index_docs_count{job=\"meilisearch\", index=\"$Index\", instance=\"$instance\"}",
+          "expr": "meilisearch_index_docs_count{job=\"$job\", index=\"$Index\", instance=\"$instance\"}",
           "hide": false,
           "range": true,
           "refId": "A"
@@ -228,7 +228,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "round(increase(meilisearch_http_requests_total{method=\"POST\", path=\"/indexes/$Index/search\", job=\"meilisearch\"}[1h]))",
+          "expr": "round(increase(meilisearch_http_requests_total{method=\"POST\", path=\"/indexes/$Index/search\", job=\"$job\"}[1h]))",
           "interval": "",
           "legendFormat": "",
           "range": true,
@@ -288,7 +288,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "round(increase(meilisearch_http_requests_total{method=\"POST\", path=\"/indexes/$Index/search\", job=\"meilisearch\"}[24h]))",
+          "expr": "round(increase(meilisearch_http_requests_total{method=\"POST\", path=\"/indexes/$Index/search\", job=\"$job\"}[24h]))",
           "interval": "",
           "legendFormat": "",
           "range": true,
@@ -348,7 +348,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "round(increase(meilisearch_http_requests_total{method=\"POST\", path=\"/indexes/$Index/search\", job=\"meilisearch\"}[30d]))",
+          "expr": "round(increase(meilisearch_http_requests_total{method=\"POST\", path=\"/indexes/$Index/search\", job=\"$job\"}[30d]))",
           "interval": "",
           "legendFormat": "",
           "range": true,
@@ -447,7 +447,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "meilisearch_db_size_bytes{job=\"meilisearch\", instance=\"$instance\"}",
+          "expr": "meilisearch_db_size_bytes{job=\"$job\", instance=\"$instance\"}",
           "interval": "",
           "legendFormat": "Database size on disk",
           "range": true,
@@ -458,7 +458,7 @@
             "type": "prometheus"
           },
           "editorMode": "builder",
-          "expr": "meilisearch_used_db_size_bytes{job=\"meilisearch\", instance=\"$instance\"}",
+          "expr": "meilisearch_used_db_size_bytes{job=\"$job\", instance=\"$instance\"}",
           "hide": false,
           "legendFormat": "Used bytes",
           "range": true,
@@ -553,7 +553,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "rate(meilisearch_http_response_time_seconds_sum{instance=\"$instance\", job=\"meilisearch\"}[5m]) / rate(meilisearch_http_response_time_seconds_count[5m])",
+          "expr": "rate(meilisearch_http_response_time_seconds_sum{instance=\"$instance\", job=\"$job\"}[5m]) / rate(meilisearch_http_response_time_seconds_count[5m])",
           "interval": "",
           "legendFormat": "{{method}} {{path}}",
           "range": true,
@@ -646,7 +646,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "rate(meilisearch_http_requests_total{instance=\"$instance\", job=\"meilisearch\"}[5m])",
+          "expr": "rate(meilisearch_http_requests_total{instance=\"$instance\", job=\"$job\"}[5m])",
           "interval": "",
           "legendFormat": "{{method}} {{path}}",
           "range": true,
@@ -744,7 +744,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "sum by(le) (increase(meilisearch_http_response_time_seconds_bucket{path=\"/indexes/$Index/search\", instance=\"$instance\", job=\"meilisearch\"}[30s]))",
+          "expr": "sum by(le) (increase(meilisearch_http_response_time_seconds_bucket{path=\"/indexes/$Index/search\", instance=\"$instance\", job=\"$job\"}[30s]))",
           "format": "heatmap",
           "interval": "",
           "legendFormat": "{{le}}",
@@ -854,7 +854,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "meilisearch_nb_tasks{instance=\"$instance\", job=\"meilisearch\", kind=\"statuses\"}",
+          "expr": "meilisearch_nb_tasks{instance=\"$instance\", job=\"$job\", kind=\"statuses\"}",
           "interval": "",
           "legendFormat": "{{value}} ",
           "range": true,
@@ -947,7 +947,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "meilisearch_nb_tasks{instance=\"$instance\", job=\"meilisearch\", kind=\"types\"}",
+          "expr": "meilisearch_nb_tasks{instance=\"$instance\", job=\"$job\", kind=\"types\"}",
           "interval": "",
           "legendFormat": "{{value}} ",
           "range": true,
@@ -1040,7 +1040,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "meilisearch_nb_tasks{instance=\"$instance\", job=\"meilisearch\", kind=\"indexes\"}",
+          "expr": "meilisearch_nb_tasks{instance=\"$instance\", job=\"$job\", kind=\"indexes\"}",
           "interval": "",
           "legendFormat": "{{value}} ",
           "range": true,
@@ -1161,7 +1161,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "rate(process_cpu_seconds_total{job=\"meilisearch\", instance=\"$instance\"}[1m])",
+          "expr": "rate(process_cpu_seconds_total{job=\"$job\", instance=\"$instance\"}[1m])",
           "interval": "",
           "legendFormat": "process",
           "range": true,
@@ -1264,7 +1264,7 @@
           },
           "editorMode": "builder",
           "exemplar": true,
-          "expr": "process_resident_memory_bytes{job=\"meilisearch\", instance=\"$instance\"} / 1024 / 1024",
+          "expr": "process_resident_memory_bytes{job=\"$job\", instance=\"$instance\"} / 1024 / 1024",
           "interval": "",
           "legendFormat": "process",
           "range": true,
@@ -1342,6 +1342,33 @@
         "skipUrlSync": false,
         "sort": 0,
         "type": "query"
+      },
+      {
+        "current": {
+          "selected": true,
+          "text": "meilisearch",
+          "value": "meilisearch"
+        },
+        "datasource": {
+          "type": "prometheus"
+        },
+        "definition": "label_values(job)",
+        "description": "Prometheus job_name from scrape config (default is meilisearch)",
+        "hide": 0,
+        "includeAll": false,
+        "label": "Job",
+        "multi": false,
+        "name": "job",
+        "options": [],
+        "query": {
+          "query": "label_values(job)",
+          "refId": "StandardVariableQuery"
+        },
+        "refresh": 1,
+        "regex": "",
+        "skipUrlSync": false,
+        "sort": 0,
+        "type": "query"
       }
     ]
   },

benchmarks/Cargo.toml

@@ -11,24 +11,24 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.70"
-csv = "1.2.1"
+anyhow = "1.0.79"
+csv = "1.3.0"
 milli = { path = "../milli" }
-mimalloc = { version = "0.1.37", default-features = false }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
+mimalloc = { version = "0.1.39", default-features = false }
+serde_json = { version = "1.0.111", features = ["preserve_order"] }
 
 [dev-dependencies]
 criterion = { version = "0.5.1", features = ["html_reports"] }
 rand = "0.8.5"
 rand_chacha = "0.3.1"
-roaring = "0.10.1"
+roaring = "0.10.2"
 
 [build-dependencies]
-anyhow = "1.0.70"
-bytes = "1.4.0"
+anyhow = "1.0.79"
+bytes = "1.5.0"
 convert_case = "0.6.0"
-flate2 = "1.0.25"
-reqwest = { version = "0.11.16", features = ["blocking", "rustls-tls"], default-features = false }
+flate2 = "1.0.28"
+reqwest = { version = "0.11.23", features = ["blocking", "rustls-tls"], default-features = false }
 
 [features]
 default = ["milli/all-tokenizations"]

build-info/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "build-info"
+version.workspace = true
+authors.workspace = true
+description.workspace = true
+homepage.workspace = true
+readme.workspace = true
+edition.workspace = true
+license.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+time = { version = "0.3.34", features = ["parsing"] }
+
+[build-dependencies]
+anyhow = "1.0.80"
+vergen-git2 = "1.0.0-beta.2"

build-info/build.rs

@@ -0,0 +1,22 @@
+fn main() {
+    if let Err(err) = emit_git_variables() {
+        println!("cargo:warning=vergen: {}", err);
+    }
+}
+
+fn emit_git_variables() -> anyhow::Result<()> {
+    // Note: any code that needs VERGEN_ environment variables should take care to define them manually in the Dockerfile and pass them
+    // in the corresponding GitHub workflow (publish_docker.yml).
+    // This is due to the Dockerfile building the binary outside of the git directory.
+    let mut builder = vergen_git2::Git2Builder::default();
+
+    builder.branch(true);
+    builder.commit_timestamp(true);
+    builder.commit_message(true);
+    builder.describe(true, true, None);
+    builder.sha(false);
+
+    let git2 = builder.build()?;
+
+    vergen_git2::Emitter::default().fail_on_error().add_instructions(&git2)?.emit()
+}

build-info/src/lib.rs

@@ -0,0 +1,203 @@
+use time::format_description::well_known::Iso8601;
+
+#[derive(Debug, Clone)]
+pub struct BuildInfo {
+    pub branch: Option<&'static str>,
+    pub describe: Option<DescribeResult>,
+    pub commit_sha1: Option<&'static str>,
+    pub commit_msg: Option<&'static str>,
+    pub commit_timestamp: Option<time::OffsetDateTime>,
+}
+
+impl BuildInfo {
+    pub fn from_build() -> Self {
+        let branch: Option<&'static str> = option_env!("VERGEN_GIT_BRANCH");
+        let describe = DescribeResult::from_build();
+        let commit_sha1 = option_env!("VERGEN_GIT_SHA");
+        let commit_msg = option_env!("VERGEN_GIT_COMMIT_MESSAGE");
+        let commit_timestamp = option_env!("VERGEN_GIT_COMMIT_TIMESTAMP");
+
+        let commit_timestamp = commit_timestamp.and_then(|commit_timestamp| {
+            time::OffsetDateTime::parse(commit_timestamp, &Iso8601::DEFAULT).ok()
+        });
+
+        Self { branch, describe, commit_sha1, commit_msg, commit_timestamp }
+    }
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
+pub enum DescribeResult {
+    Prototype { name: &'static str },
+    Release { version: &'static str, major: u64, minor: u64, patch: u64 },
+    Prerelease { version: &'static str, major: u64, minor: u64, patch: u64, rc: u64 },
+    NotATag { describe: &'static str },
+}
+
+impl DescribeResult {
+    pub fn new(describe: &'static str) -> Self {
+        if let Some(name) = prototype_name(describe) {
+            Self::Prototype { name }
+        } else if let Some(release) = release_version(describe) {
+            release
+        } else if let Some(prerelease) = prerelease_version(describe) {
+            prerelease
+        } else {
+            Self::NotATag { describe }
+        }
+    }
+
+    pub fn from_build() -> Option<Self> {
+        let describe: &'static str = option_env!("VERGEN_GIT_DESCRIBE")?;
+        Some(Self::new(describe))
+    }
+
+    pub fn as_tag(&self) -> Option<&'static str> {
+        match self {
+            DescribeResult::Prototype { name } => Some(name),
+            DescribeResult::Release { version, .. } => Some(version),
+            DescribeResult::Prerelease { version, .. } => Some(version),
+            DescribeResult::NotATag { describe: _ } => None,
+        }
+    }
+
+    pub fn as_prototype(&self) -> Option<&'static str> {
+        match self {
+            DescribeResult::Prototype { name } => Some(name),
+            DescribeResult::Release { .. }
+            | DescribeResult::Prerelease { .. }
+            | DescribeResult::NotATag { .. } => None,
+        }
+    }
+}
+
+/// Parses the input as a prototype name.
+///
+/// Returns `Some(prototype_name)` if the following conditions are met on this value:
+///
+/// 1. starts with `prototype-`,
+/// 2. ends with `-<some_number>`,
+/// 3. does not end with `<some_number>-<some_number>`.
+///
+/// Otherwise, returns `None`.
+fn prototype_name(describe: &'static str) -> Option<&'static str> {
+    if !describe.starts_with("prototype-") {
+        return None;
+    }
+
+    let mut rsplit_prototype = describe.rsplit('-');
+    // last component MUST be a number
+    rsplit_prototype.next()?.parse::<u64>().ok()?;
+    // before than last component SHALL NOT be a number
+    rsplit_prototype.next()?.parse::<u64>().err()?;
+
+    Some(describe)
+}
+
+fn release_version(describe: &'static str) -> Option<DescribeResult> {
+    if !describe.starts_with('v') {
+        return None;
+    }
+
+    // full release version don't contain a `-`
+    if describe.contains('-') {
+        return None;
+    }
+
+    // full release version parse as vX.Y.Z, with X, Y, Z numbers.
+    let mut dots = describe[1..].split('.');
+    let major: u64 = dots.next()?.parse().ok()?;
+    let minor: u64 = dots.next()?.parse().ok()?;
+    let patch: u64 = dots.next()?.parse().ok()?;
+
+    if dots.next().is_some() {
+        return None;
+    }
+
+    Some(DescribeResult::Release { version: describe, major, minor, patch })
+}
+
+fn prerelease_version(describe: &'static str) -> Option<DescribeResult> {
+    // prerelease version is in the shape vM.N.P-rc.C
+    let mut hyphen = describe.rsplit('-');
+    let prerelease = hyphen.next()?;
+    if !prerelease.starts_with("rc.") {
+        return None;
+    }
+
+    let rc: u64 = prerelease[3..].parse().ok()?;
+
+    let release = hyphen.next()?;
+
+    let DescribeResult::Release { version: _, major, minor, patch } = release_version(release)?
+    else {
+        return None;
+    };
+
+    Some(DescribeResult::Prerelease { version: describe, major, minor, patch, rc })
+}
+
+#[cfg(test)]
+mod test {
+    use super::DescribeResult;
+
+    fn assert_not_a_tag(describe: &'static str) {
+        assert_eq!(DescribeResult::NotATag { describe }, DescribeResult::new(describe))
+    }
+
+    fn assert_proto(describe: &'static str) {
+        assert_eq!(DescribeResult::Prototype { name: describe }, DescribeResult::new(describe))
+    }
+
+    fn assert_release(describe: &'static str, major: u64, minor: u64, patch: u64) {
+        assert_eq!(
+            DescribeResult::Release { version: describe, major, minor, patch },
+            DescribeResult::new(describe)
+        )
+    }
+
+    fn assert_prerelease(describe: &'static str, major: u64, minor: u64, patch: u64, rc: u64) {
+        assert_eq!(
+            DescribeResult::Prerelease { version: describe, major, minor, patch, rc },
+            DescribeResult::new(describe)
+        )
+    }
+
+    #[test]
+    fn not_a_tag() {
+        assert_not_a_tag("whatever-fuzzy");
+        assert_not_a_tag("whatever-fuzzy-5-ggg-dirty");
+        assert_not_a_tag("whatever-fuzzy-120-ggg-dirty");
+
+        // technically a tag, but not a proto nor a version, so not parsed as a tag
+        assert_not_a_tag("whatever");
+
+        // dirty version
+        assert_not_a_tag("v1.7.0-1-ggga-dirty");
+        assert_not_a_tag("v1.7.0-rc.1-1-ggga-dirty");
+
+        // after version
+        assert_not_a_tag("v1.7.0-1-ggga");
+        assert_not_a_tag("v1.7.0-rc.1-1-ggga");
+
+        // after proto
+        assert_not_a_tag("protoype-tag-0-1-ggga");
+        assert_not_a_tag("protoype-tag-0-1-ggga-dirty");
+    }
+
+    #[test]
+    fn prototype() {
+        assert_proto("prototype-tag-0");
+        assert_proto("prototype-tag-10");
+        assert_proto("prototype-long-name-tag-10");
+    }
+
+    #[test]
+    fn release() {
+        assert_release("v1.7.2", 1, 7, 2);
+    }
+
+    #[test]
+    fn prerelease() {
+        assert_prerelease("v1.7.2-rc.3", 1, 7, 2, 3);
+    }
+}

dump/Cargo.toml

@@ -11,22 +11,22 @@ readme.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.70"
-flate2 = "1.0.25"
-http = "0.2.9"
-log = "0.4.17"
+anyhow = "1.0.79"
+flate2 = "1.0.28"
+http = "0.2.11"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-once_cell = "1.17.1"
-regex = "1.7.3"
-roaring = { version = "0.10.1", features = ["serde"] }
-serde = { version = "1.0.160", features = ["derive"] }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
-tar = "0.4.38"
-tempfile = "3.5.0"
-thiserror = "1.0.40"
-time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-uuid = { version = "1.3.1", features = ["serde", "v4"] }
+once_cell = "1.19.0"
+regex = "1.10.2"
+roaring = { version = "0.10.2", features = ["serde"] }
+serde = { version = "1.0.195", features = ["derive"] }
+serde_json = { version = "1.0.111", features = ["preserve_order"] }
+tar = "0.4.40"
+tempfile = "3.9.0"
+thiserror = "1.0.56"
+time = { version = "0.3.31", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+tracing = "0.1.40"
+uuid = { version = "1.6.1", features = ["serde", "v4"] }
 
 [dev-dependencies]
 big_s = "1.0.2"

dump/src/reader/compat/v1_to_v2.rs

@@ -120,7 +120,7 @@ impl From<v1::settings::Settings> for v2::Settings<v2::Unchecked> {
                                 criterion.as_ref().map(ToString::to_string)
                             }
                             Err(()) => {
-                                log::warn!(
+                                tracing::warn!(
                                     "Could not import the following ranking rule: `{}`.",
                                     ranking_rule
                                 );
@@ -152,11 +152,11 @@ impl From<v1::update::UpdateStatus> for Option<v2::updates::UpdateStatus> {
         use v2::updates::UpdateStatus as UpdateStatusV2;
         Some(match source {
             UpdateStatusV1::Enqueued { content } => {
-                log::warn!(
+                tracing::warn!(
                     "Cannot import task {} (importing enqueued tasks from v1 dumps is unsupported)",
                     content.update_id
                 );
-                log::warn!("Task will be skipped in the queue of imported tasks.");
+                tracing::warn!("Task will be skipped in the queue of imported tasks.");
 
                 return None;
             }
@@ -229,7 +229,7 @@ impl From<v1::update::UpdateType> for Option<v2::updates::UpdateMeta> {
         Some(match source {
             v1::update::UpdateType::ClearAll => v2::updates::UpdateMeta::ClearDocuments,
             v1::update::UpdateType::Customs => {
-                log::warn!("Ignoring task with type 'Customs' that is no longer supported");
+                tracing::warn!("Ignoring task with type 'Customs' that is no longer supported");
                 return None;
             }
             v1::update::UpdateType::DocumentsAddition { .. } => {
@@ -296,7 +296,7 @@ impl From<v1::settings::RankingRule> for Option<v2::settings::Criterion> {
             v1::settings::RankingRule::Proximity => Some(v2::settings::Criterion::Proximity),
             v1::settings::RankingRule::Attribute => Some(v2::settings::Criterion::Attribute),
             v1::settings::RankingRule::WordsPosition => {
-                log::warn!("Removing the 'WordsPosition' ranking rule that is no longer supported, please check the resulting ranking rules of your indexes");
+                tracing::warn!("Removing the 'WordsPosition' ranking rule that is no longer supported, please check the resulting ranking rules of your indexes");
                 None
             }
             v1::settings::RankingRule::Exactness => Some(v2::settings::Criterion::Exactness),

dump/src/reader/compat/v2_to_v3.rs

@@ -1,4 +1,3 @@
-use std::convert::TryInto;
 use std::str::FromStr;
 
 use time::OffsetDateTime;
@@ -146,8 +145,8 @@ impl From<v2::updates::UpdateStatus> for v3::updates::UpdateStatus {
                         started_processing_at: processing.started_processing_at,
                     }),
                     Err(e) => {
-                        log::warn!("Error with task {}: {}", processing.from.update_id, e);
-                        log::warn!("Task will be marked as `Failed`.");
+                        tracing::warn!("Error with task {}: {}", processing.from.update_id, e);
+                        tracing::warn!("Task will be marked as `Failed`.");
                         v3::updates::UpdateStatus::Failed(v3::updates::Failed {
                             from: v3::updates::Processing {
                                 from: v3::updates::Enqueued {
@@ -172,8 +171,8 @@ impl From<v2::updates::UpdateStatus> for v3::updates::UpdateStatus {
                         enqueued_at: enqueued.enqueued_at,
                     }),
                     Err(e) => {
-                        log::warn!("Error with task {}: {}", enqueued.update_id, e);
-                        log::warn!("Task will be marked as `Failed`.");
+                        tracing::warn!("Error with task {}: {}", enqueued.update_id, e);
+                        tracing::warn!("Task will be marked as `Failed`.");
                         v3::updates::UpdateStatus::Failed(v3::updates::Failed {
                             from: v3::updates::Processing {
                                 from: v3::updates::Enqueued {
@@ -353,7 +352,7 @@ impl From<String> for v3::Code {
             "malformed_payload" => v3::Code::MalformedPayload,
             "missing_payload" => v3::Code::MissingPayload,
             other => {
-                log::warn!("Unknown error code {}", other);
+                tracing::warn!("Unknown error code {}", other);
                 v3::Code::UnretrievableErrorCode
             }
         }

dump/src/reader/compat/v3_to_v4.rs

@@ -76,20 +76,20 @@ impl CompatV3ToV4 {
                         let index_uid = match index_uid {
                             Some(uid) => uid,
                             None => {
-                                log::warn!(
+                                tracing::warn!(
                                     "Error while importing the update {}.",
                                     task.update.id()
                                 );
-                                log::warn!(
+                                tracing::warn!(
                                     "The index associated to the uuid `{}` could not be retrieved.",
                                     task.uuid.to_string()
                                 );
                                 if task.update.is_finished() {
                                     // we're fucking with his history but not his data, that's ok-ish.
-                                    log::warn!("The index-uuid will be set as `unknown`.");
+                                    tracing::warn!("The index-uuid will be set as `unknown`.");
                                     String::from("unknown")
                                 } else {
-                                    log::warn!("The task will be ignored.");
+                                    tracing::warn!("The task will be ignored.");
                                     return None;
                                 }
                             }

dump/src/reader/compat/v4_to_v5.rs

@@ -305,7 +305,7 @@ impl From<v4::ResponseError> for v5::ResponseError {
             "invalid_api_key_expires_at" => v5::Code::InvalidApiKeyExpiresAt,
             "invalid_api_key_description" => v5::Code::InvalidApiKeyDescription,
             other => {
-                log::warn!("Unknown error code {}", other);
+                tracing::warn!("Unknown error code {}", other);
                 v5::Code::UnretrievableErrorCode
             }
         };

dump/src/reader/compat/v5_to_v6.rs

@@ -304,7 +304,7 @@ impl From<v5::ResponseError> for v6::ResponseError {
             "immutable_field" => v6::Code::BadRequest,
             "api_key_already_exists" => v6::Code::ApiKeyAlreadyExists,
             other => {
-                log::warn!("Unknown error code {}", other);
+                tracing::warn!("Unknown error code {}", other);
                 v6::Code::UnretrievableErrorCode
             }
         };
@@ -329,7 +329,7 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                                     new_ranking_rules.push(new_rule);
                                 }
                                 Err(_) => {
-                                    log::warn!("Error while importing settings. The ranking rule `{rule}` does not exist anymore.")
+                                    tracing::warn!("Error while importing settings. The ranking rule `{rule}` does not exist anymore.")
                                 }
                             }
                         }

dump/src/reader/v3/errors.rs

@@ -1,5 +1,6 @@
 use serde::{Deserialize, Serialize};
 
+#[allow(clippy::enum_variant_names)]
 #[derive(Serialize, Deserialize, Debug, Clone, Copy)]
 pub enum Code {
     // index related error

dump/src/reader/v4/errors.rs

@@ -95,6 +95,7 @@ impl fmt::Display for ErrorType {
     }
 }
 
+#[allow(clippy::enum_variant_names)]
 #[derive(Serialize, Deserialize, Debug, Clone, Copy)]
 pub enum Code {
     // index related error

dump/src/reader/v5/errors.rs

@@ -31,6 +31,7 @@ impl ResponseError {
     }
 }
 
+#[allow(clippy::enum_variant_names)]
 #[derive(Deserialize, Debug, Clone, Copy)]
 #[cfg_attr(test, derive(serde::Serialize))]
 pub enum Code {

dump/src/reader/v6/mod.rs

@@ -2,10 +2,10 @@ use std::fs::{self, File};
 use std::io::{BufRead, BufReader, ErrorKind};
 use std::path::Path;
 
-use log::debug;
 pub use meilisearch_types::milli;
 use tempfile::TempDir;
 use time::OffsetDateTime;
+use tracing::debug;
 use uuid::Uuid;
 
 use super::Document;

file-store/Cargo.toml

@@ -11,9 +11,10 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-tempfile = "3.5.0"
-thiserror = "1.0.40"
-uuid = { version = "1.3.1", features = ["serde", "v4"] }
+tempfile = "3.9.0"
+thiserror = "1.0.56"
+tracing = "0.1.40"
+uuid = { version = "1.6.1", features = ["serde", "v4"] }
 
 [dev-dependencies]
-faux = "0.1.9"
+faux = "0.1.10"

file-store/src/lib.rs

@@ -1,5 +1,5 @@
 use std::fs::File as StdFile;
-use std::ops::{Deref, DerefMut};
+use std::io::Write;
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
@@ -22,20 +22,6 @@ pub enum Error {
 
 pub type Result<T> = std::result::Result<T, Error>;
 
-impl Deref for File {
-    type Target = NamedTempFile;
-
-    fn deref(&self) -> &Self::Target {
-        &self.file
-    }
-}
-
-impl DerefMut for File {
-    fn deref_mut(&mut self) -> &mut Self::Target {
-        &mut self.file
-    }
-}
-
 #[derive(Clone, Debug)]
 pub struct FileStore {
     path: PathBuf,
@@ -56,7 +42,7 @@ impl FileStore {
         let file = NamedTempFile::new_in(&self.path)?;
         let uuid = Uuid::new_v4();
         let path = self.path.join(uuid.to_string());
-        let update_file = File { file, path };
+        let update_file = File { file: Some(file), path };
 
         Ok((uuid, update_file))
     }
@@ -67,7 +53,7 @@ impl FileStore {
         let file = NamedTempFile::new_in(&self.path)?;
         let uuid = Uuid::from_u128(uuid);
         let path = self.path.join(uuid.to_string());
-        let update_file = File { file, path };
+        let update_file = File { file: Some(file), path };
 
         Ok((uuid, update_file))
     }
@@ -75,7 +61,13 @@ impl FileStore {
     /// Returns the file corresponding to the requested uuid.
     pub fn get_update(&self, uuid: Uuid) -> Result<StdFile> {
         let path = self.get_update_path(uuid);
-        let file = StdFile::open(path)?;
+        let file = match StdFile::open(path) {
+            Ok(file) => file,
+            Err(e) => {
+                tracing::error!("Can't access update file {uuid}: {e}");
+                return Err(e.into());
+            }
+        };
         Ok(file)
     }
 
@@ -110,8 +102,12 @@ impl FileStore {
 
     pub fn delete(&self, uuid: Uuid) -> Result<()> {
         let path = self.path.join(uuid.to_string());
-        std::fs::remove_file(path)?;
-        Ok(())
+        if let Err(e) = std::fs::remove_file(path) {
+            tracing::error!("Can't delete file {uuid}: {e}");
+            Err(e.into())
+        } else {
+            Ok(())
+        }
     }
 
     /// List the Uuids of the files in the FileStore
@@ -136,16 +132,40 @@ impl FileStore {
 
 pub struct File {
     path: PathBuf,
-    file: NamedTempFile,
+    file: Option<NamedTempFile>,
 }
 
 impl File {
+    pub fn dry_file() -> Result<Self> {
+        Ok(Self { path: PathBuf::new(), file: None })
+    }
+
     pub fn persist(self) -> Result<()> {
-        self.file.persist(&self.path)?;
+        if let Some(file) = self.file {
+            file.persist(&self.path)?;
+        }
         Ok(())
     }
 }
 
+impl Write for File {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        if let Some(file) = self.file.as_mut() {
+            file.write(buf)
+        } else {
+            Ok(buf.len())
+        }
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        if let Some(file) = self.file.as_mut() {
+            file.flush()
+        } else {
+            Ok(())
+        }
+    }
+}
+
 #[cfg(test)]
 mod test {
     use std::io::Write;

filter-parser/Cargo.toml

@@ -13,8 +13,8 @@ license.workspace = true
 
 [dependencies]
 nom = "7.1.3"
-nom_locate = "4.1.0"
-unescaper = "0.1.2"
+nom_locate = "4.2.0"
+unescaper = "0.1.3"
 
 [dev-dependencies]
-insta = "1.29.0"
+insta = "1.34.0"

fuzzers/Cargo.toml

@@ -11,10 +11,10 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-arbitrary = { version = "1.3.0", features = ["derive"] }
-clap = { version = "4.3.0", features = ["derive"] }
-fastrand = "2.0.0"
+arbitrary = { version = "1.3.2", features = ["derive"] }
+clap = { version = "4.4.17", features = ["derive"] }
+fastrand = "2.0.1"
 milli = { path = "../milli" }
-serde = { version = "1.0.160", features = ["derive"] }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
-tempfile = "3.5.0"
+serde = { version = "1.0.195", features = ["derive"] }
+serde_json = { version = "1.0.111", features = ["preserve_order"] }
+tempfile = "3.9.0"

index-scheduler/Cargo.toml

@@ -11,32 +11,37 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.70"
+anyhow = "1.0.79"
 bincode = "1.3.3"
-csv = "1.2.1"
+csv = "1.3.0"
 derive_builder = "0.12.0"
 dump = { path = "../dump" }
-enum-iterator = "1.4.0"
+enum-iterator = "1.5.0"
 file-store = { path = "../file-store" }
 flate2 = "1.0.28"
-log = "0.4.17"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
 page_size = "0.5.0"
 puffin = { version = "0.16.0", features = ["serialization"] }
-roaring = { version = "0.10.1", features = ["serde"] }
-serde = { version = "1.0.160", features = ["derive"] }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
+rayon = "1.8.1"
+roaring = { version = "0.10.2", features = ["serde"] }
+serde = { version = "1.0.195", features = ["derive"] }
+serde_json = { version = "1.0.111", features = ["preserve_order"] }
 synchronoise = "1.0.1"
-tempfile = "3.5.0"
-thiserror = "1.0.40"
-time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+tempfile = "3.9.0"
+thiserror = "1.0.56"
+time = { version = "0.3.31", features = [
+    "serde-well-known",
+    "formatting",
+    "parsing",
+    "macros",
+] }
+tracing = "0.1.40"
 ureq = "2.9.1"
-uuid = { version = "1.3.1", features = ["serde", "v4"] }
+uuid = { version = "1.6.1", features = ["serde", "v4"] }
 
 [dev-dependencies]
 big_s = "1.0.2"
-crossbeam = "0.8.2"
-insta = { version = "1.29.0", features = ["json", "redactions"] }
+crossbeam = "0.8.4"
+insta = { version = "1.34.0", features = ["json", "redactions"] }
 meili-snap = { path = "../meili-snap" }
-nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"}

index-scheduler/src/batch.rs

@@ -24,7 +24,6 @@ use std::fs::{self, File};
 use std::io::BufWriter;
 
 use dump::IndexMetadata;
-use log::{debug, error, info, trace};
 use meilisearch_types::error::Code;
 use meilisearch_types::heed::{RoTxn, RwTxn};
 use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader};
@@ -60,7 +59,7 @@ pub(crate) enum Batch {
         /// The list of tasks that were processing when this task cancelation appeared.
         previous_processing_tasks: RoaringBitmap,
     },
-    TaskDeletion(Task),
+    TaskDeletions(Vec<Task>),
     SnapshotCreation(Vec<Task>),
     Dump(Task),
     IndexOperation {
@@ -143,23 +142,28 @@ pub(crate) enum IndexOperation {
 
 impl Batch {
     /// Return the task ids associated with this batch.
-    pub fn ids(&self) -> Vec<TaskId> {
+    pub fn ids(&self) -> RoaringBitmap {
         match self {
             Batch::TaskCancelation { task, .. }
-            | Batch::TaskDeletion(task)
             | Batch::Dump(task)
             | Batch::IndexCreation { task, .. }
-            | Batch::IndexUpdate { task, .. } => vec![task.uid],
-            Batch::SnapshotCreation(tasks) | Batch::IndexDeletion { tasks, .. } => {
-                tasks.iter().map(|task| task.uid).collect()
+            | Batch::IndexUpdate { task, .. } => {
+                RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
+            }
+            Batch::SnapshotCreation(tasks)
+            | Batch::TaskDeletions(tasks)
+            | Batch::IndexDeletion { tasks, .. } => {
+                RoaringBitmap::from_iter(tasks.iter().map(|task| task.uid))
             }
             Batch::IndexOperation { op, .. } => match op {
                 IndexOperation::DocumentOperation { tasks, .. }
                 | IndexOperation::Settings { tasks, .. }
                 | IndexOperation::DocumentClear { tasks, .. } => {
-                    tasks.iter().map(|task| task.uid).collect()
+                    RoaringBitmap::from_iter(tasks.iter().map(|task| task.uid))
+                }
+                IndexOperation::IndexDocumentDeletionByFilter { task, .. } => {
+                    RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
                 }
-                IndexOperation::IndexDocumentDeletionByFilter { task, .. } => vec![task.uid],
                 IndexOperation::SettingsAndDocumentOperation {
                     document_import_tasks: tasks,
                     settings_tasks: other,
@@ -169,9 +173,11 @@ impl Batch {
                     cleared_tasks: tasks,
                     settings_tasks: other,
                     ..
-                } => tasks.iter().chain(other).map(|task| task.uid).collect(),
+                } => RoaringBitmap::from_iter(tasks.iter().chain(other).map(|task| task.uid)),
             },
-            Batch::IndexSwap { task } => vec![task.uid],
+            Batch::IndexSwap { task } => {
+                RoaringBitmap::from_sorted_iter(std::iter::once(task.uid)).unwrap()
+            }
         }
     }
 
@@ -180,7 +186,7 @@ impl Batch {
         use Batch::*;
         match self {
             TaskCancelation { .. }
-            | TaskDeletion(_)
+            | TaskDeletions(_)
             | SnapshotCreation(_)
             | Dump(_)
             | IndexSwap { .. } => None,
@@ -199,7 +205,7 @@ impl fmt::Display for Batch {
         let tasks = self.ids();
         match self {
             Batch::TaskCancelation { .. } => f.write_str("TaskCancelation")?,
-            Batch::TaskDeletion(_) => f.write_str("TaskDeletion")?,
+            Batch::TaskDeletions(_) => f.write_str("TaskDeletion")?,
             Batch::SnapshotCreation(_) => f.write_str("SnapshotCreation")?,
             Batch::Dump(_) => f.write_str("Dump")?,
             Batch::IndexOperation { op, .. } => write!(f, "{op}")?,
@@ -515,6 +521,7 @@ impl IndexScheduler {
     /// 3. We get the *next* snapshot to process.
     /// 4. We get the *next* dump to process.
     /// 5. We get the *next* tasks to process for a specific index.
+    #[tracing::instrument(level = "trace", skip(self, rtxn), target = "indexing::scheduler")]
     pub(crate) fn create_next_batch(&self, rtxn: &RoTxn) -> Result<Option<Batch>> {
         #[cfg(test)]
         self.maybe_fail(crate::tests::FailureLocation::InsideCreateBatch)?;
@@ -539,9 +546,9 @@ impl IndexScheduler {
 
         // 2. we get the next task to delete
         let to_delete = self.get_kind(rtxn, Kind::TaskDeletion)? & enqueued;
-        if let Some(task_id) = to_delete.min() {
-            let task = self.get_task(rtxn, task_id)?.ok_or(Error::CorruptedTaskQueue)?;
-            return Ok(Some(Batch::TaskDeletion(task)));
+        if !to_delete.is_empty() {
+            let tasks = self.get_existing_tasks(rtxn, to_delete)?;
+            return Ok(Some(Batch::TaskDeletions(tasks)));
         }
 
         // 3. we batch the snapshot.
@@ -620,6 +627,7 @@ impl IndexScheduler {
     /// The list of tasks that were processed. The metadata of each task in the returned
     /// list is updated accordingly, with the exception of the its date fields
     /// [`finished_at`](meilisearch_types::tasks::Task::finished_at) and [`started_at`](meilisearch_types::tasks::Task::started_at).
+    #[tracing::instrument(level = "trace", skip(self, batch), target = "indexing::scheduler", fields(batch=batch.to_string()))]
     pub(crate) fn process_batch(&self, batch: Batch) -> Result<Vec<Task>> {
         #[cfg(test)]
         {
@@ -669,9 +677,10 @@ impl IndexScheduler {
                     Ok(()) => {
                         for content_uuid in canceled_tasks_content_uuids {
                             if let Err(error) = self.delete_update_file(content_uuid) {
-                                error!(
-                                    "We failed deleting the content file indentified as {}: {}",
-                                    content_uuid, error
+                                tracing::error!(
+                                    file_content_uuid = %content_uuid,
+                                    %error,
+                                    "Failed deleting content file"
                                 )
                             }
                         }
@@ -681,31 +690,43 @@ impl IndexScheduler {
 
                 Ok(vec![task])
             }
-            Batch::TaskDeletion(mut task) => {
+            Batch::TaskDeletions(mut tasks) => {
                 // 1. Retrieve the tasks that matched the query at enqueue-time.
-                let matched_tasks =
+                let mut matched_tasks = RoaringBitmap::new();
+
+                for task in tasks.iter() {
                     if let KindWithContent::TaskDeletion { tasks, query: _ } = &task.kind {
-                        tasks
+                        matched_tasks |= tasks;
                     } else {
                         unreachable!()
+                    }
+                }
+
+                let mut wtxn = self.env.write_txn()?;
+                let mut deleted_tasks = self.delete_matched_tasks(&mut wtxn, &matched_tasks)?;
+                wtxn.commit()?;
+
+                for task in tasks.iter_mut() {
+                    task.status = Status::Succeeded;
+                    let KindWithContent::TaskDeletion { tasks, query: _ } = &task.kind else {
+                        unreachable!()
                     };
 
-                let mut wtxn = self.env.write_txn()?;
-                let deleted_tasks_count = self.delete_matched_tasks(&mut wtxn, matched_tasks)?;
+                    let deleted_tasks_count = deleted_tasks.intersection_len(tasks);
+                    deleted_tasks -= tasks;
 
-                task.status = Status::Succeeded;
-                match &mut task.details {
-                    Some(Details::TaskDeletion {
-                        matched_tasks: _,
-                        deleted_tasks,
-                        original_filter: _,
-                    }) => {
-                        *deleted_tasks = Some(deleted_tasks_count);
+                    match &mut task.details {
+                        Some(Details::TaskDeletion {
+                            matched_tasks: _,
+                            deleted_tasks,
+                            original_filter: _,
+                        }) => {
+                            *deleted_tasks = Some(deleted_tasks_count);
+                        }
+                        _ => unreachable!(),
                     }
-                    _ => unreachable!(),
                 }
-                wtxn.commit()?;
-                Ok(vec![task])
+                Ok(tasks)
             }
             Batch::SnapshotCreation(mut tasks) => {
                 fs::create_dir_all(&self.snapshots_path)?;
@@ -941,6 +962,9 @@ impl IndexScheduler {
 
                 let mut index_wtxn = index.write_txn()?;
                 let tasks = self.apply_index_operation(&mut index_wtxn, &index, op)?;
+
+                index.check_document_facet_consistency(&index_wtxn)?.check();
+
                 index_wtxn.commit()?;
 
                 // if the update processed successfully, we're going to store the new
@@ -958,7 +982,10 @@ impl IndexScheduler {
 
                 match res {
                     Ok(_) => (),
-                    Err(e) => error!("Could not write the stats of the index {}", e),
+                    Err(e) => tracing::error!(
+                        error = &e as &dyn std::error::Error,
+                        "Could not write the stats of the index"
+                    ),
                 }
 
                 Ok(tasks)
@@ -986,7 +1013,7 @@ impl IndexScheduler {
                     builder.set_primary_key(primary_key);
                     let must_stop_processing = self.must_stop_processing.clone();
                     builder.execute(
-                        |indexing_step| debug!("update: {:?}", indexing_step),
+                        |indexing_step| tracing::debug!(update = ?indexing_step),
                         || must_stop_processing.get(),
                     )?;
                     index_wtxn.commit()?;
@@ -1013,7 +1040,10 @@ impl IndexScheduler {
 
                 match res {
                     Ok(_) => (),
-                    Err(e) => error!("Could not write the stats of the index {}", e),
+                    Err(e) => tracing::error!(
+                        error = &e as &dyn std::error::Error,
+                        "Could not write the stats of the index"
+                    ),
                 }
 
                 Ok(vec![task])
@@ -1132,6 +1162,11 @@ impl IndexScheduler {
     ///
     /// ## Return
     /// The list of processed tasks.
+    #[tracing::instrument(
+        level = "trace",
+        skip(self, index_wtxn, index),
+        target = "indexing::scheduler"
+    )]
     fn apply_index_operation<'i>(
         &self,
         index_wtxn: &mut RwTxn<'i>,
@@ -1192,7 +1227,7 @@ impl IndexScheduler {
                                 milli::update::Settings::new(index_wtxn, index, indexer_config);
                             builder.set_primary_key(primary_key);
                             builder.execute(
-                                |indexing_step| debug!("update: {:?}", indexing_step),
+                                |indexing_step| tracing::debug!(update = ?indexing_step),
                                 || must_stop_processing.clone().get(),
                             )?;
                             primary_key_has_been_set = true;
@@ -1211,7 +1246,7 @@ impl IndexScheduler {
                     index,
                     indexer_config,
                     config,
-                    |indexing_step| trace!("update: {:?}", indexing_step),
+                    |indexing_step| tracing::trace!(?indexing_step, "Update"),
                     || must_stop_processing.get(),
                 )?;
 
@@ -1283,7 +1318,7 @@ impl IndexScheduler {
 
                 if !tasks.iter().all(|res| res.error.is_some()) {
                     let addition = builder.execute()?;
-                    info!("document addition done: {:?}", addition);
+                    tracing::info!(indexing_result = ?addition, "document indexing done");
                 } else if primary_key_has_been_set {
                     // Everything failed but we've set a primary key.
                     // We need to remove it.
@@ -1291,7 +1326,7 @@ impl IndexScheduler {
                         milli::update::Settings::new(index_wtxn, index, indexer_config);
                     builder.reset_primary_key();
                     builder.execute(
-                        |indexing_step| trace!("update: {:?}", indexing_step),
+                        |indexing_step| tracing::trace!(update = ?indexing_step),
                         || must_stop_processing.clone().get(),
                     )?;
                 }
@@ -1307,6 +1342,7 @@ impl IndexScheduler {
                     } else {
                         unreachable!()
                     };
+
                 let deleted_documents = delete_document_by_filter(
                     index_wtxn,
                     filter,
@@ -1361,7 +1397,7 @@ impl IndexScheduler {
 
                 let must_stop_processing = self.must_stop_processing.clone();
                 builder.execute(
-                    |indexing_step| debug!("update: {:?}", indexing_step),
+                    |indexing_step| tracing::debug!(update = ?indexing_step),
                     || must_stop_processing.get(),
                 )?;
 
@@ -1435,7 +1471,11 @@ impl IndexScheduler {
     /// Delete each given task from all the databases (if it is deleteable).
     ///
     /// Return the number of tasks that were actually deleted.
-    fn delete_matched_tasks(&self, wtxn: &mut RwTxn, matched_tasks: &RoaringBitmap) -> Result<u64> {
+    fn delete_matched_tasks(
+        &self,
+        wtxn: &mut RwTxn,
+        matched_tasks: &RoaringBitmap,
+    ) -> Result<RoaringBitmap> {
         // 1. Remove from this list the tasks that we are not allowed to delete
         let enqueued_tasks = self.get_status(wtxn, Status::Enqueued)?;
         let processing_tasks = &self.processing_tasks.read().unwrap().processing.clone();
@@ -1500,7 +1540,7 @@ impl IndexScheduler {
             }
         }
 
-        Ok(to_delete_tasks.len())
+        Ok(to_delete_tasks)
     }
 
     /// Cancel each given task from all the databases (if it is cancelable).
@@ -1569,7 +1609,7 @@ fn delete_document_by_filter<'a>(
             index,
             indexer_config,
             config,
-            |indexing_step| debug!("update: {:?}", indexing_step),
+            |indexing_step| tracing::debug!(update = ?indexing_step),
             || must_stop_processing.get(),
         )?;
 

index-scheduler/src/error.rs

@@ -48,6 +48,8 @@ impl From<DateField> for Code {
 pub enum Error {
     #[error("{1}")]
     WithCustomErrorCode(Code, Box<Self>),
+    #[error("Received bad task id: {received} should be >= to {expected}.")]
+    BadTaskId { received: TaskId, expected: TaskId },
     #[error("Index `{0}` not found.")]
     IndexNotFound(String),
     #[error("Index `{0}` already exists.")]
@@ -161,6 +163,7 @@ impl Error {
         match self {
             Error::IndexNotFound(_)
             | Error::WithCustomErrorCode(_, _)
+            | Error::BadTaskId { .. }
             | Error::IndexAlreadyExists(_)
             | Error::SwapDuplicateIndexFound(_)
             | Error::SwapDuplicateIndexesFound(_)
@@ -205,6 +208,7 @@ impl ErrorCode for Error {
     fn error_code(&self) -> Code {
         match self {
             Error::WithCustomErrorCode(code, _) => *code,
+            Error::BadTaskId { .. } => Code::BadRequest,
             Error::IndexNotFound(_) => Code::IndexNotFound,
             Error::IndexAlreadyExists(_) => Code::IndexAlreadyExists,
             Error::SwapDuplicateIndexesFound(_) => Code::InvalidSwapDuplicateIndexFound,

index-scheduler/src/features.rs

@@ -30,19 +30,6 @@ impl RoFeatures {
         self.runtime
     }
 
-    pub fn check_score_details(&self) -> Result<()> {
-        if self.runtime.score_details {
-            Ok(())
-        } else {
-            Err(FeatureNotEnabledError {
-                disabled_action: "Computing score details",
-                feature: "score details",
-                issue_link: "https://github.com/meilisearch/product/discussions/674",
-            }
-            .into())
-        }
-    }
-
     pub fn check_metrics(&self) -> Result<()> {
         if self.runtime.metrics {
             Ok(())
@@ -56,6 +43,19 @@ impl RoFeatures {
         }
     }
 
+    pub fn check_logs_route(&self) -> Result<()> {
+        if self.runtime.logs_route {
+            Ok(())
+        } else {
+            Err(FeatureNotEnabledError {
+                disabled_action: "Modifying logs through the `/logs/*` routes",
+                feature: "logs route",
+                issue_link: "https://github.com/orgs/meilisearch/discussions/721",
+            }
+            .into())
+        }
+    }
+
     pub fn check_vector(&self, disabled_action: &'static str) -> Result<()> {
         if self.runtime.vector_store {
             Ok(())
@@ -94,6 +94,7 @@ impl FeatureData {
             runtime_features_db.get(&txn, EXPERIMENTAL_FEATURES)?.unwrap_or_default();
         let runtime = Arc::new(RwLock::new(RuntimeTogglableFeatures {
             metrics: instance_features.metrics || persisted_features.metrics,
+            logs_route: instance_features.logs_route || persisted_features.logs_route,
             ..persisted_features
         }));
 

index-scheduler/src/index_mapper/mod.rs

@@ -3,13 +3,13 @@ use std::sync::{Arc, RwLock};
 use std::time::Duration;
 use std::{fs, thread};
 
-use log::error;
 use meilisearch_types::heed::types::{SerdeJson, Str};
 use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn};
 use meilisearch_types::milli::update::IndexerConfig;
 use meilisearch_types::milli::{FieldDistribution, Index};
 use serde::{Deserialize, Serialize};
 use time::OffsetDateTime;
+use tracing::error;
 use uuid::Uuid;
 
 use self::index_map::IndexMap;

index-scheduler/src/insta_snapshot.rs

@@ -15,6 +15,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
 
     let IndexScheduler {
         autobatching_enabled,
+        cleanup_enabled: _,
         must_stop_processing: _,
         processing_tasks,
         file_store,

index-scheduler/src/snapshots/lib.rs/task_deletion_delete_same_task_twice/task_deletion_processed.snap

@@ -34,12 +34,10 @@ catto: { number_of_documents: 1, field_distribution: {"id": 1} }
 [timestamp] [3,]
 ----------------------------------------------------------------------
 ### Started At:
-[timestamp] [2,]
-[timestamp] [3,]
+[timestamp] [2,3,]
 ----------------------------------------------------------------------
 ### Finished At:
-[timestamp] [2,]
-[timestamp] [3,]
+[timestamp] [2,3,]
 ----------------------------------------------------------------------
 ### File Store:
 00000000-0000-0000-0000-000000000001

index-scheduler/src/snapshots/lib.rs/test_disable_auto_deletion_of_tasks/task_deletion_have_not_been_enqueued.snap

@@ -0,0 +1,90 @@
+---
+source: index-scheduler/src/lib.rs
+---
+[
+  {
+    "uid": 0,
+    "enqueuedAt": "[date]",
+    "startedAt": "[date]",
+    "finishedAt": "[date]",
+    "error": null,
+    "canceledBy": null,
+    "details": {
+      "IndexInfo": {
+        "primary_key": null
+      }
+    },
+    "status": "succeeded",
+    "kind": {
+      "indexCreation": {
+        "index_uid": "doggo",
+        "primary_key": null
+      }
+    }
+  },
+  {
+    "uid": 1,
+    "enqueuedAt": "[date]",
+    "startedAt": "[date]",
+    "finishedAt": "[date]",
+    "error": {
+      "message": "Index `doggo` already exists.",
+      "code": "index_already_exists",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#index_already_exists"
+    },
+    "canceledBy": null,
+    "details": {
+      "IndexInfo": {
+        "primary_key": null
+      }
+    },
+    "status": "failed",
+    "kind": {
+      "indexCreation": {
+        "index_uid": "doggo",
+        "primary_key": null
+      }
+    }
+  },
+  {
+    "uid": 2,
+    "enqueuedAt": "[date]",
+    "startedAt": "[date]",
+    "finishedAt": "[date]",
+    "error": null,
+    "canceledBy": null,
+    "details": {
+      "IndexInfo": {
+        "primary_key": null
+      }
+    },
+    "status": "enqueued",
+    "kind": {
+      "indexCreation": {
+        "index_uid": "doggo",
+        "primary_key": null
+      }
+    }
+  },
+  {
+    "uid": 3,
+    "enqueuedAt": "[date]",
+    "startedAt": "[date]",
+    "finishedAt": "[date]",
+    "error": null,
+    "canceledBy": null,
+    "details": {
+      "IndexInfo": {
+        "primary_key": null
+      }
+    },
+    "status": "enqueued",
+    "kind": {
+      "indexCreation": {
+        "index_uid": "doggo",
+        "primary_key": null
+      }
+    }
+  }
+]

index-scheduler/src/snapshots/lib.rs/test_disable_auto_deletion_of_tasks/task_queue_is_full.snap

@@ -0,0 +1,90 @@
+---
+source: index-scheduler/src/lib.rs
+---
+[
+  {
+    "uid": 0,
+    "enqueuedAt": "[date]",
+    "startedAt": "[date]",
+    "finishedAt": "[date]",
+    "error": null,
+    "canceledBy": null,
+    "details": {
+      "IndexInfo": {
+        "primary_key": null
+      }
+    },
+    "status": "succeeded",
+    "kind": {
+      "indexCreation": {
+        "index_uid": "doggo",
+        "primary_key": null
+      }
+    }
+  },
+  {
+    "uid": 1,
+    "enqueuedAt": "[date]",
+    "startedAt": "[date]",
+    "finishedAt": "[date]",
+    "error": {
+      "message": "Index `doggo` already exists.",
+      "code": "index_already_exists",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#index_already_exists"
+    },
+    "canceledBy": null,
+    "details": {
+      "IndexInfo": {
+        "primary_key": null
+      }
+    },
+    "status": "failed",
+    "kind": {
+      "indexCreation": {
+        "index_uid": "doggo",
+        "primary_key": null
+      }
+    }
+  },
+  {
+    "uid": 2,
+    "enqueuedAt": "[date]",
+    "startedAt": "[date]",
+    "finishedAt": "[date]",
+    "error": null,
+    "canceledBy": null,
+    "details": {
+      "IndexInfo": {
+        "primary_key": null
+      }
+    },
+    "status": "enqueued",
+    "kind": {
+      "indexCreation": {
+        "index_uid": "doggo",
+        "primary_key": null
+      }
+    }
+  },
+  {
+    "uid": 3,
+    "enqueuedAt": "[date]",
+    "startedAt": "[date]",
+    "finishedAt": "[date]",
+    "error": null,
+    "canceledBy": null,
+    "details": {
+      "IndexInfo": {
+        "primary_key": null
+      }
+    },
+    "status": "enqueued",
+    "kind": {
+      "indexCreation": {
+        "index_uid": "doggo",
+        "primary_key": null
+      }
+    }
+  }
+]

index-scheduler/src/uuid_codec.rs

@@ -1,5 +1,4 @@
 use std::borrow::Cow;
-use std::convert::TryInto;
 
 use meilisearch_types::heed::{BoxedError, BytesDecode, BytesEncode};
 use uuid::Uuid;

meili-snap/Cargo.toml

@@ -11,6 +11,6 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-insta = { version = "^1.29.0", features = ["json", "redactions"] }
+insta = { version = "^1.34.0", features = ["json", "redactions"] }
 md5 = "0.7.0"
-once_cell = "1.17"
+once_cell = "1.19"

meilisearch-auth/Cargo.toml

@@ -11,16 +11,16 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-base64 = "0.21.0"
-enum-iterator = "1.4.0"
+base64 = "0.21.7"
+enum-iterator = "1.5.0"
 hmac = "0.12.1"
 maplit = "1.0.2"
 meilisearch-types = { path = "../meilisearch-types" }
 rand = "0.8.5"
-roaring = { version = "0.10.1", features = ["serde"] }
-serde = { version = "1.0.160", features = ["derive"] }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
-sha2 = "0.10.6"
-thiserror = "1.0.40"
-time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-uuid = { version = "1.3.1", features = ["serde", "v4"] }
+roaring = { version = "0.10.2", features = ["serde"] }
+serde = { version = "1.0.195", features = ["derive"] }
+serde_json = { version = "1.0.111", features = ["preserve_order"] }
+sha2 = "0.10.8"
+thiserror = "1.0.56"
+time = { version = "0.3.31", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+uuid = { version = "1.6.1", features = ["serde", "v4"] }

meilisearch-auth/src/store.rs

@@ -1,7 +1,6 @@
 use std::borrow::Cow;
 use std::cmp::Reverse;
 use std::collections::HashSet;
-use std::convert::{TryFrom, TryInto};
 use std::fs::create_dir_all;
 use std::path::Path;
 use std::result::Result as StdResult;

meilisearch-types/Cargo.toml

@@ -11,31 +11,31 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-actix-web = { version = "4.3.1", default-features = false }
-anyhow = "1.0.70"
+actix-web = { version = "4.4.1", default-features = false }
+anyhow = "1.0.79"
 convert_case = "0.6.0"
-csv = "1.2.1"
-deserr = { version = "0.6.0", features = ["actix-web"] }
-either = { version = "1.8.1", features = ["serde"] }
-enum-iterator = "1.4.0"
+csv = "1.3.0"
+deserr = { version = "0.6.1", features = ["actix-web"] }
+either = { version = "1.9.0", features = ["serde"] }
+enum-iterator = "1.5.0"
 file-store = { path = "../file-store" }
-flate2 = "1.0.25"
+flate2 = "1.0.28"
 fst = "0.4.7"
 memmap2 = "0.7.1"
 milli = { path = "../milli" }
-roaring = { version = "0.10.1", features = ["serde"] }
-serde = { version = "1.0.160", features = ["derive"] }
+roaring = { version = "0.10.2", features = ["serde"] }
+serde = { version = "1.0.195", features = ["derive"] }
 serde-cs = "0.2.4"
-serde_json = "1.0.95"
-tar = "0.4.38"
-tempfile = "3.5.0"
-thiserror = "1.0.40"
-time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-tokio = "1.27"
-uuid = { version = "1.3.1", features = ["serde", "v4"] }
+serde_json = "1.0.111"
+tar = "0.4.40"
+tempfile = "3.9.0"
+thiserror = "1.0.56"
+time = { version = "0.3.31", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+tokio = "1.35"
+uuid = { version = "1.6.1", features = ["serde", "v4"] }
 
 [dev-dependencies]
-insta = "1.29.0"
+insta = "1.34.0"
 meili-snap = { path = "../meili-snap" }
 
 [features]
@@ -54,3 +54,5 @@ thai = ["milli/thai"]
 greek = ["milli/greek"]
 # allow khmer specialized tokenization
 khmer = ["milli/khmer"]
+# allow vietnamese specialized tokenization
+vietnamese = ["milli/vietnamese"]

meilisearch-types/src/document_formats.rs

@@ -1,6 +1,6 @@
 use std::fmt::{self, Debug, Display};
 use std::fs::File;
-use std::io::{self, Seek, Write};
+use std::io::{self, BufWriter, Write};
 use std::marker::PhantomData;
 
 use memmap2::MmapOptions;
@@ -104,8 +104,8 @@ impl ErrorCode for DocumentFormatError {
 }
 
 /// Reads CSV from input and write an obkv batch to writer.
-pub fn read_csv(file: &File, writer: impl Write + Seek, delimiter: u8) -> Result<u64> {
-    let mut builder = DocumentsBatchBuilder::new(writer);
+pub fn read_csv(file: &File, writer: impl Write, delimiter: u8) -> Result<u64> {
+    let mut builder = DocumentsBatchBuilder::new(BufWriter::new(writer));
     let mmap = unsafe { MmapOptions::new().map(file)? };
     let csv = csv::ReaderBuilder::new().delimiter(delimiter).from_reader(mmap.as_ref());
     builder.append_csv(csv).map_err(|e| (PayloadType::Csv { delimiter }, e))?;
@@ -116,9 +116,9 @@ pub fn read_csv(file: &File, writer: impl Write + Seek, delimiter: u8) -> Result
     Ok(count as u64)
 }
 
-/// Reads JSON from temporary file  and write an obkv batch to writer.
-pub fn read_json(file: &File, writer: impl Write + Seek) -> Result<u64> {
-    let mut builder = DocumentsBatchBuilder::new(writer);
+/// Reads JSON from temporary file and write an obkv batch to writer.
+pub fn read_json(file: &File, writer: impl Write) -> Result<u64> {
+    let mut builder = DocumentsBatchBuilder::new(BufWriter::new(writer));
     let mmap = unsafe { MmapOptions::new().map(file)? };
     let mut deserializer = serde_json::Deserializer::from_slice(&mmap);
 
@@ -151,8 +151,8 @@ pub fn read_json(file: &File, writer: impl Write + Seek) -> Result<u64> {
 }
 
 /// Reads JSON from temporary file  and write an obkv batch to writer.
-pub fn read_ndjson(file: &File, writer: impl Write + Seek) -> Result<u64> {
-    let mut builder = DocumentsBatchBuilder::new(writer);
+pub fn read_ndjson(file: &File, writer: impl Write) -> Result<u64> {
+    let mut builder = DocumentsBatchBuilder::new(BufWriter::new(writer));
     let mmap = unsafe { MmapOptions::new().map(file)? };
 
     for result in serde_json::Deserializer::from_slice(&mmap).into_iter() {

meilisearch-types/src/error.rs

@@ -310,6 +310,8 @@ TooManyVectors                        , InvalidRequest       , BAD_REQUEST ;
 UnretrievableDocument                 , Internal             , BAD_REQUEST ;
 UnretrievableErrorCode                , InvalidRequest       , BAD_REQUEST ;
 UnsupportedMediaType                  , InvalidRequest       , UNSUPPORTED_MEDIA_TYPE ;
+
+// Experimental features
 VectorEmbeddingError                  , InvalidRequest       , BAD_REQUEST
 }
 
@@ -347,6 +349,9 @@ impl ErrorCode for milli::Error {
                     UserError::InvalidFieldForSource { .. }
                     | UserError::MissingFieldForSource { .. }
                     | UserError::InvalidOpenAiModel { .. }
+                    | UserError::InvalidOpenAiModelDimensions { .. }
+                    | UserError::InvalidOpenAiModelDimensionsMax { .. }
+                    | UserError::InvalidSettingsDimensions { .. }
                     | UserError::InvalidPrompt(_) => Code::InvalidSettingsEmbedders,
                     UserError::TooManyEmbedders(_) => Code::InvalidSettingsEmbedders,
                     UserError::InvalidPromptForEmbeddings(..) => Code::InvalidSettingsEmbedders,

meilisearch-types/src/features.rs

@@ -3,13 +3,14 @@ use serde::{Deserialize, Serialize};
 #[derive(Serialize, Deserialize, Debug, Clone, Copy, Default, PartialEq, Eq)]
 #[serde(rename_all = "camelCase", default)]
 pub struct RuntimeTogglableFeatures {
-    pub score_details: bool,
     pub vector_store: bool,
     pub metrics: bool,
+    pub logs_route: bool,
     pub export_puffin_reports: bool,
 }
 
 #[derive(Default, Debug, Clone, Copy)]
 pub struct InstanceTogglableFeatures {
     pub metrics: bool,
+    pub logs_route: bool,
 }

meilisearch/Cargo.toml

@@ -13,14 +13,14 @@ license.workspace = true
 default-run = "meilisearch"
 
 [dependencies]
-actix-cors = "0.6.4"
-actix-http = { version = "3.3.1", default-features = false, features = [
+actix-cors = "0.7.0"
+actix-http = { version = "3.5.1", default-features = false, features = [
     "compress-brotli",
     "compress-gzip",
     "rustls",
 ] }
 actix-utils = "3.0.1"
-actix-web = { version = "4.3.1", default-features = false, features = [
+actix-web = { version = "4.4.1", default-features = false, features = [
     "macros",
     "compress-brotli",
     "compress-gzip",
@@ -28,109 +28,111 @@ actix-web = { version = "4.3.1", default-features = false, features = [
     "rustls",
 ] }
 actix-web-static-files = { git = "https://github.com/kilork/actix-web-static-files.git", rev = "2d3b6160", optional = true }
-anyhow = { version = "1.0.70", features = ["backtrace"] }
+anyhow = { version = "1.0.79", features = ["backtrace"] }
 async-stream = "0.3.5"
-async-trait = "0.1.68"
-bstr = "1.4.0"
+async-trait = "0.1.77"
+bstr = "1.9.0"
 byte-unit = { version = "4.0.19", default-features = false, features = [
     "std",
     "serde",
 ] }
-bytes = "1.4.0"
-clap = { version = "4.2.1", features = ["derive", "env"] }
-crossbeam-channel = "0.5.8"
-deserr = { version = "0.6.0", features = ["actix-web"] }
+bytes = "1.5.0"
+clap = { version = "4.4.17", features = ["derive", "env"] }
+crossbeam-channel = "0.5.11"
+deserr = { version = "0.6.1", features = ["actix-web"] }
 dump = { path = "../dump" }
-either = "1.8.1"
-env_logger = "0.10.0"
+either = "1.9.0"
 file-store = { path = "../file-store" }
-flate2 = "1.0.25"
+flate2 = "1.0.28"
 fst = "0.4.7"
-futures = "0.3.28"
-futures-util = "0.3.28"
-http = "0.2.9"
+futures = "0.3.30"
+futures-util = "0.3.30"
+http = "0.2.11"
 index-scheduler = { path = "../index-scheduler" }
-indexmap = { version = "2.0.0", features = ["serde"] }
-is-terminal = "0.4.8"
+indexmap = { version = "2.1.0", features = ["serde"] }
+is-terminal = "0.4.10"
 itertools = "0.11.0"
 jsonwebtoken = "8.3.0"
 lazy_static = "1.4.0"
-log = "0.4.17"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-mimalloc = { version = "0.1.37", default-features = false }
+mimalloc = { version = "0.1.39", default-features = false }
 mime = "0.3.17"
-num_cpus = "1.15.0"
-obkv = "0.2.0"
-once_cell = "1.17.1"
-ordered-float = "3.7.0"
+num_cpus = "1.16.0"
+obkv = "0.2.1"
+once_cell = "1.19.0"
+ordered-float = "4.2.0"
 parking_lot = "0.12.1"
 permissive-json-pointer = { path = "../permissive-json-pointer" }
-pin-project-lite = "0.2.9"
+pin-project-lite = "0.2.13"
 platform-dirs = "0.3.0"
 prometheus = { version = "0.13.3", features = ["process"] }
 puffin = { version = "0.16.0", features = ["serialization"] }
 rand = "0.8.5"
-rayon = "1.7.0"
-regex = "1.7.3"
-reqwest = { version = "0.11.16", features = [
+rayon = "1.8.0"
+regex = "1.10.2"
+reqwest = { version = "0.11.23", features = [
     "rustls-tls",
     "json",
 ], default-features = false }
 rustls = "0.20.8"
 rustls-pemfile = "1.0.2"
-segment = { version = "0.2.2", optional = true }
-serde = { version = "1.0.160", features = ["derive"] }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
-sha2 = "0.10.6"
-siphasher = "0.3.10"
-slice-group-by = "0.3.0"
+segment = { version = "0.2.3", optional = true }
+serde = { version = "1.0.195", features = ["derive"] }
+serde_json = { version = "1.0.111", features = ["preserve_order"] }
+sha2 = "0.10.8"
+siphasher = "1.0.0"
+slice-group-by = "0.3.1"
 static-files = { version = "0.2.3", optional = true }
-sysinfo = "0.29.7"
-tar = "0.4.38"
-tempfile = "3.5.0"
-thiserror = "1.0.40"
-time = { version = "0.3.20", features = [
+sysinfo = "0.30.5"
+tar = "0.4.40"
+tempfile = "3.9.0"
+thiserror = "1.0.56"
+time = { version = "0.3.31", features = [
     "serde-well-known",
     "formatting",
     "parsing",
     "macros",
 ] }
-tokio = { version = "1.27.0", features = ["full"] }
-tokio-stream = "0.1.12"
-toml = "0.7.3"
-uuid = { version = "1.3.1", features = ["serde", "v4"] }
-walkdir = "2.3.3"
+tokio = { version = "1.35.1", features = ["full"] }
+tokio-stream = "0.1.14"
+toml = "0.8.8"
+uuid = { version = "1.6.1", features = ["serde", "v4"] }
+walkdir = "2.4.0"
 yaup = "0.2.1"
 serde_urlencoded = "0.7.1"
-termcolor = "1.2.0"
+termcolor = "1.4.1"
 url = { version = "2.5.0", features = ["serde"] }
+tracing = "0.1.40"
+tracing-subscriber = { version = "0.3.18", features = ["json"] }
+tracing-trace = { version = "0.1.0", path = "../tracing-trace" }
+tracing-actix-web = "0.7.9"
+build-info = { version = "1.7.0", path = "../build-info" }
 
 [dev-dependencies]
-actix-rt = "2.8.0"
+actix-rt = "2.9.0"
 assert-json-diff = "2.0.2"
-brotli = "3.3.4"
-insta = "1.29.0"
-manifest-dir-macros = "0.1.16"
+brotli = "3.4.0"
+insta = "1.34.0"
+manifest-dir-macros = "0.1.18"
 maplit = "1.0.2"
 meili-snap = { path = "../meili-snap" }
-temp-env = "0.3.3"
-urlencoding = "2.1.2"
+temp-env = "0.3.6"
+urlencoding = "2.1.3"
 yaup = "0.2.1"
 
 [build-dependencies]
-anyhow = { version = "1.0.70", optional = true }
-cargo_toml = { version = "0.15.2", optional = true }
+anyhow = { version = "1.0.79", optional = true }
+cargo_toml = { version = "0.18.0", optional = true }
 hex = { version = "0.4.3", optional = true }
-reqwest = { version = "0.11.16", features = [
+reqwest = { version = "0.11.23", features = [
     "blocking",
     "rustls-tls",
 ], default-features = false, optional = true }
 sha-1 = { version = "0.10.1", optional = true }
 static-files = { version = "0.2.3", optional = true }
-tempfile = { version = "3.5.0", optional = true }
-vergen = { version = "7.5.1", default-features = false, features = ["git"] }
-zip = { version = "0.6.4", optional = true }
+tempfile = { version = "3.9.0", optional = true }
+zip = { version = "0.6.6", optional = true }
 
 [features]
 default = ["analytics", "meilisearch-types/all-tokenizations", "mini-dashboard"]
@@ -152,7 +154,8 @@ japanese = ["meilisearch-types/japanese"]
 thai = ["meilisearch-types/thai"]
 greek = ["meilisearch-types/greek"]
 khmer = ["meilisearch-types/khmer"]
+vietnamese = ["meilisearch-types/vietnamese"]
 
 [package.metadata.mini-dashboard]
-assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.12/build.zip"
-sha1 = "acfe9a018c93eb0604ea87ee87bff7df5474e18e"
+assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.13/build.zip"
+sha1 = "e20cc9b390003c6c844f4b8bcc5c5013191a77ff"

meilisearch/build.rs

@@ -1,17 +1,4 @@
-use vergen::{vergen, Config, SemverKind};
-
 fn main() {
-    // Note: any code that needs VERGEN_ environment variables should take care to define them manually in the Dockerfile and pass them
-    // in the corresponding GitHub workflow (publish_docker.yml).
-    // This is due to the Dockerfile building the binary outside of the git directory.
-    let mut config = Config::default();
-    // allow using non-annotated tags
-    *config.git_mut().semver_kind_mut() = SemverKind::Lightweight;
-
-    if let Err(e) = vergen(config) {
-        println!("cargo:warning=vergen: {}", e);
-    }
-
     #[cfg(feature = "mini-dashboard")]
     mini_dashboard::setup_mini_dashboard().expect("Could not load the mini-dashboard assets");
 }

meilisearch/src/analytics/segment_analytics.rs

@@ -18,7 +18,7 @@ use segment::message::{Identify, Track, User};
 use segment::{AutoBatcher, Batcher, HttpClient};
 use serde::Serialize;
 use serde_json::{json, Value};
-use sysinfo::{DiskExt, System, SystemExt};
+use sysinfo::{Disks, System};
 use time::OffsetDateTime;
 use tokio::select;
 use tokio::sync::mpsc::{self, Receiver, Sender};
@@ -28,7 +28,9 @@ use super::{
     config_user_id_path, DocumentDeletionKind, DocumentFetchKind, MEILISEARCH_CONFIG_PATH,
 };
 use crate::analytics::Analytics;
-use crate::option::{default_http_addr, IndexerOpts, MaxMemory, MaxThreads, ScheduleSnapshot};
+use crate::option::{
+    default_http_addr, IndexerOpts, LogMode, MaxMemory, MaxThreads, ScheduleSnapshot,
+};
 use crate::routes::indexes::documents::UpdateDocumentsQuery;
 use crate::routes::indexes::facet_search::FacetSearchQuery;
 use crate::routes::tasks::TasksFilterQuery;
@@ -250,8 +252,12 @@ impl super::Analytics for SegmentAnalytics {
 struct Infos {
     env: String,
     experimental_enable_metrics: bool,
+    experimental_logs_mode: LogMode,
+    experimental_replication_parameters: bool,
+    experimental_enable_logs_route: bool,
     experimental_reduce_indexing_memory_usage: bool,
     experimental_max_number_of_batched_tasks: usize,
+    gpu_enabled: bool,
     db_path: bool,
     import_dump: bool,
     dump_dir: bool,
@@ -287,6 +293,9 @@ impl From<Opt> for Infos {
         let Opt {
             db_path,
             experimental_enable_metrics,
+            experimental_logs_mode,
+            experimental_replication_parameters,
+            experimental_enable_logs_route,
             experimental_reduce_indexing_memory_usage,
             experimental_max_number_of_batched_tasks,
             http_addr,
@@ -333,7 +342,11 @@ impl From<Opt> for Infos {
         Self {
             env,
             experimental_enable_metrics,
+            experimental_logs_mode,
+            experimental_replication_parameters,
+            experimental_enable_logs_route,
             experimental_reduce_indexing_memory_usage,
+            gpu_enabled: meilisearch_types::milli::vector::is_cuda_enabled(),
             db_path: db_path != PathBuf::from("./data.ms"),
             import_dump: import_dump.is_some(),
             dump_dir: dump_dir != PathBuf::from("dumps/"),
@@ -386,16 +399,17 @@ impl Segment {
     fn compute_traits(opt: &Opt, stats: Stats) -> Value {
         static FIRST_START_TIMESTAMP: Lazy<Instant> = Lazy::new(Instant::now);
         static SYSTEM: Lazy<Value> = Lazy::new(|| {
+            let disks = Disks::new_with_refreshed_list();
             let mut sys = System::new_all();
             sys.refresh_all();
-            let kernel_version =
-                sys.kernel_version().and_then(|k| k.split_once('-').map(|(k, _)| k.to_string()));
+            let kernel_version = System::kernel_version()
+                .and_then(|k| k.split_once('-').map(|(k, _)| k.to_string()));
             json!({
-                    "distribution": sys.name(),
+                    "distribution": System::name(),
                     "kernel_version": kernel_version,
                     "cores": sys.cpus().len(),
                     "ram_size": sys.total_memory(),
-                    "disk_size": sys.disks().iter().map(|disk| disk.total_space()).max(),
+                    "disk_size": disks.iter().map(|disk| disk.total_space()).max(),
                     "server_provider": std::env::var("MEILI_SERVER_PROVIDER").ok(),
             })
         });
@@ -459,7 +473,9 @@ impl Segment {
             create_all_stats(index_scheduler.into(), auth_controller.into(), &AuthFilter::default())
         {
             // Replace the version number with the prototype name if any.
-            let version = if let Some(prototype) = crate::prototype_name() {
+            let version = if let Some(prototype) = build_info::DescribeResult::from_build()
+                .and_then(|describe| describe.as_prototype())
+            {
                 prototype
             } else {
                 env!("CARGO_PKG_VERSION")

meilisearch/src/error.rs

@@ -12,6 +12,8 @@ pub enum MeilisearchHttpError {
     #[error("A Content-Type header is missing. Accepted values for the Content-Type header are: {}",
             .0.iter().map(|s| format!("`{}`", s)).collect::<Vec<_>>().join(", "))]
     MissingContentType(Vec<String>),
+    #[error("The `/logs/stream` route is currently in use by someone else.")]
+    AlreadyUsedLogRoute,
     #[error("The Content-Type `{0}` does not support the use of a csv delimiter. The csv delimiter can only be used with the Content-Type `text/csv`.")]
     CsvDelimiterWithWrongContentType(String),
     #[error(
@@ -59,6 +61,7 @@ impl ErrorCode for MeilisearchHttpError {
     fn error_code(&self) -> Code {
         match self {
             MeilisearchHttpError::MissingContentType(_) => Code::MissingContentType,
+            MeilisearchHttpError::AlreadyUsedLogRoute => Code::BadRequest,
             MeilisearchHttpError::CsvDelimiterWithWrongContentType(_) => Code::InvalidContentType,
             MeilisearchHttpError::MissingPayload(_) => Code::MissingPayload,
             MeilisearchHttpError::InvalidContentType(_, _) => Code::InvalidContentType,

meilisearch/src/extractors/sequential_extractor.rs

@@ -131,6 +131,7 @@ gen_seq! { SeqFromRequestFut3; A B C }
 gen_seq! { SeqFromRequestFut4; A B C D }
 gen_seq! { SeqFromRequestFut5; A B C D E }
 gen_seq! { SeqFromRequestFut6; A B C D E F }
+gen_seq! { SeqFromRequestFut7; A B C D E F G }
 
 pin_project! {
     #[project = ExtractProj]

meilisearch/src/lib.rs

@@ -29,7 +29,6 @@ use error::PayloadError;
 use extractors::payload::PayloadConfig;
 use http::header::CONTENT_TYPE;
 use index_scheduler::{IndexScheduler, IndexSchedulerOptions};
-use log::error;
 use meilisearch_auth::AuthController;
 use meilisearch_types::milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
 use meilisearch_types::milli::update::{IndexDocumentsConfig, IndexDocumentsMethod};
@@ -39,6 +38,8 @@ use meilisearch_types::versioning::{check_version_file, create_version_file};
 use meilisearch_types::{compression, milli, VERSION_FILE_NAME};
 pub use option::Opt;
 use option::ScheduleSnapshot;
+use tracing::{error, info_span};
+use tracing_subscriber::filter::Targets;
 
 use crate::error::MeilisearchHttpError;
 
@@ -86,10 +87,35 @@ fn is_empty_db(db_path: impl AsRef<Path>) -> bool {
     }
 }
 
+/// The handle used to update the logs at runtime. Must be accessible from the `main.rs` and the `route/logs.rs`.
+pub type LogRouteHandle =
+    tracing_subscriber::reload::Handle<LogRouteType, tracing_subscriber::Registry>;
+
+pub type LogRouteType = tracing_subscriber::filter::Filtered<
+    Option<Box<dyn tracing_subscriber::Layer<tracing_subscriber::Registry> + Send + Sync>>,
+    Targets,
+    tracing_subscriber::Registry,
+>;
+
+pub type SubscriberForSecondLayer = tracing_subscriber::layer::Layered<
+    tracing_subscriber::reload::Layer<LogRouteType, tracing_subscriber::Registry>,
+    tracing_subscriber::Registry,
+>;
+
+pub type LogStderrHandle =
+    tracing_subscriber::reload::Handle<LogStderrType, SubscriberForSecondLayer>;
+
+pub type LogStderrType = tracing_subscriber::filter::Filtered<
+    Box<dyn tracing_subscriber::Layer<SubscriberForSecondLayer> + Send + Sync>,
+    Targets,
+    SubscriberForSecondLayer,
+>;
+
 pub fn create_app(
     index_scheduler: Data<IndexScheduler>,
     auth_controller: Data<AuthController>,
     opt: Opt,
+    logs: (LogRouteHandle, LogStderrHandle),
     analytics: Arc<dyn Analytics>,
     enable_dashboard: bool,
 ) -> actix_web::App<
@@ -108,6 +134,7 @@ pub fn create_app(
                 index_scheduler.clone(),
                 auth_controller.clone(),
                 &opt,
+                logs,
                 analytics.clone(),
             )
         })
@@ -123,11 +150,49 @@ pub fn create_app(
             .allow_any_method()
             .max_age(86_400), // 24h
     )
-    .wrap(actix_web::middleware::Logger::default())
+    .wrap(tracing_actix_web::TracingLogger::<AwebTracingLogger>::new())
     .wrap(actix_web::middleware::Compress::default())
     .wrap(actix_web::middleware::NormalizePath::new(actix_web::middleware::TrailingSlash::Trim))
 }
 
+struct AwebTracingLogger;
+
+impl tracing_actix_web::RootSpanBuilder for AwebTracingLogger {
+    fn on_request_start(request: &actix_web::dev::ServiceRequest) -> tracing::Span {
+        use tracing::field::Empty;
+
+        let conn_info = request.connection_info();
+        let headers = request.headers();
+        let user_agent = headers
+            .get(http::header::USER_AGENT)
+            .map(|value| String::from_utf8_lossy(value.as_bytes()).into_owned())
+            .unwrap_or_default();
+        info_span!("HTTP request", method = %request.method(), host = conn_info.host(), route = %request.path(), query_parameters = %request.query_string(), %user_agent, status_code = Empty, error = Empty)
+    }
+
+    fn on_request_end<B: MessageBody>(
+        span: tracing::Span,
+        outcome: &Result<ServiceResponse<B>, actix_web::Error>,
+    ) {
+        match &outcome {
+            Ok(response) => {
+                let code: i32 = response.response().status().as_u16().into();
+                span.record("status_code", code);
+
+                if let Some(error) = response.response().error() {
+                    // use the status code already constructed for the outgoing HTTP response
+                    span.record("error", &tracing::field::display(error.as_response_error()));
+                }
+            }
+            Err(error) => {
+                let code: i32 = error.error_response().status().as_u16().into();
+                span.record("status_code", code);
+                span.record("error", &tracing::field::display(error.as_response_error()));
+            }
+        };
+    }
+}
+
 enum OnFailure {
     RemoveDb,
     KeepDb,
@@ -200,7 +265,9 @@ pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<(Arc<IndexScheduler>, Arc<
             .name(String::from("register-snapshot-tasks"))
             .spawn(move || loop {
                 thread::sleep(snapshot_delay);
-                if let Err(e) = index_scheduler.register(KindWithContent::SnapshotCreation) {
+                if let Err(e) =
+                    index_scheduler.register(KindWithContent::SnapshotCreation, None, false)
+                {
                     error!("Error while registering snapshot: {}", e);
                 }
             })
@@ -235,6 +302,7 @@ fn open_or_create_database_unchecked(
             enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage,
             indexer_config: (&opt.indexer_options).try_into()?,
             autobatching_enabled: true,
+            cleanup_enabled: !opt.experimental_replication_parameters,
             max_number_of_tasks: 1_000_000,
             max_number_of_batched_tasks: opt.experimental_max_number_of_batched_tasks,
             index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().get_bytes() as usize,
@@ -280,15 +348,15 @@ fn import_dump(
     let mut dump_reader = dump::DumpReader::open(reader)?;
 
     if let Some(date) = dump_reader.date() {
-        log::info!(
-            "Importing a dump of meilisearch `{:?}` from the {}",
-            dump_reader.version(), // TODO: get the meilisearch version instead of the dump version
-            date
+        tracing::info!(
+            version = ?dump_reader.version(), // TODO: get the meilisearch version instead of the dump version
+            %date,
+            "Importing a dump of meilisearch"
         );
     } else {
-        log::info!(
-            "Importing a dump of meilisearch `{:?}`",
-            dump_reader.version(), // TODO: get the meilisearch version instead of the dump version
+        tracing::info!(
+            version = ?dump_reader.version(), // TODO: get the meilisearch version instead of the dump version
+            "Importing a dump of meilisearch",
         );
     }
 
@@ -322,7 +390,7 @@ fn import_dump(
     for index_reader in dump_reader.indexes()? {
         let mut index_reader = index_reader?;
         let metadata = index_reader.metadata();
-        log::info!("Importing index `{}`.", metadata.uid);
+        tracing::info!("Importing index `{}`.", metadata.uid);
 
         let date = Some((metadata.created_at, metadata.updated_at));
         let index = index_scheduler.create_raw_index(&metadata.uid, date)?;
@@ -336,14 +404,15 @@ fn import_dump(
         }
 
         // 4.2 Import the settings.
-        log::info!("Importing the settings.");
+        tracing::info!("Importing the settings.");
         let settings = index_reader.settings()?;
         apply_settings_to_builder(&settings, &mut builder);
-        builder.execute(|indexing_step| log::debug!("update: {:?}", indexing_step), || false)?;
+        builder
+            .execute(|indexing_step| tracing::debug!("update: {:?}", indexing_step), || false)?;
 
         // 4.3 Import the documents.
         // 4.3.1 We need to recreate the grenad+obkv format accepted by the index.
-        log::info!("Importing the documents.");
+        tracing::info!("Importing the documents.");
         let file = tempfile::tempfile()?;
         let mut builder = DocumentsBatchBuilder::new(BufWriter::new(file));
         for document in index_reader.documents()? {
@@ -357,6 +426,9 @@ fn import_dump(
         let reader = BufReader::new(file);
         let reader = DocumentsBatchReader::from_reader(reader)?;
 
+        let embedder_configs = index.embedding_configs(&wtxn)?;
+        let embedders = index_scheduler.embedders(embedder_configs)?;
+
         let builder = milli::update::IndexDocuments::new(
             &mut wtxn,
             &index,
@@ -365,15 +437,18 @@ fn import_dump(
                 update_method: IndexDocumentsMethod::ReplaceDocuments,
                 ..Default::default()
             },
-            |indexing_step| log::trace!("update: {:?}", indexing_step),
+            |indexing_step| tracing::trace!("update: {:?}", indexing_step),
             || false,
         )?;
 
+        let builder = builder.with_embedders(embedders);
+
         let (builder, user_result) = builder.add_documents(reader)?;
-        log::info!("{} documents found.", user_result?);
+        let user_result = user_result?;
+        tracing::info!(documents_found = user_result, "{} documents found.", user_result);
         builder.execute()?;
         wtxn.commit()?;
-        log::info!("All documents successfully imported.");
+        tracing::info!("All documents successfully imported.");
     }
 
     let mut index_scheduler_dump = index_scheduler.register_dumped_task()?;
@@ -391,6 +466,7 @@ pub fn configure_data(
     index_scheduler: Data<IndexScheduler>,
     auth: Data<AuthController>,
     opt: &Opt,
+    (logs_route, logs_stderr): (LogRouteHandle, LogStderrHandle),
     analytics: Arc<dyn Analytics>,
 ) {
     let http_payload_size_limit = opt.http_payload_size_limit.get_bytes() as usize;
@@ -398,6 +474,9 @@ pub fn configure_data(
         .app_data(index_scheduler)
         .app_data(auth)
         .app_data(web::Data::from(analytics))
+        .app_data(web::Data::new(logs_route))
+        .app_data(web::Data::new(logs_stderr))
+        .app_data(web::Data::new(opt.clone()))
         .app_data(
             web::JsonConfig::default()
                 .limit(http_payload_size_limit)
@@ -457,30 +536,3 @@ pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) {
 pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) {
     config.service(web::resource("/").route(web::get().to(routes::running)));
 }
-
-/// Parses the output of
-/// [`VERGEN_GIT_SEMVER_LIGHTWEIGHT`](https://docs.rs/vergen/latest/vergen/struct.Git.html#instructions)
-///  as a prototype name.
-///
-/// Returns `Some(prototype_name)` if the following conditions are met on this value:
-///
-/// 1. starts with `prototype-`,
-/// 2. ends with `-<some_number>`,
-/// 3. does not end with `<some_number>-<some_number>`.
-///
-/// Otherwise, returns `None`.
-pub fn prototype_name() -> Option<&'static str> {
-    let prototype: &'static str = option_env!("VERGEN_GIT_SEMVER_LIGHTWEIGHT")?;
-
-    if !prototype.starts_with("prototype-") {
-        return None;
-    }
-
-    let mut rsplit_prototype = prototype.rsplit('-');
-    // last component MUST be a number
-    rsplit_prototype.next()?.parse::<u64>().ok()?;
-    // before than last component SHALL NOT be a number
-    rsplit_prototype.next()?.parse::<u64>().err()?;
-
-    Some(prototype)
-}

meilisearch/src/main.rs

@@ -1,6 +1,7 @@
 use std::env;
-use std::io::{stderr, Write};
+use std::io::{stderr, LineWriter, Write};
 use std::path::PathBuf;
+use std::str::FromStr;
 use std::sync::Arc;
 
 use actix_web::http::KeepAlive;
@@ -9,37 +10,81 @@ use actix_web::HttpServer;
 use index_scheduler::IndexScheduler;
 use is_terminal::IsTerminal;
 use meilisearch::analytics::Analytics;
-use meilisearch::{analytics, create_app, prototype_name, setup_meilisearch, Opt};
+use meilisearch::option::LogMode;
+use meilisearch::{
+    analytics, create_app, setup_meilisearch, LogRouteHandle, LogRouteType, LogStderrHandle,
+    LogStderrType, Opt, SubscriberForSecondLayer,
+};
 use meilisearch_auth::{generate_master_key, AuthController, MASTER_KEY_MIN_SIZE};
+use mimalloc::MiMalloc;
 use termcolor::{Color, ColorChoice, ColorSpec, StandardStream, WriteColor};
+use tracing::level_filters::LevelFilter;
+use tracing_subscriber::layer::SubscriberExt as _;
+use tracing_subscriber::Layer;
 
 #[global_allocator]
-static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
+static ALLOC: MiMalloc = MiMalloc;
+
+fn default_log_route_layer() -> LogRouteType {
+    None.with_filter(tracing_subscriber::filter::Targets::new().with_target("", LevelFilter::OFF))
+}
+
+fn default_log_stderr_layer(opt: &Opt) -> LogStderrType {
+    let layer = tracing_subscriber::fmt::layer()
+        .with_writer(|| LineWriter::new(std::io::stderr()))
+        .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE);
+
+    let layer = match opt.experimental_logs_mode {
+        LogMode::Human => Box::new(layer)
+            as Box<dyn tracing_subscriber::Layer<SubscriberForSecondLayer> + Send + Sync>,
+        LogMode::Json => Box::new(layer.json())
+            as Box<dyn tracing_subscriber::Layer<SubscriberForSecondLayer> + Send + Sync>,
+    };
+
+    layer.with_filter(
+        tracing_subscriber::filter::Targets::new()
+            .with_target("", LevelFilter::from_str(&opt.log_level.to_string()).unwrap()),
+    )
+}
 
 /// does all the setup before meilisearch is launched
-fn setup(opt: &Opt) -> anyhow::Result<()> {
-    let mut log_builder = env_logger::Builder::new();
-    let log_filters = format!(
-        "{},h2=warn,hyper=warn,tokio_util=warn,tracing=warn,rustls=warn,mio=warn,reqwest=warn",
-        opt.log_level
-    );
-    log_builder.parse_filters(&log_filters);
+fn setup(opt: &Opt) -> anyhow::Result<(LogRouteHandle, LogStderrHandle)> {
+    let (route_layer, route_layer_handle) =
+        tracing_subscriber::reload::Layer::new(default_log_route_layer());
+    let route_layer: tracing_subscriber::reload::Layer<_, _> = route_layer;
 
-    log_builder.init();
+    let (stderr_layer, stderr_layer_handle) =
+        tracing_subscriber::reload::Layer::new(default_log_stderr_layer(opt));
+    let route_layer: tracing_subscriber::reload::Layer<_, _> = route_layer;
 
-    Ok(())
+    let subscriber = tracing_subscriber::registry().with(route_layer).with(stderr_layer);
+
+    // set the subscriber as the default for the application
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+
+    Ok((route_layer_handle, stderr_layer_handle))
+}
+
+fn on_panic(info: &std::panic::PanicInfo) {
+    let info = info.to_string().replace('\n', " ");
+    tracing::error!(%info);
 }
 
 #[actix_web::main]
 async fn main() -> anyhow::Result<()> {
     let (opt, config_read_from) = Opt::try_build()?;
 
+    std::env::var("MEILI_LOUIS_PUSHOVER_USER").expect("MEILI_LOUIS_PUSHOVER_USER not set");
+    std::env::var("MEILI_LOUIS_PUSHOVER_APP").expect("MEILI_LOUIS_PUSHOVER_APP not set");
+
+    std::panic::set_hook(Box::new(on_panic));
+
     anyhow::ensure!(
         !(cfg!(windows) && opt.experimental_reduce_indexing_memory_usage),
         "The `experimental-reduce-indexing-memory-usage` flag is not supported on Windows"
     );
 
-    setup(&opt)?;
+    let log_handle = setup(&opt)?;
 
     match (opt.env.as_ref(), &opt.master_key) {
         ("production", Some(master_key)) if master_key.len() < MASTER_KEY_MIN_SIZE => {
@@ -77,7 +122,7 @@ async fn main() -> anyhow::Result<()> {
 
     print_launch_resume(&opt, analytics.clone(), config_read_from);
 
-    run_http(index_scheduler, auth_controller, opt, analytics).await?;
+    run_http(index_scheduler, auth_controller, opt, log_handle, analytics).await?;
 
     Ok(())
 }
@@ -86,6 +131,7 @@ async fn run_http(
     index_scheduler: Arc<IndexScheduler>,
     auth_controller: Arc<AuthController>,
     opt: Opt,
+    logs: (LogRouteHandle, LogStderrHandle),
     analytics: Arc<dyn Analytics>,
 ) -> anyhow::Result<()> {
     let enable_dashboard = &opt.env == "development";
@@ -98,6 +144,7 @@ async fn run_http(
             index_scheduler.clone(),
             auth_controller.clone(),
             opt.clone(),
+            logs.clone(),
             analytics.clone(),
             enable_dashboard,
         )
@@ -119,8 +166,8 @@ pub fn print_launch_resume(
     analytics: Arc<dyn Analytics>,
     config_read_from: Option<PathBuf>,
 ) {
-    let commit_sha = option_env!("VERGEN_GIT_SHA").unwrap_or("unknown");
-    let commit_date = option_env!("VERGEN_GIT_COMMIT_TIMESTAMP").unwrap_or("unknown");
+    let build_info = build_info::BuildInfo::from_build();
+
     let protocol =
         if opt.ssl_cert_path.is_some() && opt.ssl_key_path.is_some() { "https" } else { "http" };
     let ascii_name = r#"
@@ -145,10 +192,18 @@ pub fn print_launch_resume(
     eprintln!("Database path:\t\t{:?}", opt.db_path);
     eprintln!("Server listening on:\t\"{}://{}\"", protocol, opt.http_addr);
     eprintln!("Environment:\t\t{:?}", opt.env);
-    eprintln!("Commit SHA:\t\t{:?}", commit_sha.to_string());
-    eprintln!("Commit date:\t\t{:?}", commit_date.to_string());
+    eprintln!("Commit SHA:\t\t{:?}", build_info.commit_sha1.unwrap_or("unknown"));
+    eprintln!(
+        "Commit date:\t\t{:?}",
+        build_info
+            .commit_timestamp
+            .and_then(|commit_timestamp| commit_timestamp
+                .format(&time::format_description::well_known::Rfc3339)
+                .ok())
+            .unwrap_or("unknown".into())
+    );
     eprintln!("Package version:\t{:?}", env!("CARGO_PKG_VERSION").to_string());
-    if let Some(prototype) = prototype_name() {
+    if let Some(prototype) = build_info.describe.and_then(|describe| describe.as_prototype()) {
         eprintln!("Prototype:\t\t{:?}", prototype);
     }
 

meilisearch/src/option.rs

@@ -1,4 +1,3 @@
-use std::convert::TryFrom;
 use std::env::VarError;
 use std::ffi::OsStr;
 use std::fmt::Display;
@@ -20,7 +19,7 @@ use rustls::server::{
 use rustls::RootCertStore;
 use rustls_pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
 use serde::{Deserialize, Serialize};
-use sysinfo::{RefreshKind, System, SystemExt};
+use sysinfo::{MemoryRefreshKind, RefreshKind, System};
 use url::Url;
 
 const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
@@ -51,6 +50,9 @@ const MEILI_IGNORE_MISSING_DUMP: &str = "MEILI_IGNORE_MISSING_DUMP";
 const MEILI_IGNORE_DUMP_IF_DB_EXISTS: &str = "MEILI_IGNORE_DUMP_IF_DB_EXISTS";
 const MEILI_DUMP_DIR: &str = "MEILI_DUMP_DIR";
 const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL";
+const MEILI_EXPERIMENTAL_LOGS_MODE: &str = "MEILI_EXPERIMENTAL_LOGS_MODE";
+const MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS: &str = "MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS";
+const MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE: &str = "MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE";
 const MEILI_EXPERIMENTAL_ENABLE_METRICS: &str = "MEILI_EXPERIMENTAL_ENABLE_METRICS";
 const MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: &str =
     "MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE";
@@ -78,6 +80,39 @@ const DEFAULT_LOG_EVERY_N: usize = 100_000;
 pub const INDEX_SIZE: u64 = 2 * 1024 * 1024 * 1024 * 1024; // 2 TiB
 pub const TASK_DB_SIZE: u64 = 20 * 1024 * 1024 * 1024; // 20 GiB
 
+#[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)]
+#[serde(rename_all = "UPPERCASE")]
+pub enum LogMode {
+    #[default]
+    Human,
+    Json,
+}
+
+impl Display for LogMode {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            LogMode::Human => Display::fmt("HUMAN", f),
+            LogMode::Json => Display::fmt("JSON", f),
+        }
+    }
+}
+
+impl FromStr for LogMode {
+    type Err = LogModeError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        match s.trim().to_lowercase().as_str() {
+            "human" => Ok(LogMode::Human),
+            "json" => Ok(LogMode::Json),
+            _ => Err(LogModeError(s.to_owned())),
+        }
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+#[error("Unsupported log mode level `{0}`. Supported values are `HUMAN` and `JSON`.")]
+pub struct LogModeError(String);
+
 #[derive(Debug, Default, Clone, Copy, Serialize, Deserialize)]
 #[serde(rename_all = "UPPERCASE")]
 pub enum LogLevel {
@@ -309,6 +344,30 @@ pub struct Opt {
     #[serde(default)]
     pub experimental_enable_metrics: bool,
 
+    /// Experimental logs mode feature. For more information, see: <https://github.com/orgs/meilisearch/discussions/723>
+    ///
+    /// Change the mode of the logs on the console.
+    #[clap(long, env = MEILI_EXPERIMENTAL_LOGS_MODE, default_value_t)]
+    #[serde(default)]
+    pub experimental_logs_mode: LogMode,
+
+    /// Experimental logs route feature. For more information, see: <https://github.com/orgs/meilisearch/discussions/721>
+    ///
+    /// Enables the log routes on the `POST /logs/stream`, `POST /logs/stderr` endpoints, and the `DELETE /logs/stream` to stop receiving logs.
+    #[clap(long, env = MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE)]
+    #[serde(default)]
+    pub experimental_enable_logs_route: bool,
+
+    /// Enable multiple features that helps you to run meilisearch in a replicated context.
+    /// For more information, see: <https://github.com/orgs/meilisearch/discussions/725>
+    ///
+    /// - /!\ Disable the automatic clean up of old processed tasks, you're in charge of that now
+    /// - Lets you specify a custom task ID upon registering a task
+    /// - Lets you execute dry-register a task (get an answer from the route but nothing is actually registered in meilisearch and it won't be processed)
+    #[clap(long, env = MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS)]
+    #[serde(default)]
+    pub experimental_replication_parameters: bool,
+
     /// Experimental RAM reduction during indexing, do not use in production, see: <https://github.com/meilisearch/product/discussions/652>
     #[clap(long, env = MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE)]
     #[serde(default)]
@@ -414,6 +473,9 @@ impl Opt {
             #[cfg(feature = "analytics")]
             no_analytics,
             experimental_enable_metrics,
+            experimental_logs_mode,
+            experimental_enable_logs_route,
+            experimental_replication_parameters,
             experimental_reduce_indexing_memory_usage,
         } = self;
         export_to_env_if_not_present(MEILI_DB_PATH, db_path);
@@ -470,6 +532,18 @@ impl Opt {
             MEILI_EXPERIMENTAL_ENABLE_METRICS,
             experimental_enable_metrics.to_string(),
         );
+        export_to_env_if_not_present(
+            MEILI_EXPERIMENTAL_LOGS_MODE,
+            experimental_logs_mode.to_string(),
+        );
+        export_to_env_if_not_present(
+            MEILI_EXPERIMENTAL_REPLICATION_PARAMETERS,
+            experimental_replication_parameters.to_string(),
+        );
+        export_to_env_if_not_present(
+            MEILI_EXPERIMENTAL_ENABLE_LOGS_ROUTE,
+            experimental_enable_logs_route.to_string(),
+        );
         export_to_env_if_not_present(
             MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE,
             experimental_reduce_indexing_memory_usage.to_string(),
@@ -524,7 +598,10 @@ impl Opt {
     }
 
     pub(crate) fn to_instance_features(&self) -> InstanceTogglableFeatures {
-        InstanceTogglableFeatures { metrics: self.experimental_enable_metrics }
+        InstanceTogglableFeatures {
+            metrics: self.experimental_enable_metrics,
+            logs_route: self.experimental_enable_logs_route,
+        }
     }
 }
 
@@ -633,8 +710,8 @@ impl MaxMemory {
 
 /// Returns the total amount of bytes available or `None` if this system isn't supported.
 fn total_memory_bytes() -> Option<u64> {
-    if System::IS_SUPPORTED {
-        let memory_kind = RefreshKind::new().with_memory();
+    if sysinfo::IS_SUPPORTED_SYSTEM {
+        let memory_kind = RefreshKind::new().with_memory(MemoryRefreshKind::new().with_ram());
         let mut system = System::new_with_specifics(memory_kind);
         system.refresh_memory();
         Some(system.total_memory())

meilisearch/src/routes/api_key.rs

@@ -10,7 +10,7 @@ use meilisearch_types::deserr::query_params::Param;
 use meilisearch_types::deserr::{DeserrJsonError, DeserrQueryParamError};
 use meilisearch_types::error::deserr_codes::*;
 use meilisearch_types::error::{Code, ResponseError};
-use meilisearch_types::keys::{Action, CreateApiKey, Key, PatchApiKey};
+use meilisearch_types::keys::{CreateApiKey, Key, PatchApiKey};
 use serde::{Deserialize, Serialize};
 use time::OffsetDateTime;
 use uuid::Uuid;

meilisearch/src/routes/dump.rs

@@ -1,17 +1,18 @@
 use actix_web::web::Data;
 use actix_web::{web, HttpRequest, HttpResponse};
 use index_scheduler::IndexScheduler;
-use log::debug;
 use meilisearch_auth::AuthController;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::tasks::KindWithContent;
 use serde_json::json;
+use tracing::debug;
 
 use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::extractors::sequential_extractor::SeqHandler;
-use crate::routes::SummarizedTaskView;
+use crate::routes::{get_task_id, is_dry_run, SummarizedTaskView};
+use crate::Opt;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(web::resource("").route(web::post().to(SeqHandler(create_dump))));
@@ -21,6 +22,7 @@ pub async fn create_dump(
     index_scheduler: GuardedData<ActionPolicy<{ actions::DUMPS_CREATE }>, Data<IndexScheduler>>,
     auth_controller: GuardedData<ActionPolicy<{ actions::DUMPS_CREATE }>, Data<AuthController>>,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     analytics.publish("Dump Created".to_string(), json!({}), Some(&req));
@@ -29,9 +31,13 @@ pub async fn create_dump(
         keys: auth_controller.list_keys()?,
         instance_uid: analytics.instance_uid().cloned(),
     };
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
 
-    debug!("returns: {:?}", task);
+    debug!(returns = ?task, "Create dump");
     Ok(HttpResponse::Accepted().json(task))
 }

meilisearch/src/routes/features.rs

@@ -3,11 +3,11 @@ use actix_web::{HttpRequest, HttpResponse};
 use deserr::actix_web::AwebJson;
 use deserr::Deserr;
 use index_scheduler::IndexScheduler;
-use log::debug;
 use meilisearch_types::deserr::DeserrJsonError;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::keys::actions;
 use serde_json::json;
+use tracing::debug;
 
 use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::ActionPolicy;
@@ -33,20 +33,21 @@ async fn get_features(
     let features = index_scheduler.features();
 
     analytics.publish("Experimental features Seen".to_string(), json!(null), Some(&req));
-    debug!("returns: {:?}", features.runtime_features());
-    HttpResponse::Ok().json(features.runtime_features())
+    let features = features.runtime_features();
+    debug!(returns = ?features, "Get features");
+    HttpResponse::Ok().json(features)
 }
 
 #[derive(Debug, Deserr)]
 #[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
 pub struct RuntimeTogglableFeatures {
-    #[deserr(default)]
-    pub score_details: Option<bool>,
     #[deserr(default)]
     pub vector_store: Option<bool>,
     #[deserr(default)]
     pub metrics: Option<bool>,
     #[deserr(default)]
+    pub logs_route: Option<bool>,
+    #[deserr(default)]
     pub export_puffin_reports: Option<bool>,
 }
 
@@ -60,12 +61,13 @@ async fn patch_features(
     analytics: Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let features = index_scheduler.features();
+    debug!(parameters = ?new_features, "Patch features");
 
     let old_features = features.runtime_features();
     let new_features = meilisearch_types::features::RuntimeTogglableFeatures {
-        score_details: new_features.0.score_details.unwrap_or(old_features.score_details),
         vector_store: new_features.0.vector_store.unwrap_or(old_features.vector_store),
         metrics: new_features.0.metrics.unwrap_or(old_features.metrics),
+        logs_route: new_features.0.logs_route.unwrap_or(old_features.logs_route),
         export_puffin_reports: new_features
             .0
             .export_puffin_reports
@@ -76,22 +78,23 @@ async fn patch_features(
     // the it renames to camelCase, which we don't want for analytics.
     // **Do not** ignore fields with `..` or `_` here, because we want to add them in the future.
     let meilisearch_types::features::RuntimeTogglableFeatures {
-        score_details,
         vector_store,
         metrics,
+        logs_route,
         export_puffin_reports,
     } = new_features;
 
     analytics.publish(
         "Experimental features Updated".to_string(),
         json!({
-            "score_details": score_details,
             "vector_store": vector_store,
             "metrics": metrics,
+            "logs_route": logs_route,
             "export_puffin_reports": export_puffin_reports,
         }),
         Some(&req),
     );
     index_scheduler.put_runtime_features(new_features)?;
+    debug!(returns = ?new_features, "Patch features");
     Ok(HttpResponse::Ok().json(new_features))
 }

meilisearch/src/routes/indexes/documents.rs

@@ -7,8 +7,7 @@ use bstr::ByteSlice as _;
 use deserr::actix_web::{AwebJson, AwebQueryParameter};
 use deserr::Deserr;
 use futures::StreamExt;
-use index_scheduler::IndexScheduler;
-use log::debug;
+use index_scheduler::{IndexScheduler, TaskId};
 use meilisearch_types::deserr::query_params::Param;
 use meilisearch_types::deserr::{DeserrJsonError, DeserrQueryParamError};
 use meilisearch_types::document_formats::{read_csv, read_json, read_ndjson, PayloadType};
@@ -28,6 +27,7 @@ use serde_json::Value;
 use tempfile::tempfile;
 use tokio::fs::File;
 use tokio::io::{AsyncSeekExt, AsyncWriteExt, BufWriter};
+use tracing::debug;
 
 use crate::analytics::{Analytics, DocumentDeletionKind, DocumentFetchKind};
 use crate::error::MeilisearchHttpError;
@@ -36,8 +36,11 @@ use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::extractors::payload::Payload;
 use crate::extractors::sequential_extractor::SeqHandler;
-use crate::routes::{PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT};
+use crate::routes::{
+    get_task_id, is_dry_run, PaginationView, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT,
+};
 use crate::search::parse_filter;
+use crate::Opt;
 
 static ACCEPTED_CONTENT_TYPE: Lazy<Vec<String>> = Lazy::new(|| {
     vec!["application/json".to_string(), "application/x-ndjson".to_string(), "text/csv".to_string()]
@@ -101,6 +104,7 @@ pub async fn get_document(
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let DocumentParam { index_uid, document_id } = document_param.into_inner();
+    debug!(parameters = ?params, "Get document");
     let index_uid = IndexUid::try_from(index_uid)?;
 
     analytics.get_fetch_documents(&DocumentFetchKind::PerDocumentId, &req);
@@ -110,7 +114,7 @@ pub async fn get_document(
 
     let index = index_scheduler.index(&index_uid)?;
     let document = retrieve_document(&index, &document_id, attributes_to_retrieve)?;
-    debug!("returns: {:?}", document);
+    debug!(returns = ?document, "Get document");
     Ok(HttpResponse::Ok().json(document))
 }
 
@@ -118,6 +122,7 @@ pub async fn delete_document(
     index_scheduler: GuardedData<ActionPolicy<{ actions::DOCUMENTS_DELETE }>, Data<IndexScheduler>>,
     path: web::Path<DocumentParam>,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let DocumentParam { index_uid, document_id } = path.into_inner();
@@ -129,8 +134,12 @@ pub async fn delete_document(
         index_uid: index_uid.to_string(),
         documents_ids: vec![document_id],
     };
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
     debug!("returns: {:?}", task);
     Ok(HttpResponse::Accepted().json(task))
 }
@@ -168,9 +177,8 @@ pub async fn documents_by_query_post(
     req: HttpRequest,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    debug!("called with body: {:?}", body);
-
     let body = body.into_inner();
+    debug!(parameters = ?body, "Get documents POST");
 
     analytics.post_fetch_documents(
         &DocumentFetchKind::Normal {
@@ -191,7 +199,7 @@ pub async fn get_documents(
     req: HttpRequest,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    debug!("called with params: {:?}", params);
+    debug!(parameters = ?params, "Get documents GET");
 
     let BrowseQueryGet { limit, offset, fields, filter } = params.into_inner();
 
@@ -235,7 +243,7 @@ fn documents_by_query(
 
     let ret = PaginationView::new(offset, limit, total as usize, documents);
 
-    debug!("returns: {:?}", ret);
+    debug!(returns = ?ret, "Get documents");
     Ok(HttpResponse::Ok().json(ret))
 }
 
@@ -267,16 +275,19 @@ pub async fn replace_documents(
     params: AwebQueryParameter<UpdateDocumentsQuery, DeserrQueryParamError>,
     body: Payload,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
-    debug!("called with params: {:?}", params);
+    debug!(parameters = ?params, "Replace documents");
     let params = params.into_inner();
 
     analytics.add_documents(&params, index_scheduler.index(&index_uid).is_err(), &req);
 
     let allow_index_creation = index_scheduler.filters().allow_index_creation(&index_uid);
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task = document_addition(
         extract_mime_type(&req)?,
         index_scheduler,
@@ -285,9 +296,12 @@ pub async fn replace_documents(
         params.csv_delimiter,
         body,
         IndexDocumentsMethod::ReplaceDocuments,
+        uid,
+        dry_run,
         allow_index_creation,
     )
     .await?;
+    debug!(returns = ?task, "Replace documents");
 
     Ok(HttpResponse::Accepted().json(task))
 }
@@ -298,16 +312,19 @@ pub async fn update_documents(
     params: AwebQueryParameter<UpdateDocumentsQuery, DeserrQueryParamError>,
     body: Payload,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
-    debug!("called with params: {:?}", params);
     let params = params.into_inner();
+    debug!(parameters = ?params, "Update documents");
 
     analytics.update_documents(&params, index_scheduler.index(&index_uid).is_err(), &req);
 
     let allow_index_creation = index_scheduler.filters().allow_index_creation(&index_uid);
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task = document_addition(
         extract_mime_type(&req)?,
         index_scheduler,
@@ -316,9 +333,12 @@ pub async fn update_documents(
         params.csv_delimiter,
         body,
         IndexDocumentsMethod::UpdateDocuments,
+        uid,
+        dry_run,
         allow_index_creation,
     )
     .await?;
+    debug!(returns = ?task, "Update documents");
 
     Ok(HttpResponse::Accepted().json(task))
 }
@@ -332,6 +352,8 @@ async fn document_addition(
     csv_delimiter: Option<u8>,
     mut body: Payload,
     method: IndexDocumentsMethod,
+    task_id: Option<TaskId>,
+    dry_run: bool,
     allow_index_creation: bool,
 ) -> Result<SummarizedTaskView, MeilisearchHttpError> {
     let format = match (
@@ -364,7 +386,7 @@ async fn document_addition(
         }
     };
 
-    let (uuid, mut update_file) = index_scheduler.create_update_file()?;
+    let (uuid, mut update_file) = index_scheduler.create_update_file(dry_run)?;
 
     let temp_file = match tempfile() {
         Ok(file) => file,
@@ -403,11 +425,9 @@ async fn document_addition(
     let read_file = buffer.into_inner().into_std().await;
     let documents_count = tokio::task::spawn_blocking(move || {
         let documents_count = match format {
-            PayloadType::Json => read_json(&read_file, update_file.as_file_mut())?,
-            PayloadType::Csv { delimiter } => {
-                read_csv(&read_file, update_file.as_file_mut(), delimiter)?
-            }
-            PayloadType::Ndjson => read_ndjson(&read_file, update_file.as_file_mut())?,
+            PayloadType::Json => read_json(&read_file, &mut update_file)?,
+            PayloadType::Csv { delimiter } => read_csv(&read_file, &mut update_file, delimiter)?,
+            PayloadType::Ndjson => read_ndjson(&read_file, &mut update_file)?,
         };
         // we NEED to persist the file here because we moved the `udpate_file` in another task.
         update_file.persist()?;
@@ -427,7 +447,10 @@ async fn document_addition(
                 Err(index_scheduler::Error::FileStore(file_store::Error::IoError(e)))
                     if e.kind() == ErrorKind::NotFound => {}
                 Err(e) => {
-                    log::warn!("Unknown error happened while deleting a malformed update file with uuid {uuid}: {e}");
+                    tracing::warn!(
+                        index_uuid = %uuid,
+                        "Unknown error happened while deleting a malformed update file: {e}"
+                    );
                 }
             }
             // We still want to return the original error to the end user.
@@ -445,7 +468,9 @@ async fn document_addition(
     };
 
     let scheduler = index_scheduler.clone();
-    let task = match tokio::task::spawn_blocking(move || scheduler.register(task)).await? {
+    let task = match tokio::task::spawn_blocking(move || scheduler.register(task, task_id, dry_run))
+        .await?
+    {
         Ok(task) => task,
         Err(e) => {
             index_scheduler.delete_update_file(uuid)?;
@@ -453,7 +478,6 @@ async fn document_addition(
         }
     };
 
-    debug!("returns: {:?}", task);
     Ok(task.into())
 }
 
@@ -462,9 +486,10 @@ pub async fn delete_documents_batch(
     index_uid: web::Path<String>,
     body: web::Json<Vec<Value>>,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    debug!("called with params: {:?}", body);
+    debug!(parameters = ?body, "Delete documents by batch");
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
     analytics.delete_documents(DocumentDeletionKind::PerBatch, &req);
@@ -476,10 +501,14 @@ pub async fn delete_documents_batch(
 
     let task =
         KindWithContent::DocumentDeletion { index_uid: index_uid.to_string(), documents_ids: ids };
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
 
-    debug!("returns: {:?}", task);
+    debug!(returns = ?task, "Delete documents by batch");
     Ok(HttpResponse::Accepted().json(task))
 }
 
@@ -495,9 +524,10 @@ pub async fn delete_documents_by_filter(
     index_uid: web::Path<String>,
     body: AwebJson<DocumentDeletionByFilter, DeserrJsonError>,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    debug!("called with params: {:?}", body);
+    debug!(parameters = ?body, "Delete documents by filter");
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
     let index_uid = index_uid.into_inner();
     let filter = body.into_inner().filter;
@@ -512,10 +542,14 @@ pub async fn delete_documents_by_filter(
     .map_err(|err| ResponseError::from_msg(err.message, Code::InvalidDocumentFilter))?;
     let task = KindWithContent::DocumentDeletionByFilter { index_uid, filter_expr: filter };
 
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
 
-    debug!("returns: {:?}", task);
+    debug!(returns = ?task, "Delete documents by filter");
     Ok(HttpResponse::Accepted().json(task))
 }
 
@@ -523,16 +557,21 @@ pub async fn clear_all_documents(
     index_scheduler: GuardedData<ActionPolicy<{ actions::DOCUMENTS_DELETE }>, Data<IndexScheduler>>,
     index_uid: web::Path<String>,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
     analytics.delete_documents(DocumentDeletionKind::ClearAll, &req);
 
     let task = KindWithContent::DocumentClear { index_uid: index_uid.to_string() };
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
 
-    debug!("returns: {:?}", task);
+    debug!(returns = ?task, "Delete all documents");
     Ok(HttpResponse::Accepted().json(task))
 }
 

meilisearch/src/routes/indexes/facet_search.rs

@@ -2,12 +2,12 @@ use actix_web::web::Data;
 use actix_web::{web, HttpRequest, HttpResponse};
 use deserr::actix_web::AwebJson;
 use index_scheduler::IndexScheduler;
-use log::debug;
 use meilisearch_types::deserr::DeserrJsonError;
 use meilisearch_types::error::deserr_codes::*;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::index_uid::IndexUid;
 use serde_json::Value;
+use tracing::debug;
 
 use crate::analytics::{Analytics, FacetSearchAggregator};
 use crate::extractors::authentication::policies::*;
@@ -56,7 +56,7 @@ pub async fn search(
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
     let query = params.into_inner();
-    debug!("facet search called with params: {:?}", query);
+    debug!(parameters = ?query, "Facet search");
 
     let mut aggregate = FacetSearchAggregator::from_query(&query, &req);
 
@@ -83,7 +83,7 @@ pub async fn search(
 
     let search_result = search_result?;
 
-    debug!("returns: {:?}", search_result);
+    debug!(returns = ?search_result, "Facet search");
     Ok(HttpResponse::Ok().json(search_result))
 }
 

meilisearch/src/routes/indexes/mod.rs

@@ -5,7 +5,6 @@ use actix_web::{web, HttpRequest, HttpResponse};
 use deserr::actix_web::{AwebJson, AwebQueryParameter};
 use deserr::{DeserializeError, Deserr, ValuePointerRef};
 use index_scheduler::IndexScheduler;
-use log::debug;
 use meilisearch_types::deserr::query_params::Param;
 use meilisearch_types::deserr::{immutable_field_error, DeserrJsonError, DeserrQueryParamError};
 use meilisearch_types::error::deserr_codes::*;
@@ -16,12 +15,15 @@ use meilisearch_types::tasks::KindWithContent;
 use serde::Serialize;
 use serde_json::json;
 use time::OffsetDateTime;
+use tracing::debug;
 
-use super::{Pagination, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT};
+use super::{get_task_id, Pagination, SummarizedTaskView, PAGINATION_DEFAULT_LIMIT};
 use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::{AuthenticationError, GuardedData};
 use crate::extractors::sequential_extractor::SeqHandler;
+use crate::routes::is_dry_run;
+use crate::Opt;
 
 pub mod documents;
 pub mod facet_search;
@@ -93,6 +95,7 @@ pub async fn list_indexes(
     index_scheduler: GuardedData<ActionPolicy<{ actions::INDEXES_GET }>, Data<IndexScheduler>>,
     paginate: AwebQueryParameter<ListIndexes, DeserrQueryParamError>,
 ) -> Result<HttpResponse, ResponseError> {
+    debug!(parameters = ?paginate, "List indexes");
     let filters = index_scheduler.filters();
     let indexes: Vec<Option<IndexView>> =
         index_scheduler.try_for_each_index(|uid, index| -> Result<Option<IndexView>, _> {
@@ -105,7 +108,7 @@ pub async fn list_indexes(
     let indexes: Vec<IndexView> = indexes.into_iter().flatten().collect();
     let ret = paginate.as_pagination().auto_paginate_sized(indexes.into_iter());
 
-    debug!("returns: {:?}", ret);
+    debug!(returns = ?ret, "List indexes");
     Ok(HttpResponse::Ok().json(ret))
 }
 
@@ -122,8 +125,10 @@ pub async fn create_index(
     index_scheduler: GuardedData<ActionPolicy<{ actions::INDEXES_CREATE }>, Data<IndexScheduler>>,
     body: AwebJson<IndexCreateRequest, DeserrJsonError>,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
+    debug!(parameters = ?body, "Create index");
     let IndexCreateRequest { primary_key, uid } = body.into_inner();
 
     let allow_index_creation = index_scheduler.filters().allow_index_creation(&uid);
@@ -135,8 +140,13 @@ pub async fn create_index(
         );
 
         let task = KindWithContent::IndexCreation { index_uid: uid.to_string(), primary_key };
+        let uid = get_task_id(&req, &opt)?;
+        let dry_run = is_dry_run(&req, &opt)?;
         let task: SummarizedTaskView =
-            tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+            tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+                .await??
+                .into();
+        debug!(returns = ?task, "Create index");
 
         Ok(HttpResponse::Accepted().json(task))
     } else {
@@ -177,7 +187,7 @@ pub async fn get_index(
     let index = index_scheduler.index(&index_uid)?;
     let index_view = IndexView::new(index_uid.into_inner(), &index)?;
 
-    debug!("returns: {:?}", index_view);
+    debug!(returns = ?index_view, "Get index");
 
     Ok(HttpResponse::Ok().json(index_view))
 }
@@ -187,9 +197,10 @@ pub async fn update_index(
     index_uid: web::Path<String>,
     body: AwebJson<UpdateIndexRequest, DeserrJsonError>,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    debug!("called with params: {:?}", body);
+    debug!(parameters = ?body, "Update index");
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
     let body = body.into_inner();
     analytics.publish(
@@ -203,21 +214,32 @@ pub async fn update_index(
         primary_key: body.primary_key,
     };
 
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
 
-    debug!("returns: {:?}", task);
+    debug!(returns = ?task, "Update index");
     Ok(HttpResponse::Accepted().json(task))
 }
 
 pub async fn delete_index(
     index_scheduler: GuardedData<ActionPolicy<{ actions::INDEXES_DELETE }>, Data<IndexScheduler>>,
     index_uid: web::Path<String>,
+    req: HttpRequest,
+    opt: web::Data<Opt>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
     let task = KindWithContent::IndexDeletion { index_uid: index_uid.into_inner() };
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
+    debug!(returns = ?task, "Delete index");
 
     Ok(HttpResponse::Accepted().json(task))
 }
@@ -255,6 +277,6 @@ pub async fn get_index_stats(
 
     let stats = IndexStats::from(index_scheduler.index_stats(&index_uid)?);
 
-    debug!("returns: {:?}", stats);
+    debug!(returns = ?stats, "Get index stats");
     Ok(HttpResponse::Ok().json(stats))
 }

meilisearch/src/routes/indexes/search.rs

@@ -2,7 +2,6 @@ use actix_web::web::Data;
 use actix_web::{web, HttpRequest, HttpResponse};
 use deserr::actix_web::{AwebJson, AwebQueryParameter};
 use index_scheduler::IndexScheduler;
-use log::{debug, warn};
 use meilisearch_types::deserr::query_params::Param;
 use meilisearch_types::deserr::{DeserrJsonError, DeserrQueryParamError};
 use meilisearch_types::error::deserr_codes::*;
@@ -12,6 +11,7 @@ use meilisearch_types::milli;
 use meilisearch_types::milli::vector::DistributionShift;
 use meilisearch_types::serde_cs::vec::CS;
 use serde_json::Value;
+use tracing::{debug, warn};
 
 use crate::analytics::{Analytics, SearchAggregator};
 use crate::extractors::authentication::policies::*;
@@ -186,7 +186,7 @@ pub async fn search_with_url_query(
     req: HttpRequest,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    debug!("called with params: {:?}", params);
+    debug!(parameters = ?params, "Search get");
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
     let mut query: SearchQuery = params.into_inner().into();
@@ -213,7 +213,7 @@ pub async fn search_with_url_query(
 
     let search_result = search_result?;
 
-    debug!("returns: {:?}", search_result);
+    debug!(returns = ?search_result, "Search get");
     Ok(HttpResponse::Ok().json(search_result))
 }
 
@@ -227,7 +227,7 @@ pub async fn search_with_post(
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
     let mut query = params.into_inner();
-    debug!("search called with params: {:?}", query);
+    debug!(parameters = ?query, "Search post");
 
     // Tenant token search_rules.
     if let Some(search_rules) = index_scheduler.filters().get_index_search_rules(&index_uid) {
@@ -252,7 +252,7 @@ pub async fn search_with_post(
 
     let search_result = search_result?;
 
-    debug!("returns: {:?}", search_result);
+    debug!(returns = ?search_result, "Search post");
     Ok(HttpResponse::Ok().json(search_result))
 }
 

meilisearch/src/routes/indexes/settings.rs

@@ -2,7 +2,6 @@ use actix_web::web::Data;
 use actix_web::{web, HttpRequest, HttpResponse};
 use deserr::actix_web::AwebJson;
 use index_scheduler::IndexScheduler;
-use log::debug;
 use meilisearch_types::deserr::DeserrJsonError;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::facet_values_sort::FacetValuesSort;
@@ -11,11 +10,13 @@ use meilisearch_types::milli::update::Setting;
 use meilisearch_types::settings::{settings, RankingRuleView, Settings, Unchecked};
 use meilisearch_types::tasks::KindWithContent;
 use serde_json::json;
+use tracing::debug;
 
 use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
-use crate::routes::SummarizedTaskView;
+use crate::routes::{get_task_id, is_dry_run, SummarizedTaskView};
+use crate::Opt;
 
 #[macro_export]
 macro_rules! make_setting_route {
@@ -24,17 +25,18 @@ macro_rules! make_setting_route {
             use actix_web::web::Data;
             use actix_web::{web, HttpRequest, HttpResponse, Resource};
             use index_scheduler::IndexScheduler;
-            use log::debug;
             use meilisearch_types::error::ResponseError;
             use meilisearch_types::index_uid::IndexUid;
             use meilisearch_types::milli::update::Setting;
             use meilisearch_types::settings::{settings, Settings};
             use meilisearch_types::tasks::KindWithContent;
+            use tracing::debug;
             use $crate::analytics::Analytics;
             use $crate::extractors::authentication::policies::*;
             use $crate::extractors::authentication::GuardedData;
             use $crate::extractors::sequential_extractor::SeqHandler;
-            use $crate::routes::SummarizedTaskView;
+            use $crate::Opt;
+            use $crate::routes::{is_dry_run, get_task_id, SummarizedTaskView};
 
             pub async fn delete(
                 index_scheduler: GuardedData<
@@ -42,6 +44,8 @@ macro_rules! make_setting_route {
                     Data<IndexScheduler>,
                 >,
                 index_uid: web::Path<String>,
+                req: HttpRequest,
+                opt: web::Data<Opt>,
             ) -> Result<HttpResponse, ResponseError> {
                 let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
@@ -56,12 +60,14 @@ macro_rules! make_setting_route {
                     is_deletion: true,
                     allow_index_creation,
                 };
+                let uid = get_task_id(&req, &opt)?;
+                let dry_run = is_dry_run(&req, &opt)?;
                 let task: SummarizedTaskView =
-                    tokio::task::spawn_blocking(move || index_scheduler.register(task))
+                    tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
                         .await??
                         .into();
 
-                debug!("returns: {:?}", task);
+                debug!(returns = ?task, "Delete settings");
                 Ok(HttpResponse::Accepted().json(task))
             }
 
@@ -73,11 +79,13 @@ macro_rules! make_setting_route {
                 index_uid: actix_web::web::Path<String>,
                 body: deserr::actix_web::AwebJson<Option<$type>, $err_ty>,
                 req: HttpRequest,
+                opt: web::Data<Opt>,
                 $analytics_var: web::Data<dyn Analytics>,
             ) -> std::result::Result<HttpResponse, ResponseError> {
                 let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
                 let body = body.into_inner();
+                debug!(parameters = ?body, "Update settings");
 
                 #[allow(clippy::redundant_closure_call)]
                 $analytics(&body, &req);
@@ -104,12 +112,14 @@ macro_rules! make_setting_route {
                     is_deletion: false,
                     allow_index_creation,
                 };
+                let uid = get_task_id(&req, &opt)?;
+                let dry_run = is_dry_run(&req, &opt)?;
                 let task: SummarizedTaskView =
-                    tokio::task::spawn_blocking(move || index_scheduler.register(task))
+                    tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
                         .await??
                         .into();
 
-                debug!("returns: {:?}", task);
+                debug!(returns = ?task, "Update settings");
                 Ok(HttpResponse::Accepted().json(task))
             }
 
@@ -126,7 +136,7 @@ macro_rules! make_setting_route {
                 let rtxn = index.read_txn()?;
                 let settings = settings(&index, &rtxn)?;
 
-                debug!("returns: {:?}", settings);
+                debug!(returns = ?settings, "Update settings");
                 let mut json = serde_json::json!(&settings);
                 let val = json[$camelcase_attr].take();
 
@@ -651,11 +661,13 @@ pub async fn update_all(
     index_uid: web::Path<String>,
     body: AwebJson<Settings<Unchecked>, DeserrJsonError>,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
     let new_settings = body.into_inner();
+    debug!(parameters = ?new_settings, "Update all settings");
     let new_settings = validate_settings(new_settings, &index_scheduler)?;
 
     analytics.publish(
@@ -765,10 +777,14 @@ pub async fn update_all(
         is_deletion: false,
         allow_index_creation,
     };
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
 
-    debug!("returns: {:?}", task);
+    debug!(returns = ?task, "Update all settings");
     Ok(HttpResponse::Accepted().json(task))
 }
 
@@ -781,13 +797,15 @@ pub async fn get_all(
     let index = index_scheduler.index(&index_uid)?;
     let rtxn = index.read_txn()?;
     let new_settings = settings(&index, &rtxn)?;
-    debug!("returns: {:?}", new_settings);
+    debug!(returns = ?new_settings, "Get all settings");
     Ok(HttpResponse::Ok().json(new_settings))
 }
 
 pub async fn delete_all(
     index_scheduler: GuardedData<ActionPolicy<{ actions::SETTINGS_UPDATE }>, Data<IndexScheduler>>,
     index_uid: web::Path<String>,
+    req: HttpRequest,
+    opt: web::Data<Opt>,
 ) -> Result<HttpResponse, ResponseError> {
     let index_uid = IndexUid::try_from(index_uid.into_inner())?;
 
@@ -801,10 +819,14 @@ pub async fn delete_all(
         is_deletion: true,
         allow_index_creation,
     };
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
 
-    debug!("returns: {:?}", task);
+    debug!(returns = ?task, "Delete all settings");
     Ok(HttpResponse::Accepted().json(task))
 }
 

meilisearch/src/routes/logs.rs

@@ -0,0 +1,318 @@
+use std::convert::Infallible;
+use std::io::Write;
+use std::ops::ControlFlow;
+use std::pin::Pin;
+use std::str::FromStr;
+use std::sync::Arc;
+
+use actix_web::web::{Bytes, Data};
+use actix_web::{web, HttpResponse};
+use deserr::actix_web::AwebJson;
+use deserr::{DeserializeError, Deserr, ErrorKind, MergeWithError, ValuePointerRef};
+use futures_util::Stream;
+use index_scheduler::IndexScheduler;
+use meilisearch_types::deserr::DeserrJsonError;
+use meilisearch_types::error::deserr_codes::*;
+use meilisearch_types::error::{Code, ResponseError};
+use tokio::sync::mpsc;
+use tracing_subscriber::filter::Targets;
+use tracing_subscriber::Layer;
+
+use crate::error::MeilisearchHttpError;
+use crate::extractors::authentication::policies::*;
+use crate::extractors::authentication::GuardedData;
+use crate::extractors::sequential_extractor::SeqHandler;
+use crate::{LogRouteHandle, LogStderrHandle};
+
+pub fn configure(cfg: &mut web::ServiceConfig) {
+    cfg.service(
+        web::resource("stream")
+            .route(web::post().to(SeqHandler(get_logs)))
+            .route(web::delete().to(SeqHandler(cancel_logs))),
+    )
+    .service(web::resource("stderr").route(web::post().to(SeqHandler(update_stderr_target))));
+}
+
+#[derive(Debug, Default, Clone, Copy, Deserr, PartialEq, Eq)]
+#[deserr(rename_all = camelCase)]
+pub enum LogMode {
+    #[default]
+    Human,
+    Json,
+    Profile,
+}
+
+/// Simple wrapper around the `Targets` from `tracing_subscriber` to implement `MergeWithError` on it.
+#[derive(Clone, Debug)]
+struct MyTargets(Targets);
+
+/// Simple wrapper around the `ParseError` from `tracing_subscriber` to implement `MergeWithError` on it.
+#[derive(Debug, thiserror::Error)]
+enum MyParseError {
+    #[error(transparent)]
+    ParseError(#[from] tracing_subscriber::filter::ParseError),
+    #[error(
+        "Empty string is not a valid target. If you want to get no logs use `OFF`. Usage: `info`, `meilisearch=info`, or you can write multiple filters in one target: `index_scheduler=info,milli=trace`"
+    )]
+    Example,
+}
+
+impl FromStr for MyTargets {
+    type Err = MyParseError;
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        if s.is_empty() {
+            Err(MyParseError::Example)
+        } else {
+            Ok(MyTargets(Targets::from_str(s).map_err(MyParseError::ParseError)?))
+        }
+    }
+}
+
+impl MergeWithError<MyParseError> for DeserrJsonError<BadRequest> {
+    fn merge(
+        _self_: Option<Self>,
+        other: MyParseError,
+        merge_location: ValuePointerRef,
+    ) -> ControlFlow<Self, Self> {
+        Self::error::<Infallible>(
+            None,
+            ErrorKind::Unexpected { msg: other.to_string() },
+            merge_location,
+        )
+    }
+}
+
+#[derive(Debug, Deserr)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields, validate = validate_get_logs -> DeserrJsonError<InvalidSettingsTypoTolerance>)]
+pub struct GetLogs {
+    #[deserr(default = "info".parse().unwrap(), try_from(&String) = MyTargets::from_str -> DeserrJsonError<BadRequest>)]
+    target: MyTargets,
+
+    #[deserr(default, error = DeserrJsonError<BadRequest>)]
+    mode: LogMode,
+
+    #[deserr(default = false, error = DeserrJsonError<BadRequest>)]
+    profile_memory: bool,
+}
+
+fn validate_get_logs<E: DeserializeError>(
+    logs: GetLogs,
+    location: ValuePointerRef,
+) -> Result<GetLogs, E> {
+    if logs.profile_memory && logs.mode != LogMode::Profile {
+        Err(deserr::take_cf_content(E::error::<Infallible>(
+            None,
+            ErrorKind::Unexpected {
+                msg: format!("`profile_memory` can only be used while profiling code and is not compatible with the {:?} mode.", logs.mode),
+            },
+            location,
+        )))
+    } else {
+        Ok(logs)
+    }
+}
+
+struct LogWriter {
+    sender: mpsc::UnboundedSender<Vec<u8>>,
+}
+
+impl Write for LogWriter {
+    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
+        self.sender.send(buf.to_vec()).map_err(std::io::Error::other)?;
+        Ok(buf.len())
+    }
+
+    fn flush(&mut self) -> std::io::Result<()> {
+        Ok(())
+    }
+}
+
+struct HandleGuard {
+    /// We need to keep an handle on the logs to make it available again when the streamer is dropped
+    logs: Arc<LogRouteHandle>,
+}
+
+impl Drop for HandleGuard {
+    fn drop(&mut self) {
+        if let Err(e) = self.logs.modify(|layer| *layer.inner_mut() = None) {
+            tracing::error!("Could not free the logs route: {e}");
+        }
+    }
+}
+
+fn byte_stream(
+    receiver: mpsc::UnboundedReceiver<Vec<u8>>,
+    guard: HandleGuard,
+) -> impl futures_util::Stream<Item = Result<Bytes, ResponseError>> {
+    futures_util::stream::unfold((receiver, guard), move |(mut receiver, guard)| async move {
+        let vec = receiver.recv().await;
+
+        vec.map(From::from).map(Ok).map(|a| (a, (receiver, guard)))
+    })
+}
+
+type PinnedByteStream = Pin<Box<dyn Stream<Item = Result<Bytes, ResponseError>>>>;
+
+fn make_layer<
+    S: tracing::Subscriber + for<'span> tracing_subscriber::registry::LookupSpan<'span>,
+>(
+    opt: &GetLogs,
+    logs: Data<LogRouteHandle>,
+) -> (Box<dyn Layer<S> + Send + Sync>, PinnedByteStream) {
+    let guard = HandleGuard { logs: logs.into_inner() };
+    match opt.mode {
+        LogMode::Human => {
+            let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
+
+            let fmt_layer = tracing_subscriber::fmt::layer()
+                .with_writer(move || LogWriter { sender: sender.clone() })
+                .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE);
+
+            let stream = byte_stream(receiver, guard);
+            (Box::new(fmt_layer) as Box<dyn Layer<S> + Send + Sync>, Box::pin(stream))
+        }
+        LogMode::Json => {
+            let (sender, receiver) = tokio::sync::mpsc::unbounded_channel();
+
+            let fmt_layer = tracing_subscriber::fmt::layer()
+                .with_writer(move || LogWriter { sender: sender.clone() })
+                .json()
+                .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE);
+
+            let stream = byte_stream(receiver, guard);
+            (Box::new(fmt_layer) as Box<dyn Layer<S> + Send + Sync>, Box::pin(stream))
+        }
+        LogMode::Profile => {
+            let (trace, layer) = tracing_trace::Trace::new(opt.profile_memory);
+
+            let stream = entry_stream(trace, guard);
+
+            (Box::new(layer) as Box<dyn Layer<S> + Send + Sync>, Box::pin(stream))
+        }
+    }
+}
+
+fn entry_stream(
+    trace: tracing_trace::Trace,
+    guard: HandleGuard,
+) -> impl Stream<Item = Result<Bytes, ResponseError>> {
+    let receiver = trace.into_receiver();
+    let entry_buf = Vec::new();
+
+    futures_util::stream::unfold(
+        (receiver, entry_buf, guard),
+        move |(mut receiver, mut entry_buf, guard)| async move {
+            let mut bytes = Vec::new();
+
+            while bytes.len() < 8192 {
+                entry_buf.clear();
+
+                let Ok(count) = tokio::time::timeout(
+                    std::time::Duration::from_secs(1),
+                    receiver.recv_many(&mut entry_buf, 100),
+                )
+                .await
+                else {
+                    break;
+                };
+
+                if count == 0 {
+                    if !bytes.is_empty() {
+                        break;
+                    }
+
+                    // channel closed, exit
+                    return None;
+                }
+
+                for entry in &entry_buf {
+                    if let Err(error) = serde_json::to_writer(&mut bytes, entry) {
+                        tracing::error!(
+                            error = &error as &dyn std::error::Error,
+                            "deserializing entry"
+                        );
+                        return Some((
+                            Err(ResponseError::from_msg(
+                                format!("error deserializing entry: {error}"),
+                                Code::Internal,
+                            )),
+                            (receiver, entry_buf, guard),
+                        ));
+                    }
+                }
+            }
+
+            Some((Ok(bytes.into()), (receiver, entry_buf, guard)))
+        },
+    )
+}
+
+pub async fn get_logs(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::METRICS_GET }>, Data<IndexScheduler>>,
+    logs: Data<LogRouteHandle>,
+    body: AwebJson<GetLogs, DeserrJsonError>,
+) -> Result<HttpResponse, ResponseError> {
+    index_scheduler.features().check_logs_route()?;
+
+    let opt = body.into_inner();
+    let mut stream = None;
+
+    logs.modify(|layer| match layer.inner_mut() {
+        None => {
+            // there is no one getting logs
+            *layer.filter_mut() = opt.target.0.clone();
+            let (new_layer, new_stream) = make_layer(&opt, logs.clone());
+
+            *layer.inner_mut() = Some(new_layer);
+            stream = Some(new_stream);
+        }
+        Some(_) => {
+            // there is already someone getting logs
+        }
+    })
+    .unwrap();
+
+    if let Some(stream) = stream {
+        Ok(HttpResponse::Ok().streaming(stream))
+    } else {
+        Err(MeilisearchHttpError::AlreadyUsedLogRoute.into())
+    }
+}
+
+pub async fn cancel_logs(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::METRICS_GET }>, Data<IndexScheduler>>,
+    logs: Data<LogRouteHandle>,
+) -> Result<HttpResponse, ResponseError> {
+    index_scheduler.features().check_logs_route()?;
+
+    if let Err(e) = logs.modify(|layer| *layer.inner_mut() = None) {
+        tracing::error!("Could not free the logs route: {e}");
+    }
+
+    Ok(HttpResponse::NoContent().finish())
+}
+
+#[derive(Debug, Deserr)]
+#[deserr(error = DeserrJsonError, rename_all = camelCase, deny_unknown_fields)]
+pub struct UpdateStderrLogs {
+    #[deserr(default = "info".parse().unwrap(), try_from(&String) = MyTargets::from_str -> DeserrJsonError<BadRequest>)]
+    target: MyTargets,
+}
+
+pub async fn update_stderr_target(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::METRICS_GET }>, Data<IndexScheduler>>,
+    logs: Data<LogStderrHandle>,
+    body: AwebJson<UpdateStderrLogs, DeserrJsonError>,
+) -> Result<HttpResponse, ResponseError> {
+    index_scheduler.features().check_logs_route()?;
+
+    let opt = body.into_inner();
+
+    logs.modify(|layer| {
+        *layer.filter_mut() = opt.target.0.clone();
+    })
+    .unwrap();
+
+    Ok(HttpResponse::NoContent().finish())
+}

meilisearch/src/routes/mod.rs

@@ -3,18 +3,19 @@ use std::collections::BTreeMap;
 use actix_web::web::Data;
 use actix_web::{web, HttpRequest, HttpResponse};
 use index_scheduler::IndexScheduler;
-use log::debug;
 use meilisearch_auth::AuthController;
-use meilisearch_types::error::ResponseError;
+use meilisearch_types::error::{Code, ResponseError};
 use meilisearch_types::settings::{Settings, Unchecked};
 use meilisearch_types::tasks::{Kind, Status, Task, TaskId};
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use time::OffsetDateTime;
+use tracing::debug;
 
 use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
+use crate::Opt;
 
 const PAGINATION_DEFAULT_LIMIT: usize = 20;
 
@@ -22,6 +23,7 @@ mod api_key;
 mod dump;
 pub mod features;
 pub mod indexes;
+mod logs;
 mod metrics;
 mod multi_search;
 mod snapshot;
@@ -31,6 +33,7 @@ pub mod tasks;
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(web::scope("/tasks").configure(tasks::configure))
         .service(web::resource("/health").route(web::get().to(get_health)))
+        .service(web::scope("/logs").configure(logs::configure))
         .service(web::scope("/keys").configure(api_key::configure))
         .service(web::scope("/dumps").configure(dump::configure))
         .service(web::scope("/snapshots").configure(snapshot::configure))
@@ -43,6 +46,56 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         .service(web::scope("/experimental-features").configure(features::configure));
 }
 
+pub fn get_task_id(req: &HttpRequest, opt: &Opt) -> Result<Option<TaskId>, ResponseError> {
+    if !opt.experimental_replication_parameters {
+        return Ok(None);
+    }
+    let task_id = req
+        .headers()
+        .get("TaskId")
+        .map(|header| {
+            header.to_str().map_err(|e| {
+                ResponseError::from_msg(
+                    format!("TaskId is not a valid utf-8 string: {e}"),
+                    Code::BadRequest,
+                )
+            })
+        })
+        .transpose()?
+        .map(|s| {
+            s.parse::<TaskId>().map_err(|e| {
+                ResponseError::from_msg(
+                    format!(
+                        "Could not parse the TaskId as a {}: {e}",
+                        std::any::type_name::<TaskId>(),
+                    ),
+                    Code::BadRequest,
+                )
+            })
+        })
+        .transpose()?;
+    Ok(task_id)
+}
+
+pub fn is_dry_run(req: &HttpRequest, opt: &Opt) -> Result<bool, ResponseError> {
+    if !opt.experimental_replication_parameters {
+        return Ok(false);
+    }
+    Ok(req
+        .headers()
+        .get("DryRun")
+        .map(|header| {
+            header.to_str().map_err(|e| {
+                ResponseError::from_msg(
+                    format!("DryRun is not a valid utf-8 string: {e}"),
+                    Code::BadRequest,
+                )
+            })
+        })
+        .transpose()?
+        .map_or(false, |s| s.to_lowercase() == "true"))
+}
+
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct SummarizedTaskView {
@@ -250,7 +303,7 @@ async fn get_stats(
 
     let stats = create_all_stats((*index_scheduler).clone(), (*auth_controller).clone(), filters)?;
 
-    debug!("returns: {:?}", stats);
+    debug!(returns = ?stats, "Get stats");
     Ok(HttpResponse::Ok().json(stats))
 }
 
@@ -306,12 +359,18 @@ async fn get_version(
 ) -> HttpResponse {
     analytics.publish("Version Seen".to_string(), json!(null), Some(&req));
 
-    let commit_sha = option_env!("VERGEN_GIT_SHA").unwrap_or("unknown");
-    let commit_date = option_env!("VERGEN_GIT_COMMIT_TIMESTAMP").unwrap_or("unknown");
+    let build_info = build_info::BuildInfo::from_build();
 
     HttpResponse::Ok().json(VersionResponse {
-        commit_sha: commit_sha.to_string(),
-        commit_date: commit_date.to_string(),
+        commit_sha: build_info.commit_sha1.unwrap_or("unknown").to_string(),
+        commit_date: build_info
+            .commit_timestamp
+            .and_then(|commit_timestamp| {
+                commit_timestamp
+                    .format(&time::format_description::well_known::Iso8601::DEFAULT)
+                    .ok()
+            })
+            .unwrap_or("unknown".into()),
         pkg_version: env!("CARGO_PKG_VERSION").to_string(),
     })
 }

meilisearch/src/routes/multi_search.rs

@@ -3,11 +3,11 @@ use actix_web::web::{self, Data};
 use actix_web::{HttpRequest, HttpResponse};
 use deserr::actix_web::AwebJson;
 use index_scheduler::IndexScheduler;
-use log::debug;
 use meilisearch_types::deserr::DeserrJsonError;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::keys::actions;
 use serde::Serialize;
+use tracing::debug;
 
 use crate::analytics::{Analytics, MultiSearchAggregator};
 use crate::extractors::authentication::policies::ActionPolicy;
@@ -52,7 +52,7 @@ pub async fn multi_search_with_post(
         for (query_index, (index_uid, mut query)) in
             queries.into_iter().map(SearchQueryWithIndex::into_index_query).enumerate()
         {
-            debug!("multi-search #{query_index}: called with params: {:?}", query);
+            debug!(on_index = query_index, parameters = ?query, "Multi-search");
 
             // Check index from API key
             if !index_scheduler.filters().is_index_authorized(&index_uid) {
@@ -107,7 +107,7 @@ pub async fn multi_search_with_post(
         err
     })?;
 
-    debug!("returns: {:?}", search_results);
+    debug!(returns = ?search_results, "Multi-search");
 
     Ok(HttpResponse::Ok().json(SearchResults { results: search_results }))
 }

meilisearch/src/routes/snapshot.rs

@@ -1,16 +1,17 @@
 use actix_web::web::Data;
 use actix_web::{web, HttpRequest, HttpResponse};
 use index_scheduler::IndexScheduler;
-use log::debug;
 use meilisearch_types::error::ResponseError;
 use meilisearch_types::tasks::KindWithContent;
 use serde_json::json;
+use tracing::debug;
 
 use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::extractors::sequential_extractor::SeqHandler;
-use crate::routes::SummarizedTaskView;
+use crate::routes::{get_task_id, is_dry_run, SummarizedTaskView};
+use crate::Opt;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(web::resource("").route(web::post().to(SeqHandler(create_snapshot))));
@@ -19,14 +20,19 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
 pub async fn create_snapshot(
     index_scheduler: GuardedData<ActionPolicy<{ actions::SNAPSHOTS_CREATE }>, Data<IndexScheduler>>,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     analytics.publish("Snapshot Created".to_string(), json!({}), Some(&req));
 
     let task = KindWithContent::SnapshotCreation;
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
 
-    debug!("returns: {:?}", task);
+    debug!(returns = ?task, "Create snapshot");
     Ok(HttpResponse::Accepted().json(task))
 }

meilisearch/src/routes/swap_indexes.rs

@@ -10,12 +10,13 @@ use meilisearch_types::index_uid::IndexUid;
 use meilisearch_types::tasks::{IndexSwap, KindWithContent};
 use serde_json::json;
 
-use super::SummarizedTaskView;
+use super::{get_task_id, is_dry_run, SummarizedTaskView};
 use crate::analytics::Analytics;
 use crate::error::MeilisearchHttpError;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::{AuthenticationError, GuardedData};
 use crate::extractors::sequential_extractor::SeqHandler;
+use crate::Opt;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(web::resource("").route(web::post().to(SeqHandler(swap_indexes))));
@@ -32,6 +33,7 @@ pub async fn swap_indexes(
     index_scheduler: GuardedData<ActionPolicy<{ actions::INDEXES_SWAP }>, Data<IndexScheduler>>,
     params: AwebJson<Vec<SwapIndexesPayload>, DeserrJsonError>,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let params = params.into_inner();
@@ -60,7 +62,11 @@ pub async fn swap_indexes(
     }
 
     let task = KindWithContent::IndexSwap { swaps };
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
     let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+        tokio::task::spawn_blocking(move || index_scheduler.register(task, uid, dry_run))
+            .await??
+            .into();
     Ok(HttpResponse::Accepted().json(task))
 }

meilisearch/src/routes/tasks.rs

@@ -18,11 +18,12 @@ use time::macros::format_description;
 use time::{Date, Duration, OffsetDateTime, Time};
 use tokio::task;
 
-use super::SummarizedTaskView;
+use super::{get_task_id, is_dry_run, SummarizedTaskView};
 use crate::analytics::Analytics;
 use crate::extractors::authentication::policies::*;
 use crate::extractors::authentication::GuardedData;
 use crate::extractors::sequential_extractor::SeqHandler;
+use crate::Opt;
 
 const DEFAULT_LIMIT: u32 = 20;
 
@@ -161,6 +162,7 @@ async fn cancel_tasks(
     index_scheduler: GuardedData<ActionPolicy<{ actions::TASKS_CANCEL }>, Data<IndexScheduler>>,
     params: AwebQueryParameter<TaskDeletionOrCancelationQuery, DeserrQueryParamError>,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let params = params.into_inner();
@@ -197,7 +199,11 @@ async fn cancel_tasks(
     let task_cancelation =
         KindWithContent::TaskCancelation { query: format!("?{}", req.query_string()), tasks };
 
-    let task = task::spawn_blocking(move || index_scheduler.register(task_cancelation)).await??;
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
+    let task =
+        task::spawn_blocking(move || index_scheduler.register(task_cancelation, uid, dry_run))
+            .await??;
     let task: SummarizedTaskView = task.into();
 
     Ok(HttpResponse::Ok().json(task))
@@ -207,6 +213,7 @@ async fn delete_tasks(
     index_scheduler: GuardedData<ActionPolicy<{ actions::TASKS_DELETE }>, Data<IndexScheduler>>,
     params: AwebQueryParameter<TaskDeletionOrCancelationQuery, DeserrQueryParamError>,
     req: HttpRequest,
+    opt: web::Data<Opt>,
     analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let params = params.into_inner();
@@ -242,7 +249,10 @@ async fn delete_tasks(
     let task_deletion =
         KindWithContent::TaskDeletion { query: format!("?{}", req.query_string()), tasks };
 
-    let task = task::spawn_blocking(move || index_scheduler.register(task_deletion)).await??;
+    let uid = get_task_id(&req, &opt)?;
+    let dry_run = is_dry_run(&req, &opt)?;
+    let task = task::spawn_blocking(move || index_scheduler.register(task_deletion, uid, dry_run))
+        .await??;
     let task: SummarizedTaskView = task.into();
 
     Ok(HttpResponse::Ok().json(task))

meilisearch/src/search.rs

@@ -441,10 +441,6 @@ fn prepare_search<'t>(
         ScoringStrategy::Skip
     });
 
-    if query.show_ranking_score_details {
-        features.check_score_details()?;
-    }
-
     if let Some(HybridQuery { embedder: Some(embedder), .. }) = &query.hybrid {
         search.embedder_name(embedder);
     }

meilisearch/tests/auth/authorization.rs

@@ -59,6 +59,8 @@ pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'
             ("POST",    "/snapshots") =>                                       hashset!{"snapshots.create", "snapshots.*", "*"},
             ("GET",     "/version") =>                                         hashset!{"version", "*"},
             ("GET",     "/metrics") =>                                         hashset!{"metrics.get", "metrics.*", "*"},
+            ("POST",    "/logs/stream") =>                                     hashset!{"metrics.get", "metrics.*", "*"},
+            ("DELETE",  "/logs/stream") =>                                     hashset!{"metrics.get", "metrics.*", "*"},
             ("PATCH",   "/keys/mykey/") =>                                     hashset!{"keys.update", "*"},
             ("GET",     "/keys/mykey/") =>                                     hashset!{"keys.get", "*"},
             ("DELETE",  "/keys/mykey/") =>                                     hashset!{"keys.delete", "*"},

meilisearch/tests/common/index.rs

@@ -100,16 +100,11 @@ impl Index<'_> {
     pub async fn raw_add_documents(
         &self,
         payload: &str,
-        content_type: Option<&str>,
+        headers: Vec<(&str, &str)>,
         query_parameter: &str,
     ) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/documents{}", urlencode(self.uid.as_ref()), query_parameter);
-
-        if let Some(content_type) = content_type {
-            self.service.post_str(url, payload, vec![("Content-Type", content_type)]).await
-        } else {
-            self.service.post_str(url, payload, Vec::new()).await
-        }
+        self.service.post_str(url, payload, headers).await
     }
 
     pub async fn update_documents(

meilisearch/tests/common/mod.rs

@@ -64,7 +64,7 @@ impl Display for Value {
         write!(
             f,
             "{}",
-            json_string!(self, { ".enqueuedAt" => "[date]", ".processedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" })
+            json_string!(self, { ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" })
         )
     }
 }

meilisearch/tests/common/server.rs

@@ -9,10 +9,12 @@ use actix_web::http::StatusCode;
 use byte_unit::{Byte, ByteUnit};
 use clap::Parser;
 use meilisearch::option::{IndexerOpts, MaxMemory, Opt};
-use meilisearch::{analytics, create_app, setup_meilisearch};
+use meilisearch::{analytics, create_app, setup_meilisearch, SubscriberForSecondLayer};
 use once_cell::sync::Lazy;
 use tempfile::TempDir;
 use tokio::time::sleep;
+use tracing::level_filters::LevelFilter;
+use tracing_subscriber::Layer;
 
 use super::index::Index;
 use super::service::Service;
@@ -81,10 +83,24 @@ impl Server {
         Response = ServiceResponse<impl MessageBody>,
         Error = actix_web::Error,
     > {
+        let (_route_layer, route_layer_handle) =
+            tracing_subscriber::reload::Layer::new(None.with_filter(
+                tracing_subscriber::filter::Targets::new().with_target("", LevelFilter::OFF),
+            ));
+        let (_stderr_layer, stderr_layer_handle) = tracing_subscriber::reload::Layer::new(
+            (Box::new(
+                tracing_subscriber::fmt::layer()
+                    .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE),
+            )
+                as Box<dyn tracing_subscriber::Layer<SubscriberForSecondLayer> + Send + Sync>)
+                .with_filter(tracing_subscriber::filter::Targets::new()),
+        );
+
         actix_web::test::init_service(create_app(
             self.service.index_scheduler.clone().into(),
             self.service.auth.clone().into(),
             self.service.options.clone(),
+            (route_layer_handle, stderr_layer_handle),
             analytics::MockAnalytics::new(&self.service.options),
             true,
         ))

meilisearch/tests/common/service.rs

@@ -5,8 +5,10 @@ use actix_web::http::StatusCode;
 use actix_web::test;
 use actix_web::test::TestRequest;
 use index_scheduler::IndexScheduler;
-use meilisearch::{analytics, create_app, Opt};
+use meilisearch::{analytics, create_app, Opt, SubscriberForSecondLayer};
 use meilisearch_auth::AuthController;
+use tracing::level_filters::LevelFilter;
+use tracing_subscriber::Layer;
 
 use crate::common::encoder::Encoder;
 use crate::common::Value;
@@ -105,10 +107,24 @@ impl Service {
     }
 
     pub async fn request(&self, mut req: test::TestRequest) -> (Value, StatusCode) {
+        let (_route_layer, route_layer_handle) =
+            tracing_subscriber::reload::Layer::new(None.with_filter(
+                tracing_subscriber::filter::Targets::new().with_target("", LevelFilter::OFF),
+            ));
+        let (_stderr_layer, stderr_layer_handle) = tracing_subscriber::reload::Layer::new(
+            (Box::new(
+                tracing_subscriber::fmt::layer()
+                    .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE),
+            )
+                as Box<dyn tracing_subscriber::Layer<SubscriberForSecondLayer> + Send + Sync>)
+                .with_filter(tracing_subscriber::filter::Targets::new()),
+        );
+
         let app = test::init_service(create_app(
             self.index_scheduler.clone().into(),
             self.auth.clone().into(),
             self.options.clone(),
+            (route_layer_handle, stderr_layer_handle),
             analytics::MockAnalytics::new(&self.options),
             true,
         ))

meilisearch/tests/documents/add_documents.rs

@@ -1,10 +1,11 @@
 use actix_web::test;
 use meili_snap::{json_string, snapshot};
+use meilisearch::Opt;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::encoder::Encoder;
-use crate::common::{GetAllDocumentsOptions, Server, Value};
+use crate::common::{default_settings, GetAllDocumentsOptions, Server, Value};
 use crate::json;
 
 /// This is the basic usage of our API and every other tests uses the content-type application/json
@@ -1760,6 +1761,181 @@ async fn add_documents_invalid_geo_field() {
       "finishedAt": "[date]"
     }
     "###);
+
+    // The three next tests are related to #4333
+
+    // _geo has a lat and lng but set to `null`
+    let documents = json!([
+        {
+            "id": "12",
+            "_geo": { "lng": null, "lat": 67}
+        }
+    ]);
+
+    let (response, code) = index.add_documents(documents, None).await;
+    snapshot!(code, @"202 Accepted");
+    let response = index.wait_task(response.uid()).await;
+    snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]" }),
+        @r###"
+    {
+      "uid": 14,
+      "indexUid": "test",
+      "status": "failed",
+      "type": "documentAdditionOrUpdate",
+      "canceledBy": null,
+      "details": {
+        "receivedDocuments": 1,
+        "indexedDocuments": 0
+      },
+      "error": {
+        "message": "Could not parse longitude in the document with the id: `12`. Was expecting a finite number but instead got `null`.",
+        "code": "invalid_document_geo_field",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_document_geo_field"
+      },
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+
+    // _geo has a lat and lng but set to `null`
+    let documents = json!([
+        {
+            "id": "12",
+            "_geo": { "lng": 35, "lat": null }
+        }
+    ]);
+
+    let (response, code) = index.add_documents(documents, None).await;
+    snapshot!(code, @"202 Accepted");
+    let response = index.wait_task(response.uid()).await;
+    snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]" }),
+        @r###"
+    {
+      "uid": 15,
+      "indexUid": "test",
+      "status": "failed",
+      "type": "documentAdditionOrUpdate",
+      "canceledBy": null,
+      "details": {
+        "receivedDocuments": 1,
+        "indexedDocuments": 0
+      },
+      "error": {
+        "message": "Could not parse latitude in the document with the id: `12`. Was expecting a finite number but instead got `null`.",
+        "code": "invalid_document_geo_field",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_document_geo_field"
+      },
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+
+    // _geo has a lat and lng but set to `null`
+    let documents = json!([
+        {
+            "id": "13",
+            "_geo": { "lng": null, "lat": null }
+        }
+    ]);
+
+    let (response, code) = index.add_documents(documents, None).await;
+    snapshot!(code, @"202 Accepted");
+    let response = index.wait_task(response.uid()).await;
+    snapshot!(json_string!(response, { ".duration" => "[duration]", ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]" }),
+        @r###"
+    {
+      "uid": 16,
+      "indexUid": "test",
+      "status": "failed",
+      "type": "documentAdditionOrUpdate",
+      "canceledBy": null,
+      "details": {
+        "receivedDocuments": 1,
+        "indexedDocuments": 0
+      },
+      "error": {
+        "message": "Could not parse latitude nor longitude in the document with the id: `13`. Was expecting finite numbers but instead got `null` and `null`.",
+        "code": "invalid_document_geo_field",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_document_geo_field"
+      },
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+}
+
+// Related to #4333
+#[actix_rt::test]
+async fn add_invalid_geo_and_then_settings() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(Some("id")).await;
+
+    // _geo is not an object
+    let documents = json!([
+        {
+            "id": "11",
+            "_geo": { "lat": null, "lng": null },
+        }
+    ]);
+    let (ret, code) = index.add_documents(documents, None).await;
+    snapshot!(code, @"202 Accepted");
+    let ret = index.wait_task(ret.uid()).await;
+    snapshot!(ret, @r###"
+    {
+      "uid": 1,
+      "indexUid": "test",
+      "status": "succeeded",
+      "type": "documentAdditionOrUpdate",
+      "canceledBy": null,
+      "details": {
+        "receivedDocuments": 1,
+        "indexedDocuments": 1
+      },
+      "error": null,
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+
+    let (ret, code) = index.update_settings(json!({"sortableAttributes": ["_geo"]})).await;
+    snapshot!(code, @"202 Accepted");
+    let ret = index.wait_task(ret.uid()).await;
+    snapshot!(ret, @r###"
+    {
+      "uid": 2,
+      "indexUid": "test",
+      "status": "failed",
+      "type": "settingsUpdate",
+      "canceledBy": null,
+      "details": {
+        "sortableAttributes": [
+          "_geo"
+        ]
+      },
+      "error": {
+        "message": "Could not parse latitude in the document with the id: `\"11\"`. Was expecting a finite number but instead got `null`.",
+        "code": "invalid_document_geo_field",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_document_geo_field"
+      },
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
 }
 
 #[actix_rt::test]
@@ -1982,3 +2158,49 @@ async fn batch_several_documents_addition() {
     assert_eq!(code, 200, "failed with `{}`", response);
     assert_eq!(response["results"].as_array().unwrap().len(), 120);
 }
+
+#[actix_rt::test]
+async fn dry_register_file() {
+    let temp = tempfile::tempdir().unwrap();
+
+    let options =
+        Opt { experimental_replication_parameters: true, ..default_settings(temp.path()) };
+    let server = Server::new_with_options(options).await.unwrap();
+    let index = server.index("tamo");
+
+    let documents = r#"
+        {
+            "id": "12",
+            "doggo": "kefir"
+        }
+    "#;
+
+    let (response, code) = index
+        .raw_add_documents(
+            documents,
+            vec![("Content-Type", "application/json"), ("DryRun", "true")],
+            "",
+        )
+        .await;
+    snapshot!(response, @r###"
+    {
+      "taskUid": 0,
+      "indexUid": "tamo",
+      "status": "enqueued",
+      "type": "documentAdditionOrUpdate",
+      "enqueuedAt": "[date]"
+    }
+    "###);
+    snapshot!(code, @"202 Accepted");
+
+    let (response, code) = index.get_task(response.uid()).await;
+    snapshot!(response, @r###"
+    {
+      "message": "Task `0` not found.",
+      "code": "task_not_found",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#task_not_found"
+    }
+    "###);
+    snapshot!(code, @"404 Not Found");
+}

meilisearch/tests/documents/errors.rs

@@ -209,7 +209,8 @@ async fn replace_documents_missing_payload() {
     let server = Server::new().await;
     let index = server.index("test");
 
-    let (response, code) = index.raw_add_documents("", Some("application/json"), "").await;
+    let (response, code) =
+        index.raw_add_documents("", vec![("Content-Type", "application/json")], "").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -220,7 +221,8 @@ async fn replace_documents_missing_payload() {
     }
     "###);
 
-    let (response, code) = index.raw_add_documents("", Some("application/x-ndjson"), "").await;
+    let (response, code) =
+        index.raw_add_documents("", vec![("Content-Type", "application/x-ndjson")], "").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -231,7 +233,8 @@ async fn replace_documents_missing_payload() {
     }
     "###);
 
-    let (response, code) = index.raw_add_documents("", Some("text/csv"), "").await;
+    let (response, code) =
+        index.raw_add_documents("", vec![("Content-Type", "text/csv")], "").await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -287,7 +290,7 @@ async fn replace_documents_missing_content_type() {
     let server = Server::new().await;
     let index = server.index("test");
 
-    let (response, code) = index.raw_add_documents("", None, "").await;
+    let (response, code) = index.raw_add_documents("", Vec::new(), "").await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
     {
@@ -299,7 +302,7 @@ async fn replace_documents_missing_content_type() {
     "###);
 
     // even with a csv delimiter specified this error is triggered first
-    let (response, code) = index.raw_add_documents("", None, "?csvDelimiter=;").await;
+    let (response, code) = index.raw_add_documents("", Vec::new(), "?csvDelimiter=;").await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
     {
@@ -345,7 +348,7 @@ async fn replace_documents_bad_content_type() {
     let server = Server::new().await;
     let index = server.index("test");
 
-    let (response, code) = index.raw_add_documents("", Some("doggo"), "").await;
+    let (response, code) = index.raw_add_documents("", vec![("Content-Type", "doggo")], "").await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
     {
@@ -379,8 +382,9 @@ async fn replace_documents_bad_csv_delimiter() {
     let server = Server::new().await;
     let index = server.index("test");
 
-    let (response, code) =
-        index.raw_add_documents("", Some("application/json"), "?csvDelimiter").await;
+    let (response, code) = index
+        .raw_add_documents("", vec![("Content-Type", "application/json")], "?csvDelimiter")
+        .await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -391,8 +395,9 @@ async fn replace_documents_bad_csv_delimiter() {
     }
     "###);
 
-    let (response, code) =
-        index.raw_add_documents("", Some("application/json"), "?csvDelimiter=doggo").await;
+    let (response, code) = index
+        .raw_add_documents("", vec![("Content-Type", "application/json")], "?csvDelimiter=doggo")
+        .await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
@@ -404,7 +409,11 @@ async fn replace_documents_bad_csv_delimiter() {
     "###);
 
     let (response, code) = index
-        .raw_add_documents("", Some("application/json"), &format!("?csvDelimiter={}", encode("🍰")))
+        .raw_add_documents(
+            "",
+            vec![("Content-Type", "application/json")],
+            &format!("?csvDelimiter={}", encode("🍰")),
+        )
         .await;
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
@@ -469,8 +478,9 @@ async fn replace_documents_csv_delimiter_with_bad_content_type() {
     let server = Server::new().await;
     let index = server.index("test");
 
-    let (response, code) =
-        index.raw_add_documents("", Some("application/json"), "?csvDelimiter=a").await;
+    let (response, code) = index
+        .raw_add_documents("", vec![("Content-Type", "application/json")], "?csvDelimiter=a")
+        .await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
     {
@@ -481,8 +491,9 @@ async fn replace_documents_csv_delimiter_with_bad_content_type() {
     }
     "###);
 
-    let (response, code) =
-        index.raw_add_documents("", Some("application/x-ndjson"), "?csvDelimiter=a").await;
+    let (response, code) = index
+        .raw_add_documents("", vec![("Content-Type", "application/x-ndjson")], "?csvDelimiter=a")
+        .await;
     snapshot!(code, @"415 Unsupported Media Type");
     snapshot!(json_string!(response), @r###"
     {

meilisearch/tests/documents/update_documents.rs

@@ -1,4 +1,4 @@
-use meili_snap::snapshot;
+use meili_snap::{json_string, snapshot};
 
 use crate::common::encoder::Encoder;
 use crate::common::{GetAllDocumentsOptions, Server};
@@ -209,3 +209,93 @@ async fn error_update_documents_missing_document_id() {
         "https://docs.meilisearch.com/errors#missing_document_id"
     );
 }
+
+#[actix_rt::test]
+async fn update_faceted_document() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let (response, code) = index
+        .update_settings(json!({
+            "rankingRules": ["facet:asc"],
+        }))
+        .await;
+    assert_eq!("202", code.as_str(), "{:?}", response);
+    index.wait_task(0).await;
+
+    let documents: Vec<_> = (0..1000)
+        .map(|id| {
+            json!({
+                "doc_id": id,
+                "facet": (id/3),
+            })
+        })
+        .collect();
+
+    let (_response, code) = index.add_documents(documents.into(), None).await;
+    assert_eq!(code, 202);
+
+    index.wait_task(1).await;
+
+    let documents = json!([
+        {
+            "doc_id": 9,
+            "facet": 1.5,
+        }
+    ]);
+
+    let (response, code) = index.update_documents(documents, None).await;
+    assert_eq!(code, 202, "response: {}", response);
+
+    index.wait_task(2).await;
+
+    index
+        .search(json!({"limit": 10}), |response, code| {
+            snapshot!(code, @"200 OK");
+            snapshot!(json_string!(response["hits"]), @r###"
+            [
+              {
+                "doc_id": 0,
+                "facet": 0
+              },
+              {
+                "doc_id": 1,
+                "facet": 0
+              },
+              {
+                "doc_id": 2,
+                "facet": 0
+              },
+              {
+                "doc_id": 3,
+                "facet": 1
+              },
+              {
+                "doc_id": 4,
+                "facet": 1
+              },
+              {
+                "doc_id": 5,
+                "facet": 1
+              },
+              {
+                "doc_id": 9,
+                "facet": 1.5
+              },
+              {
+                "doc_id": 6,
+                "facet": 2
+              },
+              {
+                "doc_id": 7,
+                "facet": 2
+              },
+              {
+                "doc_id": 8,
+                "facet": 2
+              }
+            ]
+            "###);
+        })
+        .await;
+}

meilisearch/tests/dumps/mod.rs

@@ -1845,9 +1845,9 @@ async fn import_dump_v6_containing_experimental_features() {
     meili_snap::snapshot!(code, @"200 OK");
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
-      "scoreDetails": false,
       "vectorStore": false,
       "metrics": false,
+      "logsRoute": false,
       "exportPuffinReports": false
     }
     "###);

meilisearch/tests/features/mod.rs

@@ -18,9 +18,9 @@ async fn experimental_features() {
     meili_snap::snapshot!(code, @"200 OK");
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
-      "scoreDetails": false,
       "vectorStore": false,
       "metrics": false,
+      "logsRoute": false,
       "exportPuffinReports": false
     }
     "###);
@@ -30,9 +30,9 @@ async fn experimental_features() {
     meili_snap::snapshot!(code, @"200 OK");
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
-      "scoreDetails": false,
       "vectorStore": true,
       "metrics": false,
+      "logsRoute": false,
       "exportPuffinReports": false
     }
     "###);
@@ -42,9 +42,9 @@ async fn experimental_features() {
     meili_snap::snapshot!(code, @"200 OK");
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
-      "scoreDetails": false,
       "vectorStore": true,
       "metrics": false,
+      "logsRoute": false,
       "exportPuffinReports": false
     }
     "###);
@@ -55,9 +55,9 @@ async fn experimental_features() {
     meili_snap::snapshot!(code, @"200 OK");
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
-      "scoreDetails": false,
       "vectorStore": true,
       "metrics": false,
+      "logsRoute": false,
       "exportPuffinReports": false
     }
     "###);
@@ -68,9 +68,9 @@ async fn experimental_features() {
     meili_snap::snapshot!(code, @"200 OK");
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
-      "scoreDetails": false,
       "vectorStore": true,
       "metrics": false,
+      "logsRoute": false,
       "exportPuffinReports": false
     }
     "###);
@@ -88,9 +88,9 @@ async fn experimental_feature_metrics() {
     meili_snap::snapshot!(code, @"200 OK");
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
-      "scoreDetails": false,
       "vectorStore": false,
       "metrics": true,
+      "logsRoute": false,
       "exportPuffinReports": false
     }
     "###);
@@ -146,7 +146,7 @@ async fn errors() {
     meili_snap::snapshot!(code, @"400 Bad Request");
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
-      "message": "Unknown field `NotAFeature`: expected one of `scoreDetails`, `vectorStore`, `metrics`, `exportPuffinReports`",
+      "message": "Unknown field `NotAFeature`: expected one of `vectorStore`, `metrics`, `logsRoute`, `exportPuffinReports`",
       "code": "bad_request",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#bad_request"

meilisearch/tests/index/create_index.rs

@@ -2,9 +2,10 @@ use actix_web::http::header::ContentType;
 use actix_web::test;
 use http::header::ACCEPT_ENCODING;
 use meili_snap::{json_string, snapshot};
+use meilisearch::Opt;
 
 use crate::common::encoder::Encoder;
-use crate::common::{Server, Value};
+use crate::common::{default_settings, Server, Value};
 use crate::json;
 
 #[actix_rt::test]
@@ -199,3 +200,79 @@ async fn error_create_with_invalid_index_uid() {
     }
     "###);
 }
+
+#[actix_rt::test]
+async fn send_task_id() {
+    let temp = tempfile::tempdir().unwrap();
+
+    let options =
+        Opt { experimental_replication_parameters: true, ..default_settings(temp.path()) };
+    let server = Server::new_with_options(options).await.unwrap();
+
+    let app = server.init_web_app().await;
+    let index = server.index("catto");
+    let (response, code) = index.create(None).await;
+    snapshot!(code, @"202 Accepted");
+    snapshot!(json_string!(response, { ".enqueuedAt" => "[date]" }), @r###"
+    {
+      "taskUid": 0,
+      "indexUid": "catto",
+      "status": "enqueued",
+      "type": "indexCreation",
+      "enqueuedAt": "[date]"
+    }
+    "###);
+
+    let body = serde_json::to_string(&json!({
+        "uid": "doggo",
+        "primaryKey": None::<&str>,
+    }))
+    .unwrap();
+    let req = test::TestRequest::post()
+        .uri("/indexes")
+        .insert_header(("TaskId", "25"))
+        .insert_header(ContentType::json())
+        .set_payload(body)
+        .to_request();
+
+    let res = test::call_service(&app, req).await;
+    snapshot!(res.status(), @"202 Accepted");
+
+    let bytes = test::read_body(res).await;
+    let response = serde_json::from_slice::<Value>(&bytes).expect("Expecting valid json");
+    snapshot!(json_string!(response, { ".enqueuedAt" => "[date]" }), @r###"
+    {
+      "taskUid": 25,
+      "indexUid": "doggo",
+      "status": "enqueued",
+      "type": "indexCreation",
+      "enqueuedAt": "[date]"
+    }
+    "###);
+
+    let body = serde_json::to_string(&json!({
+        "uid": "girafo",
+        "primaryKey": None::<&str>,
+    }))
+    .unwrap();
+    let req = test::TestRequest::post()
+        .uri("/indexes")
+        .insert_header(("TaskId", "12"))
+        .insert_header(ContentType::json())
+        .set_payload(body)
+        .to_request();
+
+    let res = test::call_service(&app, req).await;
+    snapshot!(res.status(), @"400 Bad Request");
+
+    let bytes = test::read_body(res).await;
+    let response = serde_json::from_slice::<Value>(&bytes).expect("Expecting valid json");
+    snapshot!(json_string!(response), @r###"
+    {
+      "message": "Received bad task id: 12 should be >= to 26.",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+}

meilisearch/tests/integration.rs

@@ -5,6 +5,7 @@ mod documents;
 mod dumps;
 mod features;
 mod index;
+mod logs;
 mod search;
 mod settings;
 mod snapshot;

meilisearch/tests/logs/error.rs

@@ -0,0 +1,193 @@
+use meili_snap::*;
+
+use crate::common::Server;
+use crate::json;
+
+#[actix_rt::test]
+async fn logs_stream_bad_target() {
+    let server = Server::new().await;
+
+    // Wrong type
+    let (response, code) = server.service.post("/logs/stream", json!({ "target": true })).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Invalid value type at `.target`: expected a string, but found a boolean: `true`",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+
+    // Wrong type
+    let (response, code) = server.service.post("/logs/stream", json!({ "target": [] })).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Invalid value type at `.target`: expected a string, but found an array: `[]`",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+
+    // Our help message
+    let (response, code) = server.service.post("/logs/stream", json!({ "target": "" })).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Invalid value at `.target`: Empty string is not a valid target. If you want to get no logs use `OFF`. Usage: `info`, `meilisearch=info`, or you can write multiple filters in one target: `index_scheduler=info,milli=trace`",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+
+    // An error from the target parser
+    let (response, code) = server.service.post("/logs/stream", json!({ "target": "==" })).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Invalid value at `.target`: invalid filter directive: too many '=' in filter directive, expected 0 or 1",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+}
+
+#[actix_rt::test]
+async fn logs_stream_bad_mode() {
+    let server = Server::new().await;
+
+    // Wrong type
+    let (response, code) = server.service.post("/logs/stream", json!({ "mode": true })).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Invalid value type at `.mode`: expected a string, but found a boolean: `true`",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+
+    // Wrong type
+    let (response, code) = server.service.post("/logs/stream", json!({ "mode": [] })).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Invalid value type at `.mode`: expected a string, but found an array: `[]`",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+
+    // Wrong value
+    let (response, code) = server.service.post("/logs/stream", json!({ "mode": "tamo" })).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Unknown value `tamo` at `.mode`: expected one of `human`, `json`, `profile`",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+}
+
+#[actix_rt::test]
+async fn logs_stream_bad_profile_memory() {
+    let server = Server::new().await;
+
+    // Wrong type
+    let (response, code) =
+        server.service.post("/logs/stream", json!({ "profileMemory": "tamo" })).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Invalid value type at `.profileMemory`: expected a boolean, but found a string: `\"tamo\"`",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+
+    // Wrong type
+    let (response, code) =
+        server.service.post("/logs/stream", json!({ "profileMemory": ["hello", "kefir"] })).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Invalid value type at `.profileMemory`: expected a boolean, but found an array: `[\"hello\",\"kefir\"]`",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+
+    // Used with default parameters
+    let (response, code) =
+        server.service.post("/logs/stream", json!({ "profileMemory": true })).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Invalid value: `profile_memory` can only be used while profiling code and is not compatible with the Human mode.",
+      "code": "invalid_settings_typo_tolerance",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#invalid_settings_typo_tolerance"
+    }
+    "###);
+
+    // Used with an unsupported mode
+    let (response, code) =
+        server.service.post("/logs/stream", json!({ "mode": "fmt", "profileMemory": true })).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Unknown value `fmt` at `.mode`: expected one of `human`, `json`, `profile`",
+      "code": "bad_request",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#bad_request"
+    }
+    "###);
+}
+
+#[actix_rt::test]
+async fn logs_stream_without_enabling_the_route() {
+    let server = Server::new().await;
+
+    let (response, code) = server.service.post("/logs/stream", json!({})).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Modifying logs through the `/logs/*` routes requires enabling the `logs route` experimental feature. See https://github.com/orgs/meilisearch/discussions/721",
+      "code": "feature_not_enabled",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
+    }
+    "###);
+
+    let (response, code) = server.service.delete("/logs/stream").await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Modifying logs through the `/logs/*` routes requires enabling the `logs route` experimental feature. See https://github.com/orgs/meilisearch/discussions/721",
+      "code": "feature_not_enabled",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
+    }
+    "###);
+
+    let (response, code) = server.service.post("/logs/stderr", json!({})).await;
+    snapshot!(code, @"400 Bad Request");
+    snapshot!(response, @r###"
+    {
+      "message": "Modifying logs through the `/logs/*` routes requires enabling the `logs route` experimental feature. See https://github.com/orgs/meilisearch/discussions/721",
+      "code": "feature_not_enabled",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
+    }
+    "###);
+}

meilisearch/tests/logs/mod.rs

@@ -0,0 +1,99 @@
+mod error;
+
+use std::rc::Rc;
+use std::str::FromStr;
+
+use actix_web::http::header::ContentType;
+use meili_snap::snapshot;
+use meilisearch::{analytics, create_app, Opt, SubscriberForSecondLayer};
+use tracing::level_filters::LevelFilter;
+use tracing_subscriber::layer::SubscriberExt;
+use tracing_subscriber::Layer;
+
+use crate::common::{default_settings, Server};
+use crate::json;
+
+#[actix_web::test]
+async fn basic_test_log_stream_route() {
+    let db_path = tempfile::tempdir().unwrap();
+    let server = Server::new_with_options(Opt {
+        experimental_enable_logs_route: true,
+        ..default_settings(db_path.path())
+    })
+    .await
+    .unwrap();
+
+    let (route_layer, route_layer_handle) =
+        tracing_subscriber::reload::Layer::new(None.with_filter(
+            tracing_subscriber::filter::Targets::new().with_target("", LevelFilter::OFF),
+        ));
+    let (_stderr_layer, stderr_layer_handle) = tracing_subscriber::reload::Layer::new(
+        (Box::new(
+            tracing_subscriber::fmt::layer()
+                .with_span_events(tracing_subscriber::fmt::format::FmtSpan::CLOSE),
+        ) as Box<dyn tracing_subscriber::Layer<SubscriberForSecondLayer> + Send + Sync>)
+            .with_filter(tracing_subscriber::filter::Targets::new()),
+    );
+
+    let subscriber = tracing_subscriber::registry().with(route_layer).with(
+        tracing_subscriber::fmt::layer()
+            .with_span_events(tracing_subscriber::fmt::format::FmtSpan::ACTIVE)
+            .with_filter(tracing_subscriber::filter::LevelFilter::from_str("OFF").unwrap()),
+    );
+
+    let app = actix_web::test::init_service(create_app(
+        server.service.index_scheduler.clone().into(),
+        server.service.auth.clone().into(),
+        server.service.options.clone(),
+        (route_layer_handle, stderr_layer_handle),
+        analytics::MockAnalytics::new(&server.service.options),
+        true,
+    ))
+    .await;
+
+    // set the subscriber as the default for the application
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+
+    let app = Rc::new(app);
+
+    // First, we start listening on the `/logs/stream` route
+    let handle_app = app.clone();
+    let handle = tokio::task::spawn_local(async move {
+        let req = actix_web::test::TestRequest::post()
+            .uri("/logs/stream")
+            .insert_header(ContentType::json())
+            .set_payload(
+                serde_json::to_vec(&json!({
+                    "mode": "human",
+                    "target": "info",
+                }))
+                .unwrap(),
+            );
+        let req = req.to_request();
+        let ret = actix_web::test::call_service(&*handle_app, req).await;
+        actix_web::test::read_body(ret).await
+    });
+
+    // We're going to create an index to get at least one info log saying we processed a batch of task
+    let (ret, _code) = server.create_index(json!({ "uid": "tamo" })).await;
+    snapshot!(ret, @r###"
+    {
+      "taskUid": 0,
+      "indexUid": "tamo",
+      "status": "enqueued",
+      "type": "indexCreation",
+      "enqueuedAt": "[date]"
+    }
+    "###);
+    server.wait_task(ret.uid()).await;
+
+    let req = actix_web::test::TestRequest::delete().uri("/logs/stream");
+    let req = req.to_request();
+    let ret = actix_web::test::call_service(&*app, req).await;
+    let code = ret.status();
+    snapshot!(code, @"204 No Content");
+
+    let logs = handle.await.unwrap();
+    let logs = String::from_utf8(logs.to_vec()).unwrap();
+    assert!(logs.contains("INFO"), "{logs}");
+}

meilisearch/tests/search/hybrid.rs

@@ -13,9 +13,9 @@ async fn index_with_documents<'a>(server: &'a Server, documents: &Value) -> Inde
     meili_snap::snapshot!(code, @"200 OK");
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
-      "scoreDetails": false,
       "vectorStore": true,
       "metrics": false,
+      "logsRoute": false,
       "exportPuffinReports": false
     }
     "###);
@@ -87,6 +87,52 @@ async fn simple_search() {
     snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_semanticScore":0.99029034},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_semanticScore":0.97434163},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_semanticScore":0.9472136}]"###);
 }
 
+#[actix_rt::test]
+async fn highlighter() {
+    let server = Server::new().await;
+    let index = index_with_documents(&server, &SIMPLE_SEARCH_DOCUMENTS).await;
+
+    let (response, code) = index
+        .search_post(json!({"q": "Captain Marvel", "vector": [1.0, 1.0],
+            "hybrid": {"semanticRatio": 0.2},
+            "attributesToHighlight": [
+                     "desc"
+                   ],
+                   "highlightPreTag": "**BEGIN**",
+                   "highlightPostTag": "**END**"
+        }))
+        .await;
+    snapshot!(code, @"200 OK");
+    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":["2.0","3.0"]}}},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1","_vectors":{"default":["1.0","3.0"]}}},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2","_vectors":{"default":["1.0","2.0"]}}}]"###);
+
+    let (response, code) = index
+        .search_post(json!({"q": "Captain Marvel", "vector": [1.0, 1.0],
+            "hybrid": {"semanticRatio": 0.8},
+            "attributesToHighlight": [
+                     "desc"
+                   ],
+                   "highlightPreTag": "**BEGIN**",
+                   "highlightPostTag": "**END**"
+        }))
+        .await;
+    snapshot!(code, @"200 OK");
+    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":["2.0","3.0"]}},"_semanticScore":0.99029034},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_formatted":{"title":"Captain Planet","desc":"He's not part of the **BEGIN**Marvel**END** Cinematic Universe","id":"2","_vectors":{"default":["1.0","2.0"]}},"_semanticScore":0.97434163},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_formatted":{"title":"Shazam!","desc":"a **BEGIN**Captain**END** **BEGIN**Marvel**END** ersatz","id":"1","_vectors":{"default":["1.0","3.0"]}},"_semanticScore":0.9472136}]"###);
+
+    // no highlighting on full semantic
+    let (response, code) = index
+        .search_post(json!({"q": "Captain Marvel", "vector": [1.0, 1.0],
+            "hybrid": {"semanticRatio": 1.0},
+            "attributesToHighlight": [
+                     "desc"
+                   ],
+                   "highlightPreTag": "**BEGIN**",
+                   "highlightPostTag": "**END**"
+        }))
+        .await;
+    snapshot!(code, @"200 OK");
+    snapshot!(response["hits"], @r###"[{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":[2.0,3.0]},"_formatted":{"title":"Captain Marvel","desc":"a Shazam ersatz","id":"3","_vectors":{"default":["2.0","3.0"]}},"_semanticScore":0.99029034},{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":[1.0,2.0]},"_formatted":{"title":"Captain Planet","desc":"He's not part of the Marvel Cinematic Universe","id":"2","_vectors":{"default":["1.0","2.0"]}},"_semanticScore":0.97434163},{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":[1.0,3.0]},"_formatted":{"title":"Shazam!","desc":"a Captain Marvel ersatz","id":"1","_vectors":{"default":["1.0","3.0"]}}}]"###);
+}
+
 #[actix_rt::test]
 async fn invalid_semantic_ratio() {
     let server = Server::new().await;

meilisearch/tests/search/mod.rs

@@ -766,38 +766,14 @@ async fn faceting_max_values_per_facet() {
 }
 
 #[actix_rt::test]
-async fn experimental_feature_score_details() {
+async fn test_score_details() {
     let server = Server::new().await;
     let index = server.index("test");
 
     let documents = DOCUMENTS.clone();
 
-    index.add_documents(json!(documents), None).await;
-    index.wait_task(0).await;
-
-    index
-        .search(
-            json!({
-                "q": "train dragon",
-                "showRankingScoreDetails": true,
-            }),
-            |response, code| {
-                meili_snap::snapshot!(code, @"400 Bad Request");
-                meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
-                {
-                  "message": "Computing score details requires enabling the `score details` experimental feature. See https://github.com/meilisearch/product/discussions/674",
-                  "code": "feature_not_enabled",
-                  "type": "invalid_request",
-                  "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
-                }
-                "###);
-            },
-        )
-        .await;
-
-    let (response, code) = server.set_features(json!({"scoreDetails": true})).await;
-    meili_snap::snapshot!(code, @"200 OK");
-    meili_snap::snapshot!(response["scoreDetails"], @"true");
+    let res = index.add_documents(json!(documents), None).await;
+    index.wait_task(res.0.uid()).await;
 
     index
         .search(

meilisearch/tests/tasks/webhook.rs

@@ -7,7 +7,7 @@ use std::sync::Arc;
 use actix_http::body::MessageBody;
 use actix_web::dev::{ServiceFactory, ServiceResponse};
 use actix_web::web::{Bytes, Data};
-use actix_web::{post, App, HttpResponse, HttpServer};
+use actix_web::{post, App, HttpRequest, HttpResponse, HttpServer};
 use meili_snap::{json_string, snapshot};
 use meilisearch::Opt;
 use tokio::sync::mpsc;
@@ -17,7 +17,17 @@ use crate::common::{default_settings, Server};
 use crate::json;
 
 #[post("/")]
-async fn forward_body(sender: Data<mpsc::UnboundedSender<Vec<u8>>>, body: Bytes) -> HttpResponse {
+async fn forward_body(
+    req: HttpRequest,
+    sender: Data<mpsc::UnboundedSender<Vec<u8>>>,
+    body: Bytes,
+) -> HttpResponse {
+    let headers = req.headers();
+    assert_eq!(headers.get("content-type").unwrap(), "application/x-ndjson");
+    assert_eq!(headers.get("transfer-encoding").unwrap(), "chunked");
+    assert_eq!(headers.get("accept-encoding").unwrap(), "gzip");
+    assert_eq!(headers.get("content-encoding").unwrap(), "gzip");
+
     let body = body.to_vec();
     sender.send(body).unwrap();
     HttpResponse::Ok().into()
@@ -44,10 +54,6 @@ struct WebhookHandle {
 }
 
 async fn create_webhook_server() -> WebhookHandle {
-    let mut log_builder = env_logger::Builder::new();
-    log_builder.parse_filters("info");
-    log_builder.init();
-
     let (sender, receiver) = mpsc::unbounded_channel();
     let sender = Arc::new(sender);
 

meilitool/Cargo.toml

@@ -9,11 +9,11 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.75"
-clap = { version = "4.2.1", features = ["derive"] }
+anyhow = "1.0.79"
+clap = { version = "4.4.17", features = ["derive"] }
 dump = { path = "../dump" }
 file-store = { path = "../file-store" }
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-time = { version = "0.3.30", features = ["formatting"] }
-uuid = { version = "1.5.0", features = ["v4"], default-features = false }
+time = { version = "0.3.31", features = ["formatting"] }
+uuid = { version = "1.6.1", features = ["v4"], default-features = false }

meilitool/src/uuid_codec.rs

@@ -1,5 +1,4 @@
 use std::borrow::Cow;
-use std::convert::TryInto;
 
 use meilisearch_types::heed::{BoxedError, BytesDecode, BytesEncode};
 use uuid::Uuid;

milli/Cargo.toml

@@ -14,14 +14,14 @@ license.workspace = true
 [dependencies]
 bimap = { version = "0.6.3", features = ["serde"] }
 bincode = "1.3.3"
-bstr = "1.4.0"
-bytemuck = { version = "1.13.1", features = ["extern_crate_alloc"] }
-byteorder = "1.4.3"
-charabia = { version = "0.8.5", default-features = false }
+bstr = "1.9.0"
+bytemuck = { version = "1.14.0", features = ["extern_crate_alloc"] }
+byteorder = "1.5.0"
+charabia = { version = "0.8.7", default-features = false }
 concat-arrays = "0.1.2"
-crossbeam-channel = "0.5.8"
-deserr = "0.6.0"
-either = { version = "1.8.1", features = ["serde"] }
+crossbeam-channel = "0.5.11"
+deserr = "0.6.1"
+either = { version = "1.9.0", features = ["serde"] }
 flatten-serde-json = { path = "../flatten-serde-json" }
 fst = "0.4.7"
 fxhash = "0.2.1"
@@ -35,32 +35,32 @@ heed = { version = "0.20.0-alpha.9", default-features = false, features = [
     "serde-bincode",
     "read-txn-no-tls",
 ] }
-indexmap = { version = "2.0.0", features = ["serde"] }
+indexmap = { version = "2.1.0", features = ["serde"] }
 json-depth-checker = { path = "../json-depth-checker" }
 levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }
 memmap2 = "0.7.1"
-obkv = "0.2.0"
-once_cell = "1.17.1"
-ordered-float = "3.6.0"
+obkv = "0.2.1"
+once_cell = "1.19.0"
+ordered-float = "4.2.0"
 rand_pcg = { version = "0.3.1", features = ["serde1"] }
-rayon = "1.7.0"
-roaring = "0.10.1"
+rayon = "1.8.0"
+roaring = "0.10.2"
 rstar = { version = "0.11.0", features = ["serde"] }
-serde = { version = "1.0.160", features = ["derive"] }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
-slice-group-by = "0.3.0"
+serde = { version = "1.0.195", features = ["derive"] }
+serde_json = { version = "1.0.111", features = ["preserve_order"] }
+slice-group-by = "0.3.1"
 smallstr = { version = "0.3.0", features = ["serde"] }
-smallvec = "1.10.0"
+smallvec = "1.12.0"
 smartstring = "1.0.1"
-tempfile = "3.5.0"
-thiserror = "1.0.40"
-time = { version = "0.3.20", features = [
+tempfile = "3.9.0"
+thiserror = "1.0.56"
+time = { version = "0.3.31", features = [
     "serde-well-known",
     "formatting",
     "parsing",
     "macros",
 ] }
-uuid = { version = "1.3.1", features = ["v4"] }
+uuid = { version = "1.6.1", features = ["v4"] }
 
 filter-parser = { path = "../filter-parser" }
 
@@ -70,32 +70,32 @@ itertools = "0.11.0"
 # profiling
 puffin = "0.16.0"
 
-# logging
-log = "0.4.17"
-logging_timer = "1.1.0"
-csv = "1.2.1"
+csv = "1.3.0"
 candle-core = { git = "https://github.com/huggingface/candle.git", version = "0.3.1" }
 candle-transformers = { git = "https://github.com/huggingface/candle.git", version = "0.3.1" }
 candle-nn = { git = "https://github.com/huggingface/candle.git", version = "0.3.1" }
-tokenizers = { git = "https://github.com/huggingface/tokenizers.git", tag = "v0.14.1", version = "0.14.1", default_features = false, features = ["onig"] }
+tokenizers = { git = "https://github.com/huggingface/tokenizers.git", tag = "v0.14.1", version = "0.14.1", default_features = false, features = [
+    "onig",
+] }
 hf-hub = { git = "https://github.com/dureuill/hf-hub.git", branch = "rust_tls", default_features = false, features = [
     "online",
 ] }
-tokio = { version = "1.34.0", features = ["rt"] }
-futures = "0.3.29"
-reqwest = { version = "0.11.16", features = [
+tokio = { version = "1.35.1", features = ["rt"] }
+futures = "0.3.30"
+reqwest = { version = "0.11.23", features = [
     "rustls-tls",
     "json",
 ], default-features = false }
-tiktoken-rs = "0.5.7"
+tiktoken-rs = "0.5.8"
 liquid = "0.26.4"
-arroy = { git = "https://github.com/meilisearch/arroy.git", version = "0.1.0" }
+arroy = "0.2.0"
 rand = "0.8.5"
+tracing = "0.1.40"
 
 [dev-dependencies]
-mimalloc = { version = "0.1.37", default-features = false }
+mimalloc = { version = "0.1.39", default-features = false }
 big_s = "1.0.2"
-insta = "1.29.0"
+insta = "1.34.0"
 maplit = "1.0.2"
 md5 = "0.7.0"
 meili-snap = { path = "../meili-snap" }
@@ -110,6 +110,7 @@ all-tokenizations = [
     "charabia/korean",
     "charabia/greek",
     "charabia/khmer",
+    "charabia/vietnamese",
 ]
 
 # Use POSIX semaphores instead of SysV semaphores in LMDB
@@ -137,3 +138,8 @@ greek = ["charabia/greek"]
 
 # allow khmer specialized tokenization
 khmer = ["charabia/khmer"]
+
+vietnamese = ["charabia/vietnamese"]
+
+# allow CUDA support, see <https://github.com/meilisearch/meilisearch/issues/4306>
+cuda = ["candle-core/cuda"]

milli/src/documents/reader.rs

@@ -25,6 +25,7 @@ impl<R: io::Read + io::Seek> DocumentsBatchReader<R> {
     ///
     /// It first retrieves the index, then moves to the first document. Use the `into_cursor`
     /// method to iterator over the documents, from the first to the last.
+    #[tracing::instrument(level = "trace", skip_all, target = "indexing::documents")]
     pub fn from_reader(reader: R) -> Result<Self, Error> {
         let reader = grenad::Reader::new(reader)?;
         let mut cursor = reader.into_cursor()?;

milli/src/error.rs

@@ -172,7 +172,7 @@ only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and undersco
     TooManyDocumentIds { primary_key: String, document: Object },
     #[error("The primary key inference failed as the engine did not find any field ending with `id` in its name. Please specify the primary key manually using the `primaryKey` query parameter.")]
     NoPrimaryKeyCandidateFound,
-    #[error("The primary key inference failed as the engine found {} fields ending with `id` in their names: '{}' and '{}'. Please specify the primary key manually using the `primaryKey` query parameter.", .candidates.len(), .candidates.get(0).unwrap(), .candidates.get(1).unwrap())]
+    #[error("The primary key inference failed as the engine found {} fields ending with `id` in their names: '{}' and '{}'. Please specify the primary key manually using the `primaryKey` query parameter.", .candidates.len(), .candidates.first().unwrap(), .candidates.get(1).unwrap())]
     MultiplePrimaryKeyCandidatesFound { candidates: Vec<String> },
     #[error("There is no more space left on the device. Consider increasing the size of the disk/partition.")]
     NoSpaceLeftOnDevice,
@@ -227,6 +227,22 @@ only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and undersco
         source_: crate::vector::settings::EmbedderSource,
         embedder_name: String,
     },
+    #[error("`.embedders.{embedder_name}.dimensions`: Model `{model}` does not support overriding its native dimensions of {expected_dimensions}. Found {dimensions}")]
+    InvalidOpenAiModelDimensions {
+        embedder_name: String,
+        model: &'static str,
+        dimensions: usize,
+        expected_dimensions: usize,
+    },
+    #[error("`.embedders.{embedder_name}.dimensions`: Model `{model}` does not support overriding its dimensions to a value higher than {max_dimensions}. Found {dimensions}")]
+    InvalidOpenAiModelDimensionsMax {
+        embedder_name: String,
+        model: &'static str,
+        dimensions: usize,
+        max_dimensions: usize,
+    },
+    #[error("`.embedders.{embedder_name}.dimensions`: `dimensions` cannot be zero")]
+    InvalidSettingsDimensions { embedder_name: String },
 }
 
 impl From<crate::vector::Error> for Error {
@@ -251,6 +267,7 @@ impl From<arroy::Error> for Error {
             arroy::Error::DatabaseFull
             | arroy::Error::InvalidItemAppend
             | arroy::Error::UnmatchingDistance { .. }
+            | arroy::Error::MissingNode
             | arroy::Error::MissingMetadata => {
                 Error::InternalError(InternalError::ArroyError(value))
             }

milli/src/index.rs

@@ -1507,6 +1507,93 @@ impl Index {
             _ => "default".to_owned(),
         })
     }
+
+    pub fn check_document_facet_consistency(
+        &self,
+        rtxn: &RoTxn<'_>,
+    ) -> Result<DocumentFacetConsistency> {
+        let documents = self.documents_ids(rtxn)?;
+
+        let field_ids_map = self.fields_ids_map(rtxn)?;
+
+        let mut facets = Vec::new();
+        let mut facet_exists = Vec::new();
+        let faceted_fields = self.user_defined_faceted_fields(rtxn)?;
+        for fid in field_ids_map.ids() {
+            let facet_name = field_ids_map.name(fid).unwrap();
+            if !faceted_fields.contains(facet_name) {
+                continue;
+            };
+            let mut facet = RoaringBitmap::new();
+
+            // value doesn't matter here we'll truncate to the level
+            let key = crate::heed_codec::facet::FacetGroupKey {
+                field_id: fid,
+                level: 0,
+                left_bound: &[] as _,
+            };
+
+            for res in self
+                .facet_id_f64_docids
+                .remap_key_type::<FacetGroupKeyCodec<crate::heed_codec::BytesRefCodec>>()
+                .prefix_iter(rtxn, &key)?
+            {
+                let (_k, v) = res?;
+                facet |= v.bitmap;
+            }
+
+            for res in self
+                .facet_id_string_docids
+                .remap_key_type::<FacetGroupKeyCodec<crate::heed_codec::BytesRefCodec>>()
+                .prefix_iter(rtxn, &key)?
+            {
+                let (_k, v) = res?;
+                facet |= v.bitmap;
+            }
+
+            facets.push((field_ids_map.name(fid).unwrap().to_owned(), facet));
+            facet_exists.push(self.exists_faceted_documents_ids(rtxn, fid)?);
+        }
+
+        Ok(DocumentFacetConsistency { documents, facets, facet_exists })
+    }
+}
+
+pub struct DocumentFacetConsistency {
+    documents: RoaringBitmap,
+    facets: Vec<(String, RoaringBitmap)>,
+    facet_exists: Vec<RoaringBitmap>,
+}
+
+impl DocumentFacetConsistency {
+    pub fn check(&self) {
+        let mut inconsistencies = 0;
+        for ((field_name, facet), facet_exists) in self.facets.iter().zip(self.facet_exists.iter())
+        {
+            if field_name == "_geo" {
+                continue;
+            }
+
+            let documents = self.documents.clone() & facet_exists;
+            let missing_in_facets = &documents - facet;
+            let missing_in_documents = facet - documents;
+
+            for id in missing_in_facets {
+                tracing::error!(id, field_name, "Missing in facets");
+                inconsistencies += 1;
+            }
+            for id in missing_in_documents {
+                tracing::error!(id, field_name, "Missing in documents");
+                inconsistencies += 1;
+            }
+        }
+        if inconsistencies > 0 {
+            panic!(
+                "Panicked due to the previous {} inconsistencies between documents and facets",
+                inconsistencies
+            )
+        }
+    }
 }
 
 #[cfg(test)]

milli/src/search/facet/facet_sort_ascending.rs

@@ -181,7 +181,7 @@ mod tests {
             get_simple_string_index_with_multiple_field_ids(),
             get_random_looking_string_index_with_multiple_field_ids(),
         ];
-        for (_i, index) in indexes.iter().enumerate() {
+        for index in indexes {
             let txn = index.env.read_txn().unwrap();
             let candidates = RoaringBitmap::new();
             let mut results = String::new();
@@ -212,7 +212,7 @@ mod tests {
             get_simple_string_index_with_multiple_field_ids(),
             get_random_looking_string_index_with_multiple_field_ids(),
         ];
-        for (_i, index) in indexes.iter().enumerate() {
+        for index in indexes {
             let txn = index.env.read_txn().unwrap();
             let candidates = RoaringBitmap::new();
             let mut results = String::new();

milli/src/search/facet/facet_sort_descending.rs

@@ -195,7 +195,7 @@ mod tests {
             get_simple_string_index_with_multiple_field_ids(),
             get_random_looking_string_index_with_multiple_field_ids(),
         ];
-        for (_i, index) in indexes.iter().enumerate() {
+        for index in indexes {
             let txn = index.env.read_txn().unwrap();
             let candidates = RoaringBitmap::new();
             let mut results = String::new();
@@ -226,7 +226,7 @@ mod tests {
             get_simple_string_index_with_multiple_field_ids(),
             get_random_looking_string_index_with_multiple_field_ids(),
         ];
-        for (_i, index) in indexes.iter().enumerate() {
+        for index in indexes {
             let txn = index.env.read_txn().unwrap();
             let candidates = RoaringBitmap::new();
             let mut results = String::new();