Fix benches

Diff Indexing for the vector points

.github/ISSUE_TEMPLATE/sprint_issue.md

@@ -7,19 +7,17 @@ assignees: ''
 
 ---
 
-Related product team resources: [roadmap card]() (_internal only_) and [PRD]() (_internal only_)
+Related product team resources: [PRD]() (_internal only_)
 Related product discussion:
 Related spec: WIP
 
 ## Motivation
 
-<!---Copy/paste the information in the roadmap resources or briefly detail the product motivation. Ask product team if any hesitation.-->
+<!---Copy/paste the information in PRD or briefly detail the product motivation. Ask product team if any hesitation.-->
 
 ## Usage
 
-<!---Write a quick description of the usage if the usage has already been defined-->
-
-Refer to the final spec to know the details and the final decisions about the usage.
+<!---Link to the public part of the PRD, or to the related product discussion for experimental features-->
 
 ## TODO
 

.github/workflows/benchmarks-manual.yml

@@ -74,4 +74,4 @@ jobs:
           echo "${{ steps.file.outputs.basename }}.json has just been pushed."
           echo 'How to compare this benchmark with another one?'
           echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"
+          echo "  - Run the following command: ./benchmaks/scripts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/benchmarks-pr.yml

@@ -0,0 +1,97 @@
+name: Benchmarks (PR)
+on: issue_comment
+permissions:
+  issues: write
+
+env:
+  GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
+
+jobs:
+  run-benchmarks-on-comment:
+    if: startsWith(github.event.comment.body, '/benchmark')
+    name: Run and upload benchmarks
+    runs-on: benchmarks
+    timeout-minutes: 4320 # 72h
+    steps:
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Check for Command
+        id: command
+        uses: xt0rted/slash-command-action@v2
+        with:
+          command: benchmark
+          reaction-type: "eyes"
+          repo-token: ${{ env.GH_TOKEN }}
+
+      - uses: xt0rted/pull-request-comment-branch@v2
+        id: comment-branch
+        with:
+          repo_token: ${{ env.GH_TOKEN }}
+
+      - uses: actions/checkout@v3
+        if: success()
+        with:
+          fetch-depth: 0 # fetch full history to be able to get main commit sha
+          ref: ${{ steps.comment-branch.outputs.head_ref }}
+
+      # Set variables
+      - name: Set current branch name
+        shell: bash
+        run: echo "name=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT
+        id: current_branch
+      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
+        shell: bash
+        run: echo "name=$(git rev-parse --abbrev-ref HEAD | tr '/' '_')" >> $GITHUB_OUTPUT
+        id: normalized_current_branch
+      - name: Set shorter commit SHA
+        shell: bash
+        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
+        id: commit_sha
+      - name: Set file basename with format "dataset_branch_commitSHA"
+        shell: bash
+        run: echo "basename=$(echo ${{ steps.command.outputs.command-arguments }}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
+        id: file
+
+      # Run benchmarks
+      - name: Run benchmarks - Dataset ${{ steps.command.outputs.command-arguments }} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
+        run: |
+          cd benchmarks
+          cargo bench --bench ${{ steps.command.outputs.command-arguments }} -- --save-baseline ${{ steps.file.outputs.basename }}
+
+      # Generate critcmp files
+      - name: Install critcmp
+        uses: taiki-e/install-action@v2
+        with:
+          tool: critcmp
+      - name: Export cripcmp file
+        run: |
+          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
+
+      # Upload benchmarks
+      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
+        uses: BetaHuhn/do-spaces-action@v2
+        with:
+          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
+          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
+          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
+          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
+          source: ${{ steps.file.outputs.basename }}.json
+          out_dir: critcmp_results
+
+      # Compute the diff of the benchmarks and send a message on the GitHub PR
+      - name: Compute and send a message in the PR
+        env:
+          GITHUB_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
+        run: |
+          set -x
+          export base_ref=$(git merge-base origin/main ${{ steps.comment-branch.outputs.head_ref }} | head -c8)
+          export base_filename=$(echo ${{ steps.command.outputs.command-arguments }}_main_${base_ref}.json)
+          echo 'Here are your benchmarks diff 👊' >> body.txt
+          echo '```' >> body.txt
+          ./benchmarks/scripts/compare.sh $base_filename ${{ steps.file.outputs.basename }}.json >> body.txt
+          echo '```' >> body.txt
+          gh pr comment ${{ steps.current_branch.outputs.name }} --body-file body.txt

.github/workflows/dependency-issue.yml

@@ -2,8 +2,8 @@ name: Create issue to upgrade dependencies
 
 on:
   schedule:
-    # Run the first of the month, every 3 month
-    - cron: '0 0 1 */3 *'
+    # Run the first of the month, every 6 month
+    - cron: '0 0 1 */6 *'
   workflow_dispatch:
 
 jobs:

.github/workflows/publish-docker-images.yml

@@ -57,10 +57,10 @@ jobs:
           echo "date=$commit_date" >> $GITHUB_OUTPUT
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -70,7 +70,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v4
+        uses: docker/metadata-action@v5
         with:
           images: getmeili/meilisearch
           # Prevent `latest` to be updated for each new tag pushed.
@@ -83,7 +83,7 @@ jobs:
             type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
 
       - name: Build and push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           push: true
           platforms: linux/amd64,linux/arm64

.github/workflows/sdks-tests.yml

@@ -14,6 +14,7 @@ on:
 env:
   MEILI_MASTER_KEY: 'masterKey'
   MEILI_NO_ANALYTICS: 'true'
+  DISABLE_COVERAGE: 'true'
 
 jobs:
   define-docker-image:
@@ -30,6 +31,117 @@ jobs:
           if [[ $event == 'workflow_dispatch' ]]; then
             echo "docker-image=${{ github.event.inputs.docker_image }}" >> $GITHUB_OUTPUT
           fi
+      - name: Docker image is ${{ steps.define-image.outputs.docker-image }}
+        run: echo "Docker image is ${{ steps.define-image.outputs.docker-image }}"
+
+##########
+## SDKs ##
+##########
+
+  meilisearch-dotnet-tests:
+    needs: define-docker-image
+    name: .NET SDK tests
+    runs-on: ubuntu-latest
+    env:
+      MEILISEARCH_VERSION: ${{ needs.define-docker-image.outputs.docker-image }}
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/meilisearch-dotnet
+      - name: Setup .NET Core
+        uses: actions/setup-dotnet@v3
+        with:
+          dotnet-version: "6.0.x"
+      - name: Install dependencies
+        run: dotnet restore
+      - name: Build
+        run: dotnet build --configuration Release --no-restore
+      - name: Meilisearch (latest version) setup with Docker
+        run: docker compose up -d
+      - name: Run tests
+        run: dotnet test --no-restore --verbosity normal
+
+  meilisearch-dart-tests:
+    needs: define-docker-image
+    name: Dart SDK tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/meilisearch-dart
+      - uses: dart-lang/setup-dart@v1
+        with:
+          sdk: 3.1.1
+      - name: Install dependencies
+        run: dart pub get
+      - name: Run integration tests
+        run: dart test --concurrency=4
+
+  meilisearch-go-tests:
+    needs: define-docker-image
+    name: Go SDK tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: stable
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/meilisearch-go
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+          if [ -f Gopkg.toml ]; then
+            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+            dep ensure
+          fi
+      - name: Run integration tests
+        run: go test -v ./...
+
+  meilisearch-java-tests:
+    needs: define-docker-image
+    name: Java SDK tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/meilisearch-java
+      - name: Set up Java
+        uses: actions/setup-java@v3
+        with:
+          java-version: 8
+          distribution: 'zulu'
+          cache: gradle
+      - name: Grant execute permission for gradlew
+        run: chmod +x gradlew
+      - name: Build and run unit and integration tests
+        run: ./gradlew build integrationTest
 
   meilisearch-js-tests:
     needs: define-docker-image
@@ -66,33 +178,6 @@ jobs:
       - name: Run Browser env
         run: yarn test:env:browser
 
-  instant-meilisearch-tests:
-    needs: define-docker-image
-    name: instant-meilisearch tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/instant-meilisearch
-      - name: Setup node
-        uses: actions/setup-node@v3
-        with:
-          cache: yarn
-      - name: Install dependencies
-        run: yarn install
-      - name: Run tests
-        run: yarn test
-      - name: Build all the playgrounds and the packages
-        run: yarn build
-
   meilisearch-php-tests:
     needs: define-docker-image
     name: PHP SDK tests
@@ -111,8 +196,6 @@ jobs:
           repository: meilisearch/meilisearch-php
       - name: Install PHP
         uses: shivammathur/setup-php@v2
-        with:
-          coverage: none
       - name: Validate composer.json and composer.lock
         run: composer validate
       - name: Install dependencies
@@ -149,36 +232,6 @@ jobs:
       - name: Test with pytest
         run: pipenv run pytest
 
-  meilisearch-go-tests:
-    needs: define-docker-image
-    name: Go SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-go
-      - name: Get dependencies
-        run: |
-          go get -v -t -d ./...
-          if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-          fi
-      - name: Run integration tests
-        run: go test -v ./...
-
   meilisearch-ruby-tests:
     needs: define-docker-image
     name: Ruby SDK tests
@@ -224,3 +277,110 @@ jobs:
         run: cargo build --verbose
       - name: Run tests
         run: cargo test --verbose
+
+  meilisearch-swift-tests:
+    needs: define-docker-image
+    name: Swift SDK tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/meilisearch-swift
+      - name: Run tests
+        run: swift test
+
+########################
+## FRONT-END PLUGINS ##
+########################
+
+  meilisearch-js-plugins-tests:
+    needs: define-docker-image
+    name: meilisearch-js-plugins tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/meilisearch-js-plugins
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          cache: yarn
+      - name: Install dependencies
+        run: yarn install
+      - name: Run tests
+        run: yarn test
+      - name: Build all the playgrounds and the packages
+        run: yarn build
+
+########################
+## BACK-END PLUGINS ###
+########################
+
+  meilisearch-rails-tests:
+    needs: define-docker-image
+    name: meilisearch-rails tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/meilisearch-rails
+      - name: Set up Ruby 3
+        uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: 3
+          bundler-cache: true
+      - name: Run tests
+        run: bundle exec rspec
+
+  meilisearch-symfony-tests:
+    needs: define-docker-image
+    name: meilisearch-symfony tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/meilisearch-symfony
+      - name: Install PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          tools: composer:v2, flex
+      - name: Validate composer.json and composer.lock
+        run: composer validate
+      - name: Install dependencies
+        run: composer install --prefer-dist --no-progress --quiet
+      - name: Remove doctrine/annotations
+        run: composer remove --dev doctrine/annotations
+      - name: Run test suite
+        run: composer test:unit

.github/workflows/test-suite.yml

@@ -43,7 +43,7 @@ jobs:
           toolchain: nightly
           override: true
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.1
+        uses: Swatinem/rust-cache@v2.6.2
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -65,7 +65,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.1
+        uses: Swatinem/rust-cache@v2.6.2
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -123,7 +123,10 @@ jobs:
           override: true
       - name: Run cargo tree without default features and check lindera is not present
         run: |
-          cargo tree -f '{p} {f}' -e normal --no-default-features | grep lindera -vqz
+          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -vqz lindera; then
+            echo "lindera has been found in the sources and it shouldn't"
+            exit 1
+          fi
       - name: Run cargo tree with default features and check lindera is pressent
         run: |
           cargo tree -f '{p} {f}' -e normal | grep lindera -qz
@@ -146,7 +149,7 @@ jobs:
           toolchain: stable
           override: true
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.1
+        uses: Swatinem/rust-cache@v2.6.2
       - name: Run tests in debug
         uses: actions-rs/cargo@v1
         with:
@@ -165,7 +168,7 @@ jobs:
           override: true
           components: clippy
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.1
+        uses: Swatinem/rust-cache@v2.6.2
       - name: Run cargo clippy
         uses: actions-rs/cargo@v1
         with:
@@ -184,7 +187,7 @@ jobs:
           override: true
           components: rustfmt
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.1
+        uses: Swatinem/rust-cache@v2.6.2
       - name: Run cargo fmt
         # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
         # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate

Cargo.lock

@@ -468,7 +468,7 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b"
 
 [[package]]
 name = "benchmarks"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "anyhow",
  "bytes",
@@ -1206,7 +1206,7 @@ dependencies = [
 
 [[package]]
 name = "dump"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "anyhow",
  "big_s",
@@ -1417,7 +1417,7 @@ dependencies = [
 
 [[package]]
 name = "file-store"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "faux",
  "tempfile",
@@ -1439,11 +1439,12 @@ dependencies = [
 
 [[package]]
 name = "filter-parser"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "insta",
  "nom",
  "nom_locate",
+ "unescaper",
 ]
 
 [[package]]
@@ -1458,7 +1459,7 @@ dependencies = [
 
 [[package]]
 name = "flatten-serde-json"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "criterion",
  "serde_json",
@@ -1576,7 +1577,7 @@ dependencies = [
 
 [[package]]
 name = "fuzzers"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "arbitrary",
  "clap",
@@ -1662,12 +1663,13 @@ checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b"
 
 [[package]]
 name = "grenad"
-version = "0.4.4"
+version = "0.4.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5232b2d157b7bf63d7abe1b12177039e58db2f29e377517c0cdee1578cca4c93"
+checksum = "6a007932af5475ebb5c63bef8812bb1c36f317983bb4ca663e9d6dd58d6a0f8c"
 dependencies = [
  "bytemuck",
  "byteorder",
+ "rayon",
  "tempfile",
 ]
 
@@ -1890,7 +1892,7 @@ dependencies = [
 
 [[package]]
 name = "index-scheduler"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "anyhow",
  "big_s",
@@ -2087,7 +2089,7 @@ dependencies = [
 
 [[package]]
 name = "json-depth-checker"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "criterion",
  "serde_json",
@@ -2499,7 +2501,7 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
 
 [[package]]
 name = "meili-snap"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "insta",
  "md5",
@@ -2508,7 +2510,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "actix-cors",
  "actix-http",
@@ -2563,7 +2565,6 @@ dependencies = [
  "platform-dirs",
  "prometheus",
  "puffin",
- "puffin_http",
  "rand",
  "rayon",
  "regex",
@@ -2599,7 +2600,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-auth"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "base64 0.21.2",
  "enum-iterator",
@@ -2618,7 +2619,7 @@ dependencies = [
 
 [[package]]
 name = "meilisearch-types"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "actix-web",
  "anyhow",
@@ -2672,7 +2673,7 @@ dependencies = [
 
 [[package]]
 name = "milli"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "big_s",
  "bimap",
@@ -2703,6 +2704,7 @@ dependencies = [
  "logging_timer",
  "maplit",
  "md5",
+ "meili-snap",
  "memmap2",
  "mimalloc",
  "obkv",
@@ -2865,9 +2867,9 @@ dependencies = [
 
 [[package]]
 name = "obkv"
-version = "0.2.0"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f69e48cd7c8e5bb52a1da1287fdbfd877c32673176583ce664cd63b201aba385"
+checksum = "6c459142426056c639ff88d053ebaaaeca0ee1411c94362892398ef4ccd81080"
 
 [[package]]
 name = "once_cell"
@@ -2994,7 +2996,7 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
 
 [[package]]
 name = "permissive-json-pointer"
-version = "1.4.0"
+version = "1.4.1"
 dependencies = [
  "big_s",
  "serde_json",
@@ -3192,7 +3194,7 @@ dependencies = [
  "byteorder",
  "hex",
  "lazy_static",
- "rustix 0.36.15",
+ "rustix 0.36.16",
 ]
 
 [[package]]
@@ -3235,18 +3237,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "puffin_http"
-version = "0.13.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13bffc600c35913d282ae1e96a6ffcdf36dc7a7cdb9310e0ba15914d258c8193"
-dependencies = [
- "anyhow",
- "crossbeam-channel",
- "log",
- "puffin",
-]
-
 [[package]]
 name = "quote"
 version = "1.0.32"
@@ -3477,9 +3467,9 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.36.15"
+version = "0.36.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c37f1bd5ef1b5422177b7646cba67430579cfe2ace80f284fee876bca52ad941"
+checksum = "6da3636faa25820d8648e0e31c5d519bbb01f72fdf57131f0f5f7da5fed36eab"
 dependencies = [
  "bitflags 1.3.2",
  "errno",
@@ -3537,9 +3527,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.100.1"
+version = "0.100.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b"
+checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab"
 dependencies = [
  "ring",
  "untrusted",
@@ -4180,6 +4170,15 @@ version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9"
 
+[[package]]
+name = "unescaper"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a96a44ae11e25afb520af4534fd7b0bd8cd613e35a78def813b8cf41631fa3c8"
+dependencies = [
+ "thiserror",
+]
+
 [[package]]
 name = "unicase"
 version = "2.6.0"
@@ -4238,7 +4237,7 @@ dependencies = [
  "log",
  "once_cell",
  "rustls 0.21.6",
- "rustls-webpki 0.100.1",
+ "rustls-webpki 0.100.2",
  "url",
  "webpki-roots 0.23.1",
 ]
@@ -4433,9 +4432,9 @@ dependencies = [
 
 [[package]]
 name = "webpki"
-version = "0.22.0"
+version = "0.22.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd"
+checksum = "07ecc0cd7cac091bf682ec5efa18b1cff79d617b84181f38b3951dbe135f607f"
 dependencies = [
  "ring",
  "untrusted",
@@ -4456,7 +4455,7 @@ version = "0.23.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338"
 dependencies = [
- "rustls-webpki 0.100.1",
+ "rustls-webpki 0.100.2",
 ]
 
 [[package]]

Cargo.toml

@@ -18,7 +18,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.4.0"
+version = "1.4.1"
 authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
 description = "Meilisearch HTTP server"
 homepage = "https://meilisearch.com"

PROFILING.md

@@ -1,14 +1,14 @@
 # Profiling Meilisearch
 
-Search engine technologies are complex pieces of software that require thorough profiling tools. We chose to use [Puffin](https://github.com/EmbarkStudios/puffin), which the Rust gaming industry uses extensively. You can export and import the profiling reports using the top bar's _File_ menu options.
+Search engine technologies are complex pieces of software that require thorough profiling tools. We chose to use [Puffin](https://github.com/EmbarkStudios/puffin), which the Rust gaming industry uses extensively. You can export and import the profiling reports using the top bar's _File_ menu options [in Puffin Viewer](https://github.com/embarkstudios/puffin#ui).
 
 ![An example profiling with Puffin viewer](assets/profiling-example.png)
 
 ## Profiling the Indexing Process
 
-When you enable the `profile-with-puffin` feature of Meilisearch, a Puffin HTTP server will run on Meilisearch and listen on the default _0.0.0.0:8585_ address. This server will record a "frame" whenever it executes the `IndexScheduler::tick` method.
+When you enable [the `exportPuffinReports` experimental feature](https://www.meilisearch.com/docs/learn/experimental/overview) of Meilisearch, Puffin reports with the `.puffin` extension will be automatically exported to disk. When this option is enabled, the engine will automatically create a "frame" whenever it executes the `IndexScheduler::tick` method.
 
-Once your Meilisearch is running and awaits new indexation operations, you must [install and run the `puffin_viewer` tool](https://github.com/EmbarkStudios/puffin/tree/main/puffin_viewer) to see the profiling results. I advise you to run the viewer with the `RUST_LOG=puffin_http::client=debug` environment variable to see the client trying to connect to your server.
+[Puffin Viewer](https://github.com/EmbarkStudios/puffin/tree/main/puffin_viewer) is used to analyze the reports. Those reports show areas where Meilisearch spent time during indexing.
 
 Another piece of advice on the Puffin viewer UI interface is to consider the _Merge children with same ID_ option. It can hide the exact actual timings at which events were sent. Please turn it off when you see strange gaps on the Flamegraph. It can help.
 

README.md

@@ -25,6 +25,12 @@
 
 <p align="center">⚡ A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow 🔍</p>
 
+---
+
+### 🔥 On November 2nd, we are hosting our first-ever live demo and product updates for [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). Make sure to [register here](https://us06web.zoom.us/meeting/register/tZMlc-mqrjIsH912-HTRe-AaT-pp41bDe81a#/registration) and bring your questions for live Q&A!
+
+---
+
 Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow.
 
 <p align="center" name="demo">

benchmarks/benches/indexing.rs

@@ -6,9 +6,7 @@ use std::path::Path;
 
 use criterion::{criterion_group, criterion_main, Criterion};
 use milli::heed::{EnvOpenOptions, RwTxn};
-use milli::update::{
-    DeleteDocuments, IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings,
-};
+use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
 use milli::Index;
 use rand::seq::SliceRandom;
 use rand_chacha::rand_core::SeedableRng;
@@ -266,17 +264,7 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                let mut wtxn = index.write_txn().unwrap();
-
-                for ids in document_ids_to_delete {
-                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
-                    builder.delete_documents(&ids);
-                    builder.execute().unwrap();
-                }
-
-                wtxn.commit().unwrap();
-
-                index.prepare_for_closing().wait();
+                delete_documents_from_ids(index, document_ids_to_delete)
             },
         )
     });
@@ -613,17 +601,7 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                let mut wtxn = index.write_txn().unwrap();
-
-                for ids in document_ids_to_delete {
-                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
-                    builder.delete_documents(&ids);
-                    builder.execute().unwrap();
-                }
-
-                wtxn.commit().unwrap();
-
-                index.prepare_for_closing().wait();
+                delete_documents_from_ids(index, document_ids_to_delete)
             },
         )
     });
@@ -875,22 +853,41 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                let mut wtxn = index.write_txn().unwrap();
-
-                for ids in document_ids_to_delete {
-                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
-                    builder.delete_documents(&ids);
-                    builder.execute().unwrap();
-                }
-
-                wtxn.commit().unwrap();
-
-                index.prepare_for_closing().wait();
+                delete_documents_from_ids(index, document_ids_to_delete)
             },
         )
     });
 }
 
+fn delete_documents_from_ids(index: Index, document_ids_to_delete: Vec<RoaringBitmap>) {
+    let mut wtxn = index.write_txn().unwrap();
+
+    let indexer_config = IndexerConfig::default();
+    for ids in document_ids_to_delete {
+        let external_documents_ids = index.external_documents_ids();
+        // FIXME: for filters matching a lot of documents, this will allocate a huge vec of external docids (strings).
+        // Since what we have is an iterator, it would be better to delete in chunks
+        let external_to_internal: std::result::Result<Vec<_>, RoaringBitmap> =
+            external_documents_ids
+                .find_external_id_of(&wtxn, ids)
+                .unwrap()
+                .only_external_ids()
+                .collect();
+        let ids = external_to_internal.unwrap();
+        let config = IndexDocumentsConfig::default();
+
+        let mut builder =
+            IndexDocuments::new(&mut wtxn, &index, &indexer_config, config, |_| (), || false)
+                .unwrap();
+        (builder, _) = builder.remove_documents(ids).unwrap();
+        builder.execute().unwrap();
+    }
+
+    wtxn.commit().unwrap();
+
+    index.prepare_for_closing().wait();
+}
+
 fn indexing_movies_in_three_batches(c: &mut Criterion) {
     let mut group = c.benchmark_group("indexing");
     group.sample_size(BENCHMARK_ITERATION);
@@ -1112,17 +1109,7 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                let mut wtxn = index.write_txn().unwrap();
-
-                for ids in document_ids_to_delete {
-                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
-                    builder.delete_documents(&ids);
-                    builder.execute().unwrap();
-                }
-
-                wtxn.commit().unwrap();
-
-                index.prepare_for_closing().wait();
+                delete_documents_from_ids(index, document_ids_to_delete)
             },
         )
     });
@@ -1338,17 +1325,7 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                let mut wtxn = index.write_txn().unwrap();
-
-                for ids in document_ids_to_delete {
-                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
-                    builder.delete_documents(&ids);
-                    builder.execute().unwrap();
-                }
-
-                wtxn.commit().unwrap();
-
-                index.prepare_for_closing().wait();
+                delete_documents_from_ids(index, document_ids_to_delete)
             },
         )
     });

dump/src/reader/mod.rs

@@ -526,12 +526,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(products.metadata(), @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2022-10-09T20:27:22.688964637Z",
+          "updatedAt": "2022-10-09T20:27:23.951017769Z"
         }
         "###);
 
@@ -541,12 +541,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2022-10-09T20:27:22.197788495Z",
+          "updatedAt": "2022-10-09T20:28:01.93111053Z"
         }
         "###);
 
@@ -571,12 +571,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(spells.metadata(), @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2022-10-09T20:27:24.242683494Z",
+          "updatedAt": "2022-10-09T20:27:24.312809641Z"
         }
         "###);
 
@@ -617,12 +617,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(products.metadata(), @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2023-01-30T16:25:56.595257Z",
+          "updatedAt": "2023-01-30T16:25:58.70348Z"
         }
         "###);
 
@@ -632,12 +632,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2023-01-30T16:25:56.192178Z",
+          "updatedAt": "2023-01-30T16:25:56.455714Z"
         }
         "###);
 
@@ -647,12 +647,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"0227598af846e574139ee0b80e03a720");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(spells.metadata(), @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2023-01-30T16:25:58.876405Z",
+          "updatedAt": "2023-01-30T16:25:59.079906Z"
         }
         "###);
 

dump/src/reader/snapshots/dump__reader__test__import_dump_v1-11.snap

@@ -1,24 +0,0 @@
----
-source: dump/src/reader/mod.rs
-expression: spells.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "sortableAttributes": [],
-  "rankingRules": [
-    "typo",
-    "words",
-    "proximity",
-    "attribute",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null
-}

dump/src/reader/snapshots/dump__reader__test__import_dump_v1-5.snap

@@ -1,38 +0,0 @@
----
-source: dump/src/reader/mod.rs
-expression: products.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "sortableAttributes": [],
-  "rankingRules": [
-    "typo",
-    "words",
-    "proximity",
-    "attribute",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {
-    "android": [
-      "phone",
-      "smartphone"
-    ],
-    "iphone": [
-      "phone",
-      "smartphone"
-    ],
-    "phone": [
-      "android",
-      "iphone",
-      "smartphone"
-    ]
-  },
-  "distinctAttribute": null
-}

dump/src/reader/snapshots/dump__reader__test__import_dump_v1-8.snap

@@ -1,31 +0,0 @@
----
-source: dump/src/reader/mod.rs
-expression: movies.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [
-    "genres",
-    "id"
-  ],
-  "sortableAttributes": [
-    "genres",
-    "id"
-  ],
-  "rankingRules": [
-    "typo",
-    "words",
-    "proximity",
-    "attribute",
-    "exactness",
-    "release_date:asc"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null
-}

dump/src/reader/v2/mod.rs

@@ -46,6 +46,7 @@ pub type Checked = settings::Checked;
 pub type Unchecked = settings::Unchecked;
 
 pub type Task = updates::UpdateEntry;
+pub type Kind = updates::UpdateMeta;
 
 // everything related to the errors
 pub type ResponseError = errors::ResponseError;
@@ -107,8 +108,11 @@ impl V2Reader {
     pub fn indexes(&self) -> Result<impl Iterator<Item = Result<V2IndexReader>> + '_> {
         Ok(self.index_uuid.iter().map(|index| -> Result<_> {
             V2IndexReader::new(
-                index.uid.clone(),
                 &self.dump.path().join("indexes").join(format!("index-{}", index.uuid)),
+                index,
+                BufReader::new(
+                    File::open(self.dump.path().join("updates").join("data.jsonl")).unwrap(),
+                ),
             )
         }))
     }
@@ -143,16 +147,41 @@ pub struct V2IndexReader {
 }
 
 impl V2IndexReader {
-    pub fn new(name: String, path: &Path) -> Result<Self> {
+    pub fn new(path: &Path, index_uuid: &IndexUuid, tasks: BufReader<File>) -> Result<Self> {
         let meta = File::open(path.join("meta.json"))?;
         let meta: DumpMeta = serde_json::from_reader(meta)?;
 
+        let mut created_at = None;
+        let mut updated_at = None;
+
+        for line in tasks.lines() {
+            let task: Task = serde_json::from_str(&line?)?;
+            if !(task.uuid == index_uuid.uuid && task.is_finished()) {
+                continue;
+            }
+
+            let new_created_at = match task.update.meta() {
+                Kind::DocumentsAddition { .. } | Kind::Settings(_) => task.update.finished_at(),
+                _ => None,
+            };
+            let new_updated_at = task.update.finished_at();
+
+            if created_at.is_none() || created_at > new_created_at {
+                created_at = new_created_at;
+            }
+
+            if updated_at.is_none() || updated_at < new_updated_at {
+                updated_at = new_updated_at;
+            }
+        }
+
+        let current_time = OffsetDateTime::now_utc();
+
         let metadata = IndexMetadata {
-            uid: name,
+            uid: index_uuid.uid.clone(),
             primary_key: meta.primary_key,
-            // FIXME: Iterate over the whole task queue to find the creation and last update date.
-            created_at: OffsetDateTime::now_utc(),
-            updated_at: OffsetDateTime::now_utc(),
+            created_at: created_at.unwrap_or(current_time),
+            updated_at: updated_at.unwrap_or(current_time),
         };
 
         let ret = V2IndexReader {
@@ -248,12 +277,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(products.metadata(), @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2022-10-09T20:27:22.688964637Z",
+          "updatedAt": "2022-10-09T20:27:23.951017769Z"
         }
         "###);
 
@@ -263,12 +292,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2022-10-09T20:27:22.197788495Z",
+          "updatedAt": "2022-10-09T20:28:01.93111053Z"
         }
         "###);
 
@@ -293,12 +322,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(spells.metadata(), @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2022-10-09T20:27:24.242683494Z",
+          "updatedAt": "2022-10-09T20:27:24.312809641Z"
         }
         "###);
 
@@ -340,12 +369,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(products.metadata(), @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2023-01-30T16:25:56.595257Z",
+          "updatedAt": "2023-01-30T16:25:58.70348Z"
         }
         "###);
 
@@ -355,12 +384,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2023-01-30T16:25:56.192178Z",
+          "updatedAt": "2023-01-30T16:25:56.455714Z"
         }
         "###);
 
@@ -370,12 +399,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"0227598af846e574139ee0b80e03a720");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(spells.metadata(), @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2023-01-30T16:25:58.876405Z",
+          "updatedAt": "2023-01-30T16:25:59.079906Z"
         }
         "###);
 

dump/src/reader/v2/updates.rs

@@ -227,4 +227,14 @@ impl UpdateStatus {
             _ => None,
         }
     }
+
+    pub fn finished_at(&self) -> Option<OffsetDateTime> {
+        match self {
+            UpdateStatus::Processing(_) => None,
+            UpdateStatus::Enqueued(_) => None,
+            UpdateStatus::Processed(u) => Some(u.processed_at),
+            UpdateStatus::Aborted(_) => None,
+            UpdateStatus::Failed(u) => Some(u.failed_at),
+        }
+    }
 }

filter-parser/Cargo.toml

@@ -14,6 +14,7 @@ license.workspace = true
 [dependencies]
 nom = "7.1.3"
 nom_locate = "4.1.0"
+unescaper = "0.1.2"
 
 [dev-dependencies]
 insta = "1.29.0"

filter-parser/src/error.rs

@@ -62,6 +62,7 @@ pub enum ErrorKind<'a> {
     MisusedGeoRadius,
     MisusedGeoBoundingBox,
     InvalidPrimary,
+    InvalidEscapedNumber,
     ExpectedEof,
     ExpectedValue(ExpectedValueKind),
     MalformedValue,
@@ -147,6 +148,9 @@ impl<'a> Display for Error<'a> {
                 let text = if input.trim().is_empty() { "but instead got nothing.".to_string() } else { format!("at `{}`.", escaped_input) };
                 writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` {}", text)?
             }
+            ErrorKind::InvalidEscapedNumber => {
+                writeln!(f, "Found an invalid escaped sequence number: `{}`.", escaped_input)?
+            }
             ErrorKind::ExpectedEof => {
                 writeln!(f, "Found unexpected characters at the end of the filter: `{}`. You probably forgot an `OR` or an `AND` rule.", escaped_input)?
             }

filter-parser/src/lib.rs

@@ -545,6 +545,8 @@ impl<'a> std::fmt::Display for Token<'a> {
 
 #[cfg(test)]
 pub mod tests {
+    use FilterCondition as Fc;
+
     use super::*;
 
     /// Create a raw [Token]. You must specify the string that appear BEFORE your element followed by your element
@@ -556,14 +558,22 @@ pub mod tests {
         unsafe { Span::new_from_raw_offset(offset, lines as u32, value, "") }.into()
     }
 
+    fn p(s: &str) -> impl std::fmt::Display + '_ {
+        Fc::parse(s).unwrap().unwrap()
+    }
+
+    #[test]
+    fn parse_escaped() {
+        insta::assert_display_snapshot!(p(r#"title = 'foo\\'"#), @r#"{title} = {foo\}"#);
+        insta::assert_display_snapshot!(p(r#"title = 'foo\\\\'"#), @r#"{title} = {foo\\}"#);
+        insta::assert_display_snapshot!(p(r#"title = 'foo\\\\\\'"#), @r#"{title} = {foo\\\}"#);
+        insta::assert_display_snapshot!(p(r#"title = 'foo\\\\\\\\'"#), @r#"{title} = {foo\\\\}"#);
+        // but it also works with other sequencies
+        insta::assert_display_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}");
+    }
+
     #[test]
     fn parse() {
-        use FilterCondition as Fc;
-
-        fn p(s: &str) -> impl std::fmt::Display + '_ {
-            Fc::parse(s).unwrap().unwrap()
-        }
-
         // Test equal
         insta::assert_display_snapshot!(p("channel = Ponce"), @"{channel} = {Ponce}");
         insta::assert_display_snapshot!(p("subscribers = 12"), @"{subscribers} = {12}");

filter-parser/src/value.rs

@@ -171,7 +171,24 @@ pub fn parse_value(input: Span) -> IResult<Token> {
         })
     })?;
 
-    Ok((input, value))
+    match unescaper::unescape(value.value()) {
+        Ok(content) => {
+            if content.len() != value.value().len() {
+                Ok((input, Token::new(value.original_span(), Some(content))))
+            } else {
+                Ok((input, value))
+            }
+        }
+        Err(unescaper::Error::IncompleteStr(_)) => Err(nom::Err::Incomplete(nom::Needed::Unknown)),
+        Err(unescaper::Error::ParseIntError { .. }) => Err(nom::Err::Error(Error::new_from_kind(
+            value.original_span(),
+            ErrorKind::InvalidEscapedNumber,
+        ))),
+        Err(unescaper::Error::InvalidChar { .. }) => Err(nom::Err::Error(Error::new_from_kind(
+            value.original_span(),
+            ErrorKind::MalformedValue,
+        ))),
+    }
 }
 
 fn is_value_component(c: char) -> bool {
@@ -318,17 +335,17 @@ pub mod test {
             ("\"cha'nnel\"", "cha'nnel", false),
             ("I'm tamo", "I", false),
             // escaped thing but not quote
-            (r#""\\""#, r#"\\"#, false),
-            (r#""\\\\\\""#, r#"\\\\\\"#, false),
-            (r#""aa\\aa""#, r#"aa\\aa"#, false),
+            (r#""\\""#, r#"\"#, true),
+            (r#""\\\\\\""#, r#"\\\"#, true),
+            (r#""aa\\aa""#, r#"aa\aa"#, true),
             // with double quote
             (r#""Hello \"world\"""#, r#"Hello "world""#, true),
-            (r#""Hello \\\"world\\\"""#, r#"Hello \\"world\\""#, true),
+            (r#""Hello \\\"world\\\"""#, r#"Hello \"world\""#, true),
             (r#""I'm \"super\" tamo""#, r#"I'm "super" tamo"#, true),
             (r#""\"\"""#, r#""""#, true),
             // with simple quote
             (r#"'Hello \'world\''"#, r#"Hello 'world'"#, true),
-            (r#"'Hello \\\'world\\\''"#, r#"Hello \\'world\\'"#, true),
+            (r#"'Hello \\\'world\\\''"#, r#"Hello \'world\'"#, true),
             (r#"'I\'m "super" tamo'"#, r#"I'm "super" tamo"#, true),
             (r#"'\'\''"#, r#"''"#, true),
         ];
@@ -350,7 +367,14 @@ pub mod test {
                 "Filter `{}` was not supposed to be escaped",
                 input
             );
-            assert_eq!(token.value(), expected, "Filter `{}` failed.", input);
+            assert_eq!(
+                token.value(),
+                expected,
+                "Filter `{}` failed by giving `{}` instead of `{}`.",
+                input,
+                token.value(),
+                expected
+            );
         }
     }
 

index-scheduler/src/batch.rs

@@ -19,18 +19,18 @@ one indexing operation.
 
 use std::collections::{BTreeSet, HashSet};
 use std::ffi::OsStr;
+use std::fmt;
 use std::fs::{self, File};
 use std::io::BufWriter;
 
 use dump::IndexMetadata;
-use log::{debug, error, info};
+use log::{debug, error, info, trace};
 use meilisearch_types::error::Code;
 use meilisearch_types::heed::{RoTxn, RwTxn};
 use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader};
 use meilisearch_types::milli::heed::CompactionOption;
 use meilisearch_types::milli::update::{
-    DeleteDocuments, DocumentDeletionResult, IndexDocumentsConfig, IndexDocumentsMethod,
-    Settings as MilliSettings,
+    IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Settings as MilliSettings,
 };
 use meilisearch_types::milli::{self, Filter, BEU32};
 use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked};
@@ -43,7 +43,7 @@ use uuid::Uuid;
 
 use crate::autobatcher::{self, BatchKind};
 use crate::utils::{self, swap_index_uid_in_task};
-use crate::{Error, IndexScheduler, ProcessingTasks, Result, TaskId};
+use crate::{Error, IndexScheduler, MustStopProcessing, ProcessingTasks, Result, TaskId};
 
 /// Represents a combination of tasks that can all be processed at the same time.
 ///
@@ -67,10 +67,6 @@ pub(crate) enum Batch {
         op: IndexOperation,
         must_create_index: bool,
     },
-    IndexDocumentDeletionByFilter {
-        index_uid: String,
-        task: Task,
-    },
     IndexCreation {
         index_uid: String,
         primary_key: Option<String>,
@@ -114,6 +110,10 @@ pub(crate) enum IndexOperation {
         documents: Vec<Vec<String>>,
         tasks: Vec<Task>,
     },
+    IndexDocumentDeletionByFilter {
+        index_uid: String,
+        task: Task,
+    },
     DocumentClear {
         index_uid: String,
         tasks: Vec<Task>,
@@ -155,7 +155,6 @@ impl Batch {
             | Batch::TaskDeletion(task)
             | Batch::Dump(task)
             | Batch::IndexCreation { task, .. }
-            | Batch::IndexDocumentDeletionByFilter { task, .. }
             | Batch::IndexUpdate { task, .. } => vec![task.uid],
             Batch::SnapshotCreation(tasks) | Batch::IndexDeletion { tasks, .. } => {
                 tasks.iter().map(|task| task.uid).collect()
@@ -167,6 +166,7 @@ impl Batch {
                 | IndexOperation::DocumentClear { tasks, .. } => {
                     tasks.iter().map(|task| task.uid).collect()
                 }
+                IndexOperation::IndexDocumentDeletionByFilter { task, .. } => vec![task.uid],
                 IndexOperation::SettingsAndDocumentOperation {
                     document_import_tasks: tasks,
                     settings_tasks: other,
@@ -194,8 +194,30 @@ impl Batch {
             IndexOperation { op, .. } => Some(op.index_uid()),
             IndexCreation { index_uid, .. }
             | IndexUpdate { index_uid, .. }
-            | IndexDeletion { index_uid, .. }
-            | IndexDocumentDeletionByFilter { index_uid, .. } => Some(index_uid),
+            | IndexDeletion { index_uid, .. } => Some(index_uid),
+        }
+    }
+}
+
+impl fmt::Display for Batch {
+    /// A text used when we debug the profiling reports.
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let index_uid = self.index_uid();
+        let tasks = self.ids();
+        match self {
+            Batch::TaskCancelation { .. } => f.write_str("TaskCancelation")?,
+            Batch::TaskDeletion(_) => f.write_str("TaskDeletion")?,
+            Batch::SnapshotCreation(_) => f.write_str("SnapshotCreation")?,
+            Batch::Dump(_) => f.write_str("Dump")?,
+            Batch::IndexOperation { op, .. } => write!(f, "{op}")?,
+            Batch::IndexCreation { .. } => f.write_str("IndexCreation")?,
+            Batch::IndexUpdate { .. } => f.write_str("IndexUpdate")?,
+            Batch::IndexDeletion { .. } => f.write_str("IndexDeletion")?,
+            Batch::IndexSwap { .. } => f.write_str("IndexSwap")?,
+        };
+        match index_uid {
+            Some(name) => f.write_fmt(format_args!(" on {name:?} from tasks: {tasks:?}")),
+            None => f.write_fmt(format_args!(" from tasks: {tasks:?}")),
         }
     }
 }
@@ -205,6 +227,7 @@ impl IndexOperation {
         match self {
             IndexOperation::DocumentOperation { index_uid, .. }
             | IndexOperation::DocumentDeletion { index_uid, .. }
+            | IndexOperation::IndexDocumentDeletionByFilter { index_uid, .. }
             | IndexOperation::DocumentClear { index_uid, .. }
             | IndexOperation::Settings { index_uid, .. }
             | IndexOperation::DocumentClearAndSetting { index_uid, .. }
@@ -213,6 +236,30 @@ impl IndexOperation {
     }
 }
 
+impl fmt::Display for IndexOperation {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            IndexOperation::DocumentOperation { .. } => {
+                f.write_str("IndexOperation::DocumentOperation")
+            }
+            IndexOperation::DocumentDeletion { .. } => {
+                f.write_str("IndexOperation::DocumentDeletion")
+            }
+            IndexOperation::IndexDocumentDeletionByFilter { .. } => {
+                f.write_str("IndexOperation::IndexDocumentDeletionByFilter")
+            }
+            IndexOperation::DocumentClear { .. } => f.write_str("IndexOperation::DocumentClear"),
+            IndexOperation::Settings { .. } => f.write_str("IndexOperation::Settings"),
+            IndexOperation::DocumentClearAndSetting { .. } => {
+                f.write_str("IndexOperation::DocumentClearAndSetting")
+            }
+            IndexOperation::SettingsAndDocumentOperation { .. } => {
+                f.write_str("IndexOperation::SettingsAndDocumentOperation")
+            }
+        }
+    }
+}
+
 impl IndexScheduler {
     /// Convert an [`BatchKind`](crate::autobatcher::BatchKind) into a [`Batch`].
     ///
@@ -239,9 +286,12 @@ impl IndexScheduler {
                 let task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
                 match &task.kind {
                     KindWithContent::DocumentDeletionByFilter { index_uid, .. } => {
-                        Ok(Some(Batch::IndexDocumentDeletionByFilter {
-                            index_uid: index_uid.clone(),
-                            task,
+                        Ok(Some(Batch::IndexOperation {
+                            op: IndexOperation::IndexDocumentDeletionByFilter {
+                                index_uid: index_uid.clone(),
+                                task,
+                            },
+                            must_create_index: false,
                         }))
                     }
                     _ => unreachable!(),
@@ -578,7 +628,7 @@ impl IndexScheduler {
             self.breakpoint(crate::Breakpoint::InsideProcessBatch);
         }
 
-        puffin::profile_function!(format!("{:?}", batch));
+        puffin::profile_function!(batch.to_string());
 
         match batch {
             Batch::TaskCancelation { mut task, previous_started_at, previous_processing_tasks } => {
@@ -845,7 +895,7 @@ impl IndexScheduler {
                 })?;
 
                 // 4. Dump experimental feature settings
-                let features = self.features()?.runtime_features();
+                let features = self.features().runtime_features();
                 dump.create_experimental_features(features)?;
 
                 let dump_uid = started_at.format(format_description!(
@@ -896,51 +946,6 @@ impl IndexScheduler {
 
                 Ok(tasks)
             }
-            Batch::IndexDocumentDeletionByFilter { mut task, index_uid: _ } => {
-                let (index_uid, filter) =
-                    if let KindWithContent::DocumentDeletionByFilter { index_uid, filter_expr } =
-                        &task.kind
-                    {
-                        (index_uid, filter_expr)
-                    } else {
-                        unreachable!()
-                    };
-                let index = {
-                    let rtxn = self.env.read_txn()?;
-                    self.index_mapper.index(&rtxn, index_uid)?
-                };
-                let deleted_documents = delete_document_by_filter(filter, index);
-                let original_filter = if let Some(Details::DocumentDeletionByFilter {
-                    original_filter,
-                    deleted_documents: _,
-                }) = task.details
-                {
-                    original_filter
-                } else {
-                    // In the case of a `documentDeleteByFilter` the details MUST be set
-                    unreachable!();
-                };
-
-                match deleted_documents {
-                    Ok(deleted_documents) => {
-                        task.status = Status::Succeeded;
-                        task.details = Some(Details::DocumentDeletionByFilter {
-                            original_filter,
-                            deleted_documents: Some(deleted_documents),
-                        });
-                    }
-                    Err(e) => {
-                        task.status = Status::Failed;
-                        task.details = Some(Details::DocumentDeletionByFilter {
-                            original_filter,
-                            deleted_documents: Some(0),
-                        });
-                        task.error = Some(e.into());
-                    }
-                }
-
-                Ok(vec![task])
-            }
             Batch::IndexCreation { index_uid, primary_key, task } => {
                 let wtxn = self.env.write_txn()?;
                 if self.index_mapper.exists(&wtxn, &index_uid)? {
@@ -1185,7 +1190,7 @@ impl IndexScheduler {
                     index,
                     indexer_config,
                     config,
-                    |indexing_step| debug!("update: {:?}", indexing_step),
+                    |indexing_step| trace!("update: {:?}", indexing_step),
                     || must_stop_processing.get(),
                 )?;
 
@@ -1232,7 +1237,8 @@ impl IndexScheduler {
                             let (new_builder, user_result) =
                                 builder.remove_documents(document_ids)?;
                             builder = new_builder;
-
+                            // Uses Invariant: remove documents actually always returns Ok for the inner result
+                            let count = user_result.unwrap();
                             let provided_ids =
                                 if let Some(Details::DocumentDeletion { provided_ids, .. }) =
                                     task.details
@@ -1243,23 +1249,11 @@ impl IndexScheduler {
                                     unreachable!();
                                 };
 
-                            match user_result {
-                                Ok(count) => {
-                                    task.status = Status::Succeeded;
-                                    task.details = Some(Details::DocumentDeletion {
-                                        provided_ids,
-                                        deleted_documents: Some(count),
-                                    });
-                                }
-                                Err(e) => {
-                                    task.status = Status::Failed;
-                                    task.details = Some(Details::DocumentDeletion {
-                                        provided_ids,
-                                        deleted_documents: Some(0),
-                                    });
-                                    task.error = Some(milli::Error::from(e).into());
-                                }
-                            }
+                            task.status = Status::Succeeded;
+                            task.details = Some(Details::DocumentDeletion {
+                                provided_ids,
+                                deleted_documents: Some(count),
+                            });
                         }
                     }
                 }
@@ -1274,7 +1268,7 @@ impl IndexScheduler {
                         milli::update::Settings::new(index_wtxn, index, indexer_config);
                     builder.reset_primary_key();
                     builder.execute(
-                        |indexing_step| debug!("update: {:?}", indexing_step),
+                        |indexing_step| trace!("update: {:?}", indexing_step),
                         || must_stop_processing.clone().get(),
                     )?;
                 }
@@ -1282,23 +1276,91 @@ impl IndexScheduler {
                 Ok(tasks)
             }
             IndexOperation::DocumentDeletion { index_uid: _, documents, mut tasks } => {
-                let mut builder = milli::update::DeleteDocuments::new(index_wtxn, index)?;
-                documents.iter().flatten().for_each(|id| {
-                    builder.delete_external_id(id);
-                });
+                let indexer_config = self.index_mapper.indexer_config();
+                let config = IndexDocumentsConfig {
+                    update_method: IndexDocumentsMethod::ReplaceDocuments,
+                    ..Default::default()
+                };
+                let must_stop_processing = self.must_stop_processing.clone();
 
-                let DocumentDeletionResult { deleted_documents, .. } = builder.execute()?;
+                let mut builder = milli::update::IndexDocuments::new(
+                    index_wtxn,
+                    index,
+                    indexer_config,
+                    config,
+                    |indexing_step| trace!("update: {:?}", indexing_step),
+                    || must_stop_processing.get(),
+                )?;
+
+                let document_ids = documents.iter().flatten().cloned().collect();
+
+                let (new_builder, user_result) = builder.remove_documents(document_ids)?;
+                builder = new_builder;
+                // Uses Invariant: remove documents actually always returns Ok for the inner result
+                let count = user_result.unwrap();
 
                 for (task, documents) in tasks.iter_mut().zip(documents) {
                     task.status = Status::Succeeded;
                     task.details = Some(Details::DocumentDeletion {
                         provided_ids: documents.len(),
-                        deleted_documents: Some(deleted_documents.min(documents.len() as u64)),
+                        deleted_documents: Some(count.min(documents.len() as u64)),
                     });
                 }
 
+                if !tasks.iter().all(|res| res.error.is_some()) {
+                    let addition = builder.execute()?;
+                    info!("document deletion done: {:?}", addition);
+                }
+
                 Ok(tasks)
             }
+            IndexOperation::IndexDocumentDeletionByFilter { mut task, index_uid: _ } => {
+                let filter =
+                    if let KindWithContent::DocumentDeletionByFilter { filter_expr, .. } =
+                        &task.kind
+                    {
+                        filter_expr
+                    } else {
+                        unreachable!()
+                    };
+                let deleted_documents = delete_document_by_filter(
+                    index_wtxn,
+                    filter,
+                    self.index_mapper.indexer_config(),
+                    self.must_stop_processing.clone(),
+                    index,
+                );
+                let original_filter = if let Some(Details::DocumentDeletionByFilter {
+                    original_filter,
+                    deleted_documents: _,
+                }) = task.details
+                {
+                    original_filter
+                } else {
+                    // In the case of a `documentDeleteByFilter` the details MUST be set
+                    unreachable!();
+                };
+
+                match deleted_documents {
+                    Ok(deleted_documents) => {
+                        task.status = Status::Succeeded;
+                        task.details = Some(Details::DocumentDeletionByFilter {
+                            original_filter,
+                            deleted_documents: Some(deleted_documents),
+                        });
+                    }
+                    Err(e) => {
+                        task.status = Status::Failed;
+                        task.details = Some(Details::DocumentDeletionByFilter {
+                            original_filter,
+                            deleted_documents: Some(0),
+                        });
+                        task.error = Some(e.into());
+                    }
+                }
+
+                Ok(vec![task])
+            }
             IndexOperation::Settings { index_uid: _, settings, mut tasks } => {
                 let indexer_config = self.index_mapper.indexer_config();
                 let mut builder = milli::update::Settings::new(index_wtxn, index, indexer_config);
@@ -1498,23 +1560,56 @@ impl IndexScheduler {
     }
 }
 
-fn delete_document_by_filter(filter: &serde_json::Value, index: Index) -> Result<u64> {
+fn delete_document_by_filter<'a>(
+    wtxn: &mut RwTxn<'a, '_>,
+    filter: &serde_json::Value,
+    indexer_config: &IndexerConfig,
+    must_stop_processing: MustStopProcessing,
+    index: &'a Index,
+) -> Result<u64> {
     let filter = Filter::from_json(filter)?;
     Ok(if let Some(filter) = filter {
-        let mut wtxn = index.write_txn()?;
-
-        let candidates = filter.evaluate(&wtxn, &index).map_err(|err| match err {
+        let candidates = filter.evaluate(wtxn, index).map_err(|err| match err {
             milli::Error::UserError(milli::UserError::InvalidFilter(_)) => {
                 Error::from(err).with_custom_error_code(Code::InvalidDocumentFilter)
             }
             e => e.into(),
         })?;
-        let mut delete_operation = DeleteDocuments::new(&mut wtxn, &index)?;
-        delete_operation.delete_documents(&candidates);
-        let deleted_documents =
-            delete_operation.execute().map(|result| result.deleted_documents)?;
-        wtxn.commit()?;
-        deleted_documents
+        let external_documents_ids = index.external_documents_ids();
+        // FIXME: for filters matching a lot of documents, this will allocate a huge vec of external docids (strings).
+        // Since what we have is an iterator, it would be better to delete in chunks
+        let external_to_internal: std::result::Result<Vec<_>, RoaringBitmap> =
+            external_documents_ids
+                .find_external_id_of(wtxn, candidates)?
+                .only_external_ids()
+                .collect();
+        let document_ids = match external_to_internal {
+            Ok(external_ids) => external_ids,
+            Err(remaining_ids) => panic!("Couldn't find some external ids {:?}", remaining_ids),
+        };
+
+        let config = IndexDocumentsConfig {
+            update_method: IndexDocumentsMethod::ReplaceDocuments,
+            ..Default::default()
+        };
+
+        let mut builder = milli::update::IndexDocuments::new(
+            wtxn,
+            index,
+            indexer_config,
+            config,
+            |indexing_step| debug!("update: {:?}", indexing_step),
+            || must_stop_processing.get(),
+        )?;
+
+        let (new_builder, user_result) = builder.remove_documents(document_ids)?;
+        builder = new_builder;
+        // Uses Invariant: remove documents actually always returns Ok for the inner result
+        let count = user_result.unwrap();
+
+        let _ = builder.execute()?;
+
+        count
     } else {
         0
     })

index-scheduler/src/features.rs

@@ -1,6 +1,8 @@
+use std::sync::{Arc, RwLock};
+
 use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures};
 use meilisearch_types::heed::types::{SerdeJson, Str};
-use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn};
+use meilisearch_types::heed::{Database, Env, RwTxn};
 
 use crate::error::FeatureNotEnabledError;
 use crate::Result;
@@ -9,20 +11,19 @@ const EXPERIMENTAL_FEATURES: &str = "experimental-features";
 
 #[derive(Clone)]
 pub(crate) struct FeatureData {
-    runtime: Database<Str, SerdeJson<RuntimeTogglableFeatures>>,
-    instance: InstanceTogglableFeatures,
+    persisted: Database<Str, SerdeJson<RuntimeTogglableFeatures>>,
+    runtime: Arc<RwLock<RuntimeTogglableFeatures>>,
 }
 
 #[derive(Debug, Clone, Copy)]
 pub struct RoFeatures {
     runtime: RuntimeTogglableFeatures,
-    instance: InstanceTogglableFeatures,
 }
 
 impl RoFeatures {
-    fn new(txn: RoTxn<'_>, data: &FeatureData) -> Result<Self> {
-        let runtime = data.runtime_features(txn)?;
-        Ok(Self { runtime, instance: data.instance })
+    fn new(data: &FeatureData) -> Self {
+        let runtime = data.runtime_features();
+        Self { runtime }
     }
 
     pub fn runtime_features(&self) -> RuntimeTogglableFeatures {
@@ -43,13 +44,13 @@ impl RoFeatures {
     }
 
     pub fn check_metrics(&self) -> Result<()> {
-        if self.instance.metrics {
+        if self.runtime.metrics {
             Ok(())
         } else {
             Err(FeatureNotEnabledError {
                 disabled_action: "Getting metrics",
                 feature: "metrics",
-                issue_link: "https://github.com/meilisearch/meilisearch/discussions/3518",
+                issue_link: "https://github.com/meilisearch/product/discussions/625",
             }
             .into())
         }
@@ -67,15 +68,36 @@ impl RoFeatures {
             .into())
         }
     }
+
+    pub fn check_puffin(&self) -> Result<()> {
+        if self.runtime.export_puffin_reports {
+            Ok(())
+        } else {
+            Err(FeatureNotEnabledError {
+                disabled_action: "Outputting Puffin reports to disk",
+                feature: "export puffin reports",
+                issue_link: "https://github.com/meilisearch/product/discussions/693",
+            }
+            .into())
+        }
+    }
 }
 
 impl FeatureData {
     pub fn new(env: &Env, instance_features: InstanceTogglableFeatures) -> Result<Self> {
         let mut wtxn = env.write_txn()?;
-        let runtime_features = env.create_database(&mut wtxn, Some(EXPERIMENTAL_FEATURES))?;
+        let runtime_features_db = env.create_database(&mut wtxn, Some(EXPERIMENTAL_FEATURES))?;
         wtxn.commit()?;
 
-        Ok(Self { runtime: runtime_features, instance: instance_features })
+        let txn = env.read_txn()?;
+        let persisted_features: RuntimeTogglableFeatures =
+            runtime_features_db.get(&txn, EXPERIMENTAL_FEATURES)?.unwrap_or_default();
+        let runtime = Arc::new(RwLock::new(RuntimeTogglableFeatures {
+            metrics: instance_features.metrics || persisted_features.metrics,
+            ..persisted_features
+        }));
+
+        Ok(Self { persisted: runtime_features_db, runtime })
     }
 
     pub fn put_runtime_features(
@@ -83,16 +105,25 @@ impl FeatureData {
         mut wtxn: RwTxn,
         features: RuntimeTogglableFeatures,
     ) -> Result<()> {
-        self.runtime.put(&mut wtxn, EXPERIMENTAL_FEATURES, &features)?;
+        self.persisted.put(&mut wtxn, EXPERIMENTAL_FEATURES, &features)?;
         wtxn.commit()?;
+
+        // safe to unwrap, the lock will only fail if:
+        // 1. requested by the same thread concurrently -> it is called and released in methods that don't call each other
+        // 2. there's a panic while the thread is held -> it is only used for an assignment here.
+        let mut toggled_features = self.runtime.write().unwrap();
+        *toggled_features = features;
         Ok(())
     }
 
-    fn runtime_features(&self, txn: RoTxn) -> Result<RuntimeTogglableFeatures> {
-        Ok(self.runtime.get(&txn, EXPERIMENTAL_FEATURES)?.unwrap_or_default())
+    fn runtime_features(&self) -> RuntimeTogglableFeatures {
+        // sound to unwrap, the lock will only fail if:
+        // 1. requested by the same thread concurrently -> it is called and released in methods that don't call each other
+        // 2. there's a panic while the thread is held -> it is only used for copying the data here
+        *self.runtime.read().unwrap()
     }
 
-    pub fn features(&self, txn: RoTxn) -> Result<RoFeatures> {
-        RoFeatures::new(txn, self)
+    pub fn features(&self) -> RoFeatures {
+        RoFeatures::new(self)
     }
 }

index-scheduler/src/insta_snapshot.rs

@@ -30,6 +30,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         index_mapper,
         features: _,
         max_number_of_tasks: _,
+        puffin_frame: _,
         wake_up: _,
         dumps_path: _,
         snapshots_path: _,

index-scheduler/src/lib.rs

@@ -33,6 +33,7 @@ pub type Result<T> = std::result::Result<T, Error>;
 pub type TaskId = u32;
 
 use std::collections::{BTreeMap, HashMap};
+use std::fs::File;
 use std::ops::{Bound, RangeBounds};
 use std::path::{Path, PathBuf};
 use std::sync::atomic::AtomicBool;
@@ -52,6 +53,7 @@ use meilisearch_types::milli::documents::DocumentsBatchBuilder;
 use meilisearch_types::milli::update::IndexerConfig;
 use meilisearch_types::milli::{self, CboRoaringBitmapCodec, Index, RoaringBitmapCodec, BEU32};
 use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
+use puffin::FrameView;
 use roaring::RoaringBitmap;
 use synchronoise::SignalEvent;
 use time::format_description::well_known::Rfc3339;
@@ -314,6 +316,9 @@ pub struct IndexScheduler {
     /// the finished tasks automatically.
     pub(crate) max_number_of_tasks: usize,
 
+    /// A frame to output the indexation profiling files to disk.
+    pub(crate) puffin_frame: Arc<puffin::GlobalFrameView>,
+
     /// The path used to create the dumps.
     pub(crate) dumps_path: PathBuf,
 
@@ -364,6 +369,7 @@ impl IndexScheduler {
             wake_up: self.wake_up.clone(),
             autobatching_enabled: self.autobatching_enabled,
             max_number_of_tasks: self.max_number_of_tasks,
+            puffin_frame: self.puffin_frame.clone(),
             snapshots_path: self.snapshots_path.clone(),
             dumps_path: self.dumps_path.clone(),
             auth_path: self.auth_path.clone(),
@@ -457,6 +463,7 @@ impl IndexScheduler {
             env,
             // we want to start the loop right away in case meilisearch was ctrl+Ced while processing things
             wake_up: Arc::new(SignalEvent::auto(true)),
+            puffin_frame: Arc::new(puffin::GlobalFrameView::default()),
             autobatching_enabled: options.autobatching_enabled,
             max_number_of_tasks: options.max_number_of_tasks,
             dumps_path: options.dumps_path,
@@ -572,17 +579,46 @@ impl IndexScheduler {
                 run.wake_up.wait();
 
                 loop {
+                    let puffin_enabled = run.features().check_puffin().is_ok();
+                    puffin::set_scopes_on(puffin_enabled);
+                    puffin::GlobalProfiler::lock().new_frame();
+
                     match run.tick() {
                         Ok(TickOutcome::TickAgain(_)) => (),
                         Ok(TickOutcome::WaitForSignal) => run.wake_up.wait(),
                         Err(e) => {
-                            log::error!("{}", e);
+                            log::error!("{e}");
                             // Wait one second when an irrecoverable error occurs.
                             if !e.is_recoverable() {
                                 std::thread::sleep(Duration::from_secs(1));
                             }
                         }
                     }
+
+                    // Let's write the previous frame to disk but only if
+                    // the user wanted to profile with puffin.
+                    if puffin_enabled {
+                        let mut frame_view = run.puffin_frame.lock();
+                        if !frame_view.is_empty() {
+                            let now = OffsetDateTime::now_utc();
+                            let mut file = match File::create(format!("{}.puffin", now)) {
+                                Ok(file) => file,
+                                Err(e) => {
+                                    log::error!("{e}");
+                                    continue;
+                                }
+                            };
+                            if let Err(e) = frame_view.save_to_writer(&mut file) {
+                                log::error!("{e}");
+                            }
+                            if let Err(e) = file.sync_all() {
+                                log::error!("{e}");
+                            }
+                            // We erase this frame view as it is no more useful. We want to
+                            // measure the new frames now that we exported the previous ones.
+                            *frame_view = FrameView::default();
+                        }
+                    }
                 }
             })
             .unwrap();
@@ -1062,8 +1098,6 @@ impl IndexScheduler {
             self.breakpoint(Breakpoint::Start);
         }
 
-        puffin::GlobalProfiler::lock().new_frame();
-
         self.cleanup_task_queue()?;
 
         let rtxn = self.env.read_txn().map_err(Error::HeedTransaction)?;
@@ -1259,9 +1293,8 @@ impl IndexScheduler {
         Ok(IndexStats { is_indexing, inner_stats: index_stats })
     }
 
-    pub fn features(&self) -> Result<RoFeatures> {
-        let rtxn = self.read_txn()?;
-        self.features.features(rtxn)
+    pub fn features(&self) -> RoFeatures {
+        self.features.features()
     }
 
     pub fn put_runtime_features(&self, features: RuntimeTogglableFeatures) -> Result<()> {

meilisearch-auth/src/store.rs

@@ -129,6 +129,9 @@ impl HeedAuthStore {
                 Action::DumpsAll => {
                     actions.insert(Action::DumpsCreate);
                 }
+                Action::SnapshotsAll => {
+                    actions.insert(Action::SnapshotsCreate);
+                }
                 Action::TasksAll => {
                     actions.extend([Action::TasksGet, Action::TasksDelete, Action::TasksCancel]);
                 }

meilisearch-types/src/error.rs

@@ -324,7 +324,6 @@ impl ErrorCode for milli::Error {
                     UserError::SerdeJson(_)
                     | UserError::InvalidLmdbOpenOptions
                     | UserError::DocumentLimitReached
-                    | UserError::AccessingSoftDeletedDocument { .. }
                     | UserError::UnknownInternalDocumentId { .. } => Code::Internal,
                     UserError::InvalidStoreFile => Code::InvalidStoreFile,
                     UserError::NoSpaceLeftOnDevice => Code::NoSpaceLeftOnDevice,

meilisearch-types/src/features.rs

@@ -5,6 +5,8 @@ use serde::{Deserialize, Serialize};
 pub struct RuntimeTogglableFeatures {
     pub score_details: bool,
     pub vector_store: bool,
+    pub metrics: bool,
+    pub export_puffin_reports: bool,
 }
 
 #[derive(Default, Debug, Clone, Copy)]

meilisearch-types/src/keys.rs

@@ -257,6 +257,12 @@ pub enum Action {
     #[serde(rename = "dumps.create")]
     #[deserr(rename = "dumps.create")]
     DumpsCreate,
+    #[serde(rename = "snapshots.*")]
+    #[deserr(rename = "snapshots.*")]
+    SnapshotsAll,
+    #[serde(rename = "snapshots.create")]
+    #[deserr(rename = "snapshots.create")]
+    SnapshotsCreate,
     #[serde(rename = "version")]
     #[deserr(rename = "version")]
     Version,
@@ -309,6 +315,7 @@ impl Action {
             METRICS_GET => Some(Self::MetricsGet),
             DUMPS_ALL => Some(Self::DumpsAll),
             DUMPS_CREATE => Some(Self::DumpsCreate),
+            SNAPSHOTS_CREATE => Some(Self::SnapshotsCreate),
             VERSION => Some(Self::Version),
             KEYS_CREATE => Some(Self::KeysAdd),
             KEYS_GET => Some(Self::KeysGet),
@@ -353,6 +360,7 @@ pub mod actions {
     pub const METRICS_GET: u8 = MetricsGet.repr();
     pub const DUMPS_ALL: u8 = DumpsAll.repr();
     pub const DUMPS_CREATE: u8 = DumpsCreate.repr();
+    pub const SNAPSHOTS_CREATE: u8 = SnapshotsCreate.repr();
     pub const VERSION: u8 = Version.repr();
     pub const KEYS_CREATE: u8 = KeysAdd.repr();
     pub const KEYS_GET: u8 = KeysGet.repr();

meilisearch/Cargo.toml

@@ -69,8 +69,7 @@ permissive-json-pointer = { path = "../permissive-json-pointer" }
 pin-project-lite = "0.2.9"
 platform-dirs = "0.3.0"
 prometheus = { version = "0.13.3", features = ["process"] }
-puffin = "0.16.0"
-puffin_http = { version = "0.13.0", optional = true }
+puffin = { version = "0.16.0", features = ["serialization"] }
 rand = "0.8.5"
 rayon = "1.7.0"
 regex = "1.7.3"
@@ -135,7 +134,6 @@ zip = { version = "0.6.4", optional = true }
 [features]
 default = ["analytics", "meilisearch-types/all-tokenizations", "mini-dashboard"]
 analytics = ["segment"]
-profile-with-puffin = ["dep:puffin_http"]
 mini-dashboard = [
     "actix-web-static-files",
     "static-files",

meilisearch/src/analytics/mod.rs

@@ -1,6 +1,5 @@
 mod mock_analytics;
-// if we are in release mode and the feature analytics was enabled
-#[cfg(all(not(debug_assertions), feature = "analytics"))]
+#[cfg(feature = "analytics")]
 mod segment_analytics;
 
 use std::fs;
@@ -17,26 +16,25 @@ use serde_json::Value;
 use crate::routes::indexes::documents::UpdateDocumentsQuery;
 use crate::routes::tasks::TasksFilterQuery;
 
-// if we are in debug mode OR the analytics feature is disabled
+// if the analytics feature is disabled
 // the `SegmentAnalytics` point to the mock instead of the real analytics
-#[cfg(any(debug_assertions, not(feature = "analytics")))]
+#[cfg(not(feature = "analytics"))]
 pub type SegmentAnalytics = mock_analytics::MockAnalytics;
-#[cfg(any(debug_assertions, not(feature = "analytics")))]
+#[cfg(not(feature = "analytics"))]
 pub type SearchAggregator = mock_analytics::SearchAggregator;
-#[cfg(any(debug_assertions, not(feature = "analytics")))]
+#[cfg(not(feature = "analytics"))]
 pub type MultiSearchAggregator = mock_analytics::MultiSearchAggregator;
-#[cfg(any(debug_assertions, not(feature = "analytics")))]
+#[cfg(not(feature = "analytics"))]
 pub type FacetSearchAggregator = mock_analytics::FacetSearchAggregator;
 
-// if we are in release mode and the feature analytics was enabled
-// we use the real analytics
-#[cfg(all(not(debug_assertions), feature = "analytics"))]
+// if the feature analytics is enabled we use the real analytics
+#[cfg(feature = "analytics")]
 pub type SegmentAnalytics = segment_analytics::SegmentAnalytics;
-#[cfg(all(not(debug_assertions), feature = "analytics"))]
+#[cfg(feature = "analytics")]
 pub type SearchAggregator = segment_analytics::SearchAggregator;
-#[cfg(all(not(debug_assertions), feature = "analytics"))]
+#[cfg(feature = "analytics")]
 pub type MultiSearchAggregator = segment_analytics::MultiSearchAggregator;
-#[cfg(all(not(debug_assertions), feature = "analytics"))]
+#[cfg(feature = "analytics")]
 pub type FacetSearchAggregator = segment_analytics::FacetSearchAggregator;
 
 /// The Meilisearch config dir:

meilisearch/src/lib.rs

@@ -114,10 +114,7 @@ pub fn create_app(
         .configure(routes::configure)
         .configure(|s| dashboard(s, enable_dashboard));
 
-    let app = app.wrap(actix_web::middleware::Condition::new(
-        opt.experimental_enable_metrics,
-        middleware::RouteMetrics,
-    ));
+    let app = app.wrap(middleware::RouteMetrics);
     app.wrap(
         Cors::default()
             .send_wildcard()
@@ -365,7 +362,7 @@ fn import_dump(
                 update_method: IndexDocumentsMethod::ReplaceDocuments,
                 ..Default::default()
             },
-            |indexing_step| log::debug!("update: {:?}", indexing_step),
+            |indexing_step| log::trace!("update: {:?}", indexing_step),
             || false,
         )?;
 

meilisearch/src/main.rs

@@ -30,10 +30,6 @@ fn setup(opt: &Opt) -> anyhow::Result<()> {
 async fn main() -> anyhow::Result<()> {
     let (opt, config_read_from) = Opt::try_build()?;
 
-    #[cfg(feature = "profile-with-puffin")]
-    let _server = puffin_http::Server::new(&format!("0.0.0.0:{}", puffin_http::DEFAULT_PORT))?;
-    puffin::set_scopes_on(cfg!(feature = "profile-with-puffin"));
-
     anyhow::ensure!(
         !(cfg!(windows) && opt.experimental_reduce_indexing_memory_usage),
         "The `experimental-reduce-indexing-memory-usage` flag is not supported on Windows"

meilisearch/src/middleware.rs

@@ -3,8 +3,10 @@
 use std::future::{ready, Ready};
 
 use actix_web::dev::{self, Service, ServiceRequest, ServiceResponse, Transform};
+use actix_web::web::Data;
 use actix_web::Error;
 use futures_util::future::LocalBoxFuture;
+use index_scheduler::IndexScheduler;
 use prometheus::HistogramTimer;
 
 pub struct RouteMetrics;
@@ -47,19 +49,27 @@ where
 
     fn call(&self, req: ServiceRequest) -> Self::Future {
         let mut histogram_timer: Option<HistogramTimer> = None;
-        let request_path = req.path();
-        let is_registered_resource = req.resource_map().has_resource(request_path);
-        if is_registered_resource {
-            let request_method = req.method().to_string();
-            histogram_timer = Some(
-                crate::metrics::MEILISEARCH_HTTP_RESPONSE_TIME_SECONDS
+
+        // calling unwrap here is safe because index scheduler is added to app data while creating actix app.
+        // also, the tests will fail if this is not present.
+        let index_scheduler = req.app_data::<Data<IndexScheduler>>().unwrap();
+        let features = index_scheduler.features();
+
+        if features.check_metrics().is_ok() {
+            let request_path = req.path();
+            let is_registered_resource = req.resource_map().has_resource(request_path);
+            if is_registered_resource {
+                let request_method = req.method().to_string();
+                histogram_timer = Some(
+                    crate::metrics::MEILISEARCH_HTTP_RESPONSE_TIME_SECONDS
+                        .with_label_values(&[&request_method, request_path])
+                        .start_timer(),
+                );
+                crate::metrics::MEILISEARCH_HTTP_REQUESTS_TOTAL
                     .with_label_values(&[&request_method, request_path])
-                    .start_timer(),
-            );
-            crate::metrics::MEILISEARCH_HTTP_REQUESTS_TOTAL
-                .with_label_values(&[&request_method, request_path])
-                .inc();
-        }
+                    .inc();
+            }
+        };
 
         let fut = self.service.call(req);
 

meilisearch/src/option.rs

@@ -28,7 +28,7 @@ const MEILI_DB_PATH: &str = "MEILI_DB_PATH";
 const MEILI_HTTP_ADDR: &str = "MEILI_HTTP_ADDR";
 const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY";
 const MEILI_ENV: &str = "MEILI_ENV";
-#[cfg(all(not(debug_assertions), feature = "analytics"))]
+#[cfg(feature = "analytics")]
 const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS";
 const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT";
 const MEILI_SSL_CERT_PATH: &str = "MEILI_SSL_CERT_PATH";
@@ -159,7 +159,7 @@ pub struct Opt {
     /// Meilisearch automatically collects data from all instances that do not opt out using this flag.
     /// All gathered data is used solely for the purpose of improving Meilisearch, and can be deleted
     /// at any time.
-    #[cfg(all(not(debug_assertions), feature = "analytics"))]
+    #[cfg(feature = "analytics")]
     #[serde(default)] // we can't send true
     #[clap(long, env = MEILI_NO_ANALYTICS)]
     pub no_analytics: bool,
@@ -390,7 +390,7 @@ impl Opt {
             ignore_missing_dump: _,
             ignore_dump_if_db_exists: _,
             config_file_path: _,
-            #[cfg(all(not(debug_assertions), feature = "analytics"))]
+            #[cfg(feature = "analytics")]
             no_analytics,
             experimental_enable_metrics: enable_metrics_route,
             experimental_reduce_indexing_memory_usage: reduce_indexing_memory_usage,
@@ -401,7 +401,7 @@ impl Opt {
             export_to_env_if_not_present(MEILI_MASTER_KEY, master_key);
         }
         export_to_env_if_not_present(MEILI_ENV, env);
-        #[cfg(all(not(debug_assertions), feature = "analytics"))]
+        #[cfg(feature = "analytics")]
         {
             export_to_env_if_not_present(MEILI_NO_ANALYTICS, no_analytics.to_string());
         }

meilisearch/src/routes/features.rs

@@ -29,12 +29,12 @@ async fn get_features(
     >,
     req: HttpRequest,
     analytics: Data<dyn Analytics>,
-) -> Result<HttpResponse, ResponseError> {
-    let features = index_scheduler.features()?;
+) -> HttpResponse {
+    let features = index_scheduler.features();
 
     analytics.publish("Experimental features Seen".to_string(), json!(null), Some(&req));
     debug!("returns: {:?}", features.runtime_features());
-    Ok(HttpResponse::Ok().json(features.runtime_features()))
+    HttpResponse::Ok().json(features.runtime_features())
 }
 
 #[derive(Debug, Deserr)]
@@ -44,6 +44,10 @@ pub struct RuntimeTogglableFeatures {
     pub score_details: Option<bool>,
     #[deserr(default)]
     pub vector_store: Option<bool>,
+    #[deserr(default)]
+    pub metrics: Option<bool>,
+    #[deserr(default)]
+    pub export_puffin_reports: Option<bool>,
 }
 
 async fn patch_features(
@@ -55,26 +59,36 @@ async fn patch_features(
     req: HttpRequest,
     analytics: Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    let features = index_scheduler.features()?;
+    let features = index_scheduler.features();
 
     let old_features = features.runtime_features();
-
     let new_features = meilisearch_types::features::RuntimeTogglableFeatures {
         score_details: new_features.0.score_details.unwrap_or(old_features.score_details),
         vector_store: new_features.0.vector_store.unwrap_or(old_features.vector_store),
+        metrics: new_features.0.metrics.unwrap_or(old_features.metrics),
+        export_puffin_reports: new_features
+            .0
+            .export_puffin_reports
+            .unwrap_or(old_features.export_puffin_reports),
     };
 
     // explicitly destructure for analytics rather than using the `Serialize` implementation, because
     // the it renames to camelCase, which we don't want for analytics.
     // **Do not** ignore fields with `..` or `_` here, because we want to add them in the future.
-    let meilisearch_types::features::RuntimeTogglableFeatures { score_details, vector_store } =
-        new_features;
+    let meilisearch_types::features::RuntimeTogglableFeatures {
+        score_details,
+        vector_store,
+        metrics,
+        export_puffin_reports,
+    } = new_features;
 
     analytics.publish(
         "Experimental features Updated".to_string(),
         json!({
             "score_details": score_details,
             "vector_store": vector_store,
+            "metrics": metrics,
+            "export_puffin_reports": export_puffin_reports,
         }),
         Some(&req),
     );

meilisearch/src/routes/indexes/documents.rs

@@ -612,8 +612,8 @@ fn retrieve_document<S: AsRef<str>>(
     let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
 
     let internal_id = index
-        .external_documents_ids(&txn)?
-        .get(doc_id.as_bytes())
+        .external_documents_ids()
+        .get(&txn, doc_id)?
         .ok_or_else(|| MeilisearchHttpError::DocumentNotFound(doc_id.to_string()))?;
 
     let document = index

meilisearch/src/routes/indexes/facet_search.rs

@@ -68,7 +68,7 @@ pub async fn search(
     }
 
     let index = index_scheduler.index(&index_uid)?;
-    let features = index_scheduler.features()?;
+    let features = index_scheduler.features();
     let search_result = tokio::task::spawn_blocking(move || {
         perform_facet_search(&index, search_query, facet_query, facet_name, features)
     })

meilisearch/src/routes/indexes/search.rs

@@ -157,7 +157,7 @@ pub async fn search_with_url_query(
     let mut aggregate = SearchAggregator::from_query(&query, &req);
 
     let index = index_scheduler.index(&index_uid)?;
-    let features = index_scheduler.features()?;
+    let features = index_scheduler.features();
     let search_result =
         tokio::task::spawn_blocking(move || perform_search(&index, query, features)).await?;
     if let Ok(ref search_result) = search_result {
@@ -192,7 +192,7 @@ pub async fn search_with_post(
 
     let index = index_scheduler.index(&index_uid)?;
 
-    let features = index_scheduler.features()?;
+    let features = index_scheduler.features();
     let search_result =
         tokio::task::spawn_blocking(move || perform_search(&index, query, features)).await?;
     if let Ok(ref search_result) = search_result {

meilisearch/src/routes/metrics.rs

@@ -19,7 +19,7 @@ pub async fn get_metrics(
     index_scheduler: GuardedData<ActionPolicy<{ actions::METRICS_GET }>, Data<IndexScheduler>>,
     auth_controller: Data<AuthController>,
 ) -> Result<HttpResponse, ResponseError> {
-    index_scheduler.features()?.check_metrics()?;
+    index_scheduler.features().check_metrics()?;
     let auth_filters = index_scheduler.filters();
     if !auth_filters.all_indexes_authorized() {
         let mut error = ResponseError::from(AuthenticationError::InvalidToken);

meilisearch/src/routes/mod.rs

@@ -24,6 +24,7 @@ pub mod features;
 pub mod indexes;
 mod metrics;
 mod multi_search;
+mod snapshot;
 mod swap_indexes;
 pub mod tasks;
 
@@ -32,6 +33,7 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         .service(web::resource("/health").route(web::get().to(get_health)))
         .service(web::scope("/keys").configure(api_key::configure))
         .service(web::scope("/dumps").configure(dump::configure))
+        .service(web::scope("/snapshots").configure(snapshot::configure))
         .service(web::resource("/stats").route(web::get().to(get_stats)))
         .service(web::resource("/version").route(web::get().to(get_version)))
         .service(web::scope("/indexes").configure(indexes::configure))

meilisearch/src/routes/multi_search.rs

@@ -41,7 +41,7 @@ pub async fn multi_search_with_post(
     let queries = params.into_inner().queries;
 
     let mut multi_aggregate = MultiSearchAggregator::from_queries(&queries, &req);
-    let features = index_scheduler.features()?;
+    let features = index_scheduler.features();
 
     // Explicitly expect a `(ResponseError, usize)` for the error type rather than `ResponseError` only,
     // so that `?` doesn't work if it doesn't use `with_index`, ensuring that it is not forgotten in case of code

meilisearch/src/routes/snapshot.rs

@@ -0,0 +1,32 @@
+use actix_web::web::Data;
+use actix_web::{web, HttpRequest, HttpResponse};
+use index_scheduler::IndexScheduler;
+use log::debug;
+use meilisearch_types::error::ResponseError;
+use meilisearch_types::tasks::KindWithContent;
+use serde_json::json;
+
+use crate::analytics::Analytics;
+use crate::extractors::authentication::policies::*;
+use crate::extractors::authentication::GuardedData;
+use crate::extractors::sequential_extractor::SeqHandler;
+use crate::routes::SummarizedTaskView;
+
+pub fn configure(cfg: &mut web::ServiceConfig) {
+    cfg.service(web::resource("").route(web::post().to(SeqHandler(create_snapshot))));
+}
+
+pub async fn create_snapshot(
+    index_scheduler: GuardedData<ActionPolicy<{ actions::SNAPSHOTS_CREATE }>, Data<IndexScheduler>>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
+) -> Result<HttpResponse, ResponseError> {
+    analytics.publish("Snapshot Created".to_string(), json!({}), Some(&req));
+
+    let task = KindWithContent::SnapshotCreation;
+    let task: SummarizedTaskView =
+        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+
+    debug!("returns: {:?}", task);
+    Ok(HttpResponse::Accepted().json(task))
+}

meilisearch/tests/auth/api_keys.rs

@@ -1,8 +1,7 @@
 use std::{thread, time};
 
-use serde_json::{json, Value};
-
-use crate::common::Server;
+use crate::common::{Server, Value};
+use crate::json;
 
 #[actix_rt::test]
 async fn add_valid_api_key() {
@@ -162,7 +161,7 @@ async fn add_valid_api_key_null_description() {
     server.use_api_key("MASTER_KEY");
 
     let content = json!({
-        "description": Value::Null,
+        "description": json!(null),
         "indexes": ["products"],
         "actions": ["documents.add"],
         "expiresAt": "2050-11-13T00:00:00"
@@ -365,7 +364,7 @@ async fn error_add_api_key_invalid_index_uids() {
     server.use_api_key("MASTER_KEY");
 
     let content = json!({
-        "description": Value::Null,
+        "description": json!(null),
         "indexes": ["invalid index # / \\name with spaces"],
         "actions": [
             "documents.add"
@@ -422,7 +421,7 @@ async fn error_add_api_key_invalid_parameters_actions() {
     meili_snap::snapshot!(code, @"400 Bad Request");
     meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r###"
     {
-      "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
+      "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
       "code": "invalid_api_key_actions",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions"
@@ -507,7 +506,7 @@ async fn error_add_api_key_invalid_parameters_uid() {
 async fn error_add_api_key_parameters_uid_already_exist() {
     let mut server = Server::new_auth().await;
     server.use_api_key("MASTER_KEY");
-    let content = json!({
+    let content: Value = json!({
         "uid": "4bc0887a-0e41-4f3b-935d-0c451dcee9c8",
         "indexes": ["products"],
         "actions": ["search"],
@@ -1146,7 +1145,7 @@ async fn patch_api_key_description() {
     meili_snap::snapshot!(code, @"200 OK");
 
     // Remove the description
-    let content = json!({ "description": serde_json::Value::Null });
+    let content = json!({ "description": null });
 
     let (response, code) = server.patch_api_key(&uid, content).await;
     meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]", ".uid" => "[ignored]", ".key" => "[ignored]" }), @r###"

meilisearch/tests/auth/authorization.rs

@@ -2,11 +2,13 @@ use std::collections::{HashMap, HashSet};
 
 use ::time::format_description::well_known::Rfc3339;
 use maplit::{hashmap, hashset};
+use meilisearch::Opt;
 use once_cell::sync::Lazy;
-use serde_json::{json, Value};
+use tempfile::TempDir;
 use time::{Duration, OffsetDateTime};
 
-use crate::common::Server;
+use crate::common::{default_settings, Server, Value};
+use crate::json;
 
 pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'static str>>> =
     Lazy::new(|| {
@@ -54,6 +56,7 @@ pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'
             ("GET",     "/indexes/products/stats") =>                          hashset!{"stats.get", "stats.*", "*"},
             ("GET",     "/stats") =>                                           hashset!{"stats.get", "stats.*", "*"},
             ("POST",    "/dumps") =>                                           hashset!{"dumps.create", "dumps.*", "*"},
+            ("POST",    "/snapshots") =>                                       hashset!{"snapshots.create", "snapshots.*", "*"},
             ("GET",     "/version") =>                                         hashset!{"version", "*"},
             ("GET",     "/metrics") =>                                         hashset!{"metrics.get", "metrics.*", "*"},
             ("PATCH",   "/keys/mykey/") =>                                     hashset!{"keys.update", "*"},
@@ -194,7 +197,9 @@ async fn access_authorized_master_key() {
 
 #[actix_rt::test]
 async fn access_authorized_restricted_index() {
-    let mut server = Server::new_auth().await;
+    let dir = TempDir::new().unwrap();
+    let enable_metrics = Opt { experimental_enable_metrics: true, ..default_settings(dir.path()) };
+    let mut server = Server::new_auth_with_options(enable_metrics, dir).await;
     for ((method, route), actions) in AUTHORIZATIONS.iter() {
         for action in actions {
             // create a new API key letting only the needed action.

meilisearch/tests/auth/errors.rs

@@ -1,8 +1,8 @@
 use meili_snap::*;
-use serde_json::json;
 use uuid::Uuid;
 
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn create_api_key_bad_description() {
@@ -90,7 +90,7 @@ async fn create_api_key_bad_actions() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
+      "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
       "code": "invalid_api_key_actions",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions"

meilisearch/tests/auth/mod.rs

@@ -7,9 +7,9 @@ mod tenant_token;
 mod tenant_token_multi_search;
 
 use actix_web::http::StatusCode;
-use serde_json::{json, Value};
 
-use crate::common::Server;
+use crate::common::{Server, Value};
+use crate::json;
 
 impl Server {
     pub fn use_api_key(&mut self, api_key: impl AsRef<str>) {

meilisearch/tests/auth/tenant_token.rs

@@ -3,11 +3,11 @@ use std::collections::HashMap;
 use ::time::format_description::well_known::Rfc3339;
 use maplit::hashmap;
 use once_cell::sync::Lazy;
-use serde_json::{json, Value};
 use time::{Duration, OffsetDateTime};
 
 use super::authorization::{ALL_ACTIONS, AUTHORIZATIONS};
-use crate::common::Server;
+use crate::common::{Server, Value};
+use crate::json;
 
 fn generate_tenant_token(
     parent_uid: impl AsRef<str>,
@@ -233,31 +233,31 @@ async fn search_authorized_simple_token() {
         },
         hashmap! {
             "searchRules" => json!({"*": {}}),
-            "exp" => Value::Null
+            "exp" => json!(null)
         },
         hashmap! {
-            "searchRules" => json!({"*": Value::Null}),
-            "exp" => Value::Null
+            "searchRules" => json!({"*": null}),
+            "exp" => json!(null)
         },
         hashmap! {
             "searchRules" => json!(["*"]),
-            "exp" => Value::Null
+            "exp" => json!(null)
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
-            "exp" => Value::Null
+            "exp" => json!(null)
         },
         hashmap! {
-            "searchRules" => json!({"sales": Value::Null}),
-            "exp" => Value::Null
+            "searchRules" => json!({"sales": null}),
+            "exp" => json!(null)
         },
         hashmap! {
             "searchRules" => json!(["sales"]),
-            "exp" => Value::Null
+            "exp" => json!(null)
         },
         hashmap! {
             "searchRules" => json!(["sa*"]),
-            "exp" => Value::Null
+            "exp" => json!(null)
         },
     ];
 
@@ -386,7 +386,7 @@ async fn error_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": Value::Null}),
+            "searchRules" => json!({"*": null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -398,7 +398,7 @@ async fn error_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": Value::Null}),
+            "searchRules" => json!({"sales": null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -428,15 +428,15 @@ async fn error_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"products": {}}),
-            "exp" => Value::Null
+            "exp" => json!(null)
         },
         hashmap! {
-            "searchRules" => json!({"products": Value::Null}),
-            "exp" => Value::Null
+            "searchRules" => json!({"products": null}),
+            "exp" => json!(null)
         },
         hashmap! {
             "searchRules" => json!(["products"]),
-            "exp" => Value::Null
+            "exp" => json!(null)
         },
         // expired token
         hashmap! {
@@ -444,7 +444,7 @@ async fn error_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": Value::Null}),
+            "searchRules" => json!({"*": null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -456,7 +456,7 @@ async fn error_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": Value::Null}),
+            "searchRules" => json!({"sales": null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {

meilisearch/tests/auth/tenant_token_multi_search.rs

@@ -3,11 +3,11 @@ use std::collections::HashMap;
 use ::time::format_description::well_known::Rfc3339;
 use maplit::hashmap;
 use once_cell::sync::Lazy;
-use serde_json::{json, Value};
 use time::{Duration, OffsetDateTime};
 
 use super::authorization::ALL_ACTIONS;
-use crate::common::Server;
+use crate::common::{Server, Value};
+use crate::json;
 
 fn generate_tenant_token(
     parent_uid: impl AsRef<str>,
@@ -512,31 +512,31 @@ async fn single_search_authorized_simple_token() {
         },
         hashmap! {
             "searchRules" => json!({"*": {}}),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         hashmap! {
-            "searchRules" => json!({"*": Value::Null}),
-            "exp" => Value::Null
+            "searchRules" => json!({"*": null}),
+            "exp" => json!(null),
         },
         hashmap! {
             "searchRules" => json!(["*"]),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         hashmap! {
-            "searchRules" => json!({"sales": Value::Null}),
-            "exp" => Value::Null
+            "searchRules" => json!({"sales": null}),
+            "exp" => json!(null),
         },
         hashmap! {
             "searchRules" => json!(["sales"]),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         hashmap! {
             "searchRules" => json!(["sa*"]),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
     ];
 
@@ -564,31 +564,31 @@ async fn multi_search_authorized_simple_token() {
         },
         hashmap! {
             "searchRules" => json!({"*": {}}),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         hashmap! {
-            "searchRules" => json!({"*": Value::Null}),
-            "exp" => Value::Null
+            "searchRules" => json!({"*": null}),
+            "exp" => json!(null),
         },
         hashmap! {
             "searchRules" => json!(["*"]),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         hashmap! {
             "searchRules" => json!({"sales": {}, "products": {}}),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         hashmap! {
-            "searchRules" => json!({"sales": Value::Null, "products": Value::Null}),
-            "exp" => Value::Null
+            "searchRules" => json!({"sales": null, "products": null}),
+            "exp" => json!(null),
         },
         hashmap! {
             "searchRules" => json!(["sales", "products"]),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         hashmap! {
             "searchRules" => json!(["sa*", "pro*"]),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
     ];
 
@@ -823,7 +823,7 @@ async fn error_single_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": Value::Null}),
+            "searchRules" => json!({"*": null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -835,7 +835,7 @@ async fn error_single_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": Value::Null}),
+            "searchRules" => json!({"sales": null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -864,7 +864,7 @@ async fn error_multi_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": Value::Null}),
+            "searchRules" => json!({"*": null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -876,7 +876,7 @@ async fn error_multi_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": Value::Null, "products": Value::Null}),
+            "searchRules" => json!({"sales": null, "products": null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -919,15 +919,15 @@ async fn error_single_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"products": {}}),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         hashmap! {
-            "searchRules" => json!({"products": Value::Null}),
-            "exp" => Value::Null
+            "searchRules" => json!({"products": null}),
+            "exp" => json!(null),
         },
         hashmap! {
             "searchRules" => json!(["products"]),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         // expired token
         hashmap! {
@@ -935,7 +935,7 @@ async fn error_single_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": Value::Null}),
+            "searchRules" => json!({"*": null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -947,7 +947,7 @@ async fn error_single_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": Value::Null}),
+            "searchRules" => json!({"sales": null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -978,15 +978,15 @@ async fn error_multi_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"products": {}}),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         hashmap! {
-            "searchRules" => json!({"products": Value::Null}),
-            "exp" => Value::Null
+            "searchRules" => json!({"products": null}),
+            "exp" => json!(null),
         },
         hashmap! {
             "searchRules" => json!(["products"]),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
@@ -998,15 +998,15 @@ async fn error_multi_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         hashmap! {
-            "searchRules" => json!({"sales": Value::Null}),
-            "exp" => Value::Null
+            "searchRules" => json!({"sales": null}),
+            "exp" => json!(null),
         },
         hashmap! {
             "searchRules" => json!(["sales"]),
-            "exp" => Value::Null
+            "exp" => json!(null),
         },
         // expired token
         hashmap! {
@@ -1014,7 +1014,7 @@ async fn error_multi_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": Value::Null}),
+            "searchRules" => json!({"*": null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -1026,7 +1026,7 @@ async fn error_multi_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": Value::Null, "products": {}}),
+            "searchRules" => json!({"sales": null, "products": {}}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {

meilisearch/tests/common/index.rs

@@ -3,12 +3,13 @@ use std::panic::{catch_unwind, resume_unwind, UnwindSafe};
 use std::time::Duration;
 
 use actix_web::http::StatusCode;
-use serde_json::{json, Value};
 use tokio::time::sleep;
 use urlencoding::encode as urlencode;
 
 use super::encoder::Encoder;
 use super::service::Service;
+use super::Value;
+use crate::json;
 
 pub struct Index<'a> {
     pub uid: String,
@@ -242,7 +243,9 @@ impl Index<'_> {
 
     pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/documents/delete-batch", urlencode(self.uid.as_ref()));
-        self.service.post_encoded(url, serde_json::to_value(&ids).unwrap(), self.encoder).await
+        self.service
+            .post_encoded(url, serde_json::to_value(&ids).unwrap().into(), self.encoder)
+            .await
     }
 
     pub async fn delete_batch_raw(&self, body: Value) -> (Value, StatusCode) {

meilisearch/tests/common/mod.rs

@@ -3,9 +3,83 @@ pub mod index;
 pub mod server;
 pub mod service;
 
+use std::fmt::{self, Display};
+
 pub use index::{GetAllDocumentsOptions, GetDocumentOptions};
+use meili_snap::json_string;
+use serde::{Deserialize, Serialize};
 pub use server::{default_settings, Server};
 
+#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
+pub struct Value(pub serde_json::Value);
+
+impl Value {
+    pub fn uid(&self) -> u64 {
+        if let Some(uid) = self["uid"].as_u64() {
+            uid
+        } else if let Some(uid) = self["taskUid"].as_u64() {
+            uid
+        } else {
+            panic!("Didn't find any task id in: {self}");
+        }
+    }
+}
+
+impl From<serde_json::Value> for Value {
+    fn from(value: serde_json::Value) -> Self {
+        Value(value)
+    }
+}
+
+impl std::ops::Deref for Value {
+    type Target = serde_json::Value;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+impl PartialEq<serde_json::Value> for Value {
+    fn eq(&self, other: &serde_json::Value) -> bool {
+        &self.0 == other
+    }
+}
+
+impl PartialEq<Value> for serde_json::Value {
+    fn eq(&self, other: &Value) -> bool {
+        self == &other.0
+    }
+}
+
+impl PartialEq<&str> for Value {
+    fn eq(&self, other: &&str) -> bool {
+        self.0.eq(other)
+    }
+}
+
+impl Display for Value {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            f,
+            "{}",
+            json_string!(self, { ".enqueuedAt" => "[date]", ".processedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" })
+        )
+    }
+}
+
+impl From<Vec<Value>> for Value {
+    fn from(value: Vec<Value>) -> Self {
+        Self(value.into_iter().map(|value| value.0).collect::<serde_json::Value>())
+    }
+}
+
+#[macro_export]
+macro_rules! json {
+    ($($json:tt)+) => {
+        $crate::common::Value(serde_json::json!($($json)+))
+    };
+}
+
 /// Performs a search test on both post and get routes
 #[macro_export]
 macro_rules! test_post_get_search {

meilisearch/tests/common/server.rs

@@ -11,13 +11,14 @@ use clap::Parser;
 use meilisearch::option::{IndexerOpts, MaxMemory, Opt};
 use meilisearch::{analytics, create_app, setup_meilisearch};
 use once_cell::sync::Lazy;
-use serde_json::{json, Value};
 use tempfile::TempDir;
 use tokio::time::sleep;
 
 use super::index::Index;
 use super::service::Service;
 use crate::common::encoder::Encoder;
+use crate::common::Value;
+use crate::json;
 
 pub struct Server {
     pub service: Service,
@@ -156,6 +157,10 @@ impl Server {
         self.service.post("/dumps", json!(null)).await
     }
 
+    pub async fn create_snapshot(&self) -> (Value, StatusCode) {
+        self.service.post("/snapshots", json!(null)).await
+    }
+
     pub async fn index_swap(&self, value: Value) -> (Value, StatusCode) {
         self.service.post("/swap-indexes", value).await
     }
@@ -197,6 +202,10 @@ impl Server {
     pub async fn set_features(&self, value: Value) -> (Value, StatusCode) {
         self.service.patch("/experimental-features", value).await
     }
+
+    pub async fn get_metrics(&self) -> (Value, StatusCode) {
+        self.service.get("/metrics").await
+    }
 }
 
 pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
@@ -204,7 +213,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
         db_path: dir.as_ref().join("db"),
         dump_dir: dir.as_ref().join("dumps"),
         env: "development".to_owned(),
-        #[cfg(all(not(debug_assertions), feature = "analytics"))]
+        #[cfg(feature = "analytics")]
         no_analytics: true,
         max_index_size: Byte::from_unit(100.0, ByteUnit::MiB).unwrap(),
         max_task_db_size: Byte::from_unit(1.0, ByteUnit::GiB).unwrap(),
@@ -216,7 +225,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
             skip_index_budget: true,
             ..Parser::parse_from(None as Option<&str>)
         },
-        experimental_enable_metrics: true,
+        experimental_enable_metrics: false,
         ..Parser::parse_from(None as Option<&str>)
     }
 }

meilisearch/tests/common/service.rs

@@ -7,9 +7,9 @@ use actix_web::test::TestRequest;
 use index_scheduler::IndexScheduler;
 use meilisearch::{analytics, create_app, Opt};
 use meilisearch_auth::AuthController;
-use serde_json::Value;
 
 use crate::common::encoder::Encoder;
+use crate::common::Value;
 
 pub struct Service {
     pub index_scheduler: Arc<IndexScheduler>,

meilisearch/tests/content_type.rs

@@ -3,9 +3,8 @@
 mod common;
 
 use actix_web::test;
-use serde_json::{json, Value};
 
-use crate::common::Server;
+use crate::common::{Server, Value};
 
 enum HttpVerb {
     Put,

meilisearch/tests/documents/add_documents.rs

@@ -1,11 +1,11 @@
 use actix_web::test;
 use meili_snap::{json_string, snapshot};
-use serde_json::{json, Value};
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::encoder::Encoder;
-use crate::common::{GetAllDocumentsOptions, Server};
+use crate::common::{GetAllDocumentsOptions, Server, Value};
+use crate::json;
 
 /// This is the basic usage of our API and every other tests uses the content-type application/json
 #[actix_rt::test]

meilisearch/tests/documents/delete_documents.rs

@@ -1,7 +1,7 @@
 use meili_snap::{json_string, snapshot};
-use serde_json::json;
 
 use crate::common::{GetAllDocumentsOptions, Server};
+use crate::json;
 
 #[actix_rt::test]
 async fn delete_one_document_unexisting_index() {
@@ -154,6 +154,19 @@ async fn delete_document_by_filter() {
         )
         .await;
     index.wait_task(1).await;
+
+    let (stats, _) = index.stats().await;
+    snapshot!(json_string!(stats), @r###"
+    {
+      "numberOfDocuments": 4,
+      "isIndexing": false,
+      "fieldDistribution": {
+        "color": 3,
+        "id": 4
+      }
+    }
+    "###);
+
     let (response, code) =
         index.delete_document_by_filter(json!({ "filter": "color = blue"})).await;
     snapshot!(code, @"202 Accepted");
@@ -188,6 +201,18 @@ async fn delete_document_by_filter() {
     }
     "###);
 
+    let (stats, _) = index.stats().await;
+    snapshot!(json_string!(stats), @r###"
+    {
+      "numberOfDocuments": 2,
+      "isIndexing": false,
+      "fieldDistribution": {
+        "color": 1,
+        "id": 2
+      }
+    }
+    "###);
+
     let (documents, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await;
     snapshot!(code, @"200 OK");
     snapshot!(json_string!(documents), @r###"
@@ -241,6 +266,18 @@ async fn delete_document_by_filter() {
     }
     "###);
 
+    let (stats, _) = index.stats().await;
+    snapshot!(json_string!(stats), @r###"
+    {
+      "numberOfDocuments": 1,
+      "isIndexing": false,
+      "fieldDistribution": {
+        "color": 1,
+        "id": 1
+      }
+    }
+    "###);
+
     let (documents, code) = index.get_all_documents(GetAllDocumentsOptions::default()).await;
     snapshot!(code, @"200 OK");
     snapshot!(json_string!(documents), @r###"
@@ -360,7 +397,7 @@ async fn delete_document_by_complex_filter() {
       "canceledBy": null,
       "details": {
         "providedIds": 0,
-        "deletedDocuments": 4,
+        "deletedDocuments": 2,
         "originalFilter": "[[\"color = green\",\"color NOT EXISTS\"]]"
       },
       "error": null,

meilisearch/tests/documents/errors.rs

@@ -1,8 +1,8 @@
 use meili_snap::*;
-use serde_json::json;
 use urlencoding::encode;
 
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn get_all_documents_bad_offset() {

meilisearch/tests/documents/get_documents.rs

@@ -1,11 +1,11 @@
 use actix_web::test;
 use http::header::ACCEPT_ENCODING;
 use meili_snap::*;
-use serde_json::{json, Value};
 use urlencoding::encode as urlencode;
 
 use crate::common::encoder::Encoder;
-use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server};
+use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server, Value};
+use crate::json;
 
 // TODO: partial test since we are testing error, amd error is not yet fully implemented in
 // transplant
@@ -40,7 +40,7 @@ async fn get_document() {
     let server = Server::new().await;
     let index = server.index("test");
     index.create(None).await;
-    let documents = serde_json::json!([
+    let documents = json!([
         {
             "id": 0,
             "nested": { "content": "foobar" },
@@ -53,7 +53,7 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        serde_json::json!({
+        json!({
             "id": 0,
             "nested": { "content": "foobar" },
         })
@@ -64,7 +64,7 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        serde_json::json!({
+        json!({
             "id": 0,
         })
     );
@@ -75,7 +75,7 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        serde_json::json!({
+        json!({
             "nested": { "content": "foobar" },
         })
     );
@@ -122,7 +122,7 @@ async fn get_all_documents_no_options() {
     assert_eq!(code, 200);
     let arr = response["results"].as_array().unwrap();
     assert_eq!(arr.len(), 20);
-    let first = serde_json::json!({
+    let first = json!({
         "id":0,
         "isActive":false,
         "balance":"$2,668.55",

meilisearch/tests/documents/update_documents.rs

@@ -1,7 +1,8 @@
-use serde_json::json;
+use meili_snap::snapshot;
 
 use crate::common::encoder::Encoder;
 use crate::common::{GetAllDocumentsOptions, Server};
+use crate::json;
 
 #[actix_rt::test]
 async fn error_document_update_create_index_bad_uid() {
@@ -84,7 +85,13 @@ async fn update_document() {
 
     let (response, code) = index.get_document(1, None).await;
     assert_eq!(code, 200);
-    assert_eq!(response.to_string(), r##"{"doc_id":1,"content":"foo","other":"bar"}"##);
+    snapshot!(response, @r###"
+    {
+      "doc_id": 1,
+      "content": "foo",
+      "other": "bar"
+    }
+    "###);
 }
 
 #[actix_rt::test]
@@ -122,7 +129,13 @@ async fn update_document_gzip_encoded() {
 
     let (response, code) = index.get_document(1, None).await;
     assert_eq!(code, 200);
-    assert_eq!(response.to_string(), r##"{"doc_id":1,"content":"foo","other":"bar"}"##);
+    snapshot!(response, @r###"
+    {
+      "doc_id": 1,
+      "content": "foo",
+      "other": "bar"
+    }
+    "###);
 }
 
 #[actix_rt::test]

meilisearch/tests/dumps/mod.rs

@@ -2,10 +2,10 @@ mod data;
 
 use meili_snap::{json_string, snapshot};
 use meilisearch::Opt;
-use serde_json::json;
 
 use self::data::GetDump;
 use crate::common::{default_settings, GetAllDocumentsOptions, Server};
+use crate::json;
 
 // all the following test are ignored on windows. See #2364
 #[actix_rt::test]

meilisearch/tests/features/mod.rs

@@ -1,6 +1,8 @@
-use serde_json::json;
+use meilisearch::Opt;
+use tempfile::TempDir;
 
-use crate::common::Server;
+use crate::common::{default_settings, Server};
+use crate::json;
 
 /// Feature name to test against.
 /// This will have to be changed by a different one when that feature is stabilized.
@@ -17,7 +19,9 @@ async fn experimental_features() {
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
       "scoreDetails": false,
-      "vectorStore": false
+      "vectorStore": false,
+      "metrics": false,
+      "exportPuffinReports": false
     }
     "###);
 
@@ -27,7 +31,9 @@ async fn experimental_features() {
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
       "scoreDetails": false,
-      "vectorStore": true
+      "vectorStore": true,
+      "metrics": false,
+      "exportPuffinReports": false
     }
     "###);
 
@@ -37,7 +43,9 @@ async fn experimental_features() {
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
       "scoreDetails": false,
-      "vectorStore": true
+      "vectorStore": true,
+      "metrics": false,
+      "exportPuffinReports": false
     }
     "###);
 
@@ -48,7 +56,9 @@ async fn experimental_features() {
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
       "scoreDetails": false,
-      "vectorStore": true
+      "vectorStore": true,
+      "metrics": false,
+      "exportPuffinReports": false
     }
     "###);
 
@@ -59,11 +69,73 @@ async fn experimental_features() {
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
       "scoreDetails": false,
-      "vectorStore": true
+      "vectorStore": true,
+      "metrics": false,
+      "exportPuffinReports": false
     }
     "###);
 }
 
+#[actix_rt::test]
+async fn experimental_feature_metrics() {
+    // instance flag for metrics enables metrics at startup
+    let dir = TempDir::new().unwrap();
+    let enable_metrics = Opt { experimental_enable_metrics: true, ..default_settings(dir.path()) };
+    let server = Server::new_with_options(enable_metrics).await.unwrap();
+
+    let (response, code) = server.get_features().await;
+
+    meili_snap::snapshot!(code, @"200 OK");
+    meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
+    {
+      "scoreDetails": false,
+      "vectorStore": false,
+      "metrics": true,
+      "exportPuffinReports": false
+    }
+    "###);
+
+    let (response, code) = server.get_metrics().await;
+    meili_snap::snapshot!(code, @"200 OK");
+
+    // metrics are not returned in json format
+    // so the test server will return null
+    meili_snap::snapshot!(response, @"null");
+
+    // disabling metrics results in invalid request
+    let (response, code) = server.set_features(json!({"metrics": false})).await;
+    meili_snap::snapshot!(code, @"200 OK");
+    meili_snap::snapshot!(response["metrics"], @"false");
+
+    let (response, code) = server.get_metrics().await;
+    meili_snap::snapshot!(code, @"400 Bad Request");
+    meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
+    {
+      "message": "Getting metrics requires enabling the `metrics` experimental feature. See https://github.com/meilisearch/product/discussions/625",
+      "code": "feature_not_enabled",
+      "type": "invalid_request",
+      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
+    }
+    "###);
+
+    // enabling metrics via HTTP results in valid request
+    let (response, code) = server.set_features(json!({"metrics": true})).await;
+    meili_snap::snapshot!(code, @"200 OK");
+    meili_snap::snapshot!(response["metrics"], @"true");
+
+    let (response, code) = server.get_metrics().await;
+    meili_snap::snapshot!(code, @"200 OK");
+    meili_snap::snapshot!(response, @"null");
+
+    // startup without flag respects persisted metrics value
+    let disable_metrics =
+        Opt { experimental_enable_metrics: false, ..default_settings(dir.path()) };
+    let server_no_flag = Server::new_with_options(disable_metrics).await.unwrap();
+    let (response, code) = server_no_flag.get_metrics().await;
+    meili_snap::snapshot!(code, @"200 OK");
+    meili_snap::snapshot!(response, @"null");
+}
+
 #[actix_rt::test]
 async fn errors() {
     let server = Server::new().await;
@@ -74,7 +146,7 @@ async fn errors() {
     meili_snap::snapshot!(code, @"400 Bad Request");
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
-      "message": "Unknown field `NotAFeature`: expected one of `scoreDetails`, `vectorStore`",
+      "message": "Unknown field `NotAFeature`: expected one of `scoreDetails`, `vectorStore`, `metrics`, `exportPuffinReports`",
       "code": "bad_request",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#bad_request"

meilisearch/tests/index/create_index.rs

@@ -2,10 +2,10 @@ use actix_web::http::header::ContentType;
 use actix_web::test;
 use http::header::ACCEPT_ENCODING;
 use meili_snap::{json_string, snapshot};
-use serde_json::{json, Value};
 
 use crate::common::encoder::Encoder;
-use crate::common::Server;
+use crate::common::{Server, Value};
+use crate::json;
 
 #[actix_rt::test]
 async fn create_index_no_primary_key() {
@@ -21,7 +21,7 @@ async fn create_index_no_primary_key() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], Value::Null);
+    assert_eq!(response["details"]["primaryKey"], json!(null));
 }
 
 #[actix_rt::test]
@@ -38,7 +38,7 @@ async fn create_index_with_gzip_encoded_request() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], Value::Null);
+    assert_eq!(response["details"]["primaryKey"], json!(null));
 }
 
 #[actix_rt::test]
@@ -86,7 +86,7 @@ async fn create_index_with_zlib_encoded_request() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], Value::Null);
+    assert_eq!(response["details"]["primaryKey"], json!(null));
 }
 
 #[actix_rt::test]
@@ -103,7 +103,7 @@ async fn create_index_with_brotli_encoded_request() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], Value::Null);
+    assert_eq!(response["details"]["primaryKey"], json!(null));
 }
 
 #[actix_rt::test]
@@ -136,7 +136,7 @@ async fn create_index_with_invalid_primary_key() {
 
     let (response, code) = index.get().await;
     assert_eq!(code, 200);
-    assert_eq!(response["primaryKey"], Value::Null);
+    assert_eq!(response["primaryKey"], json!(null));
 }
 
 #[actix_rt::test]

meilisearch/tests/index/delete_index.rs

@@ -1,6 +1,5 @@
-use serde_json::json;
-
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn create_and_delete_index() {

meilisearch/tests/index/errors.rs

@@ -1,7 +1,7 @@
 use meili_snap::*;
-use serde_json::json;
 
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn get_indexes_bad_offset() {

meilisearch/tests/index/stats.rs

@@ -1,6 +1,5 @@
-use serde_json::json;
-
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn stats() {

meilisearch/tests/index/update_index.rs

@@ -1,9 +1,9 @@
-use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::encoder::Encoder;
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn update_primary_key() {

meilisearch/tests/search/distinct.rs

@@ -0,0 +1,63 @@
+use meili_snap::snapshot;
+use once_cell::sync::Lazy;
+
+use crate::common::{Server, Value};
+use crate::json;
+
+pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
+    json!([
+        {"productId": 1, "shopId": 1},
+        {"productId": 2, "shopId": 1},
+        {"productId": 3, "shopId": 2},
+        {"productId": 4, "shopId": 2},
+        {"productId": 5, "shopId": 3},
+        {"productId": 6, "shopId": 3},
+        {"productId": 7, "shopId": 4},
+        {"productId": 8, "shopId": 4},
+        {"productId": 9, "shopId": 5},
+        {"productId": 10, "shopId": 5}
+    ])
+});
+
+pub(self) static DOCUMENT_PRIMARY_KEY: &str = "productId";
+pub(self) static DOCUMENT_DISTINCT_KEY: &str = "shopId";
+
+/// testing: https://github.com/meilisearch/meilisearch/issues/4078
+#[actix_rt::test]
+async fn distinct_search_with_offset_no_ranking() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, Some(DOCUMENT_PRIMARY_KEY)).await;
+    index.update_distinct_attribute(json!(DOCUMENT_DISTINCT_KEY)).await;
+    index.wait_task(1).await;
+
+    fn get_hits(Value(response): Value) -> Vec<i64> {
+        let hits_array = response["hits"].as_array().unwrap();
+        hits_array.iter().map(|h| h[DOCUMENT_DISTINCT_KEY].as_i64().unwrap()).collect::<Vec<_>>()
+    }
+
+    let (response, code) = index.search_post(json!({"limit": 2, "offset": 0})).await;
+    let hits = get_hits(response);
+    snapshot!(code, @"200 OK");
+    snapshot!(hits.len(), @"2");
+    snapshot!(format!("{:?}", hits), @"[1, 2]");
+
+    let (response, code) = index.search_post(json!({"limit": 2, "offset": 2})).await;
+    let hits = get_hits(response);
+    snapshot!(code, @"200 OK");
+    snapshot!(hits.len(), @"2");
+    snapshot!(format!("{:?}", hits), @"[3, 4]");
+
+    let (response, code) = index.search_post(json!({"limit": 10, "offset": 4})).await;
+    let hits = get_hits(response);
+    snapshot!(code, @"200 OK");
+    snapshot!(hits.len(), @"1");
+    snapshot!(format!("{:?}", hits), @"[5]");
+
+    let (response, code) = index.search_post(json!({"limit": 10, "offset": 5})).await;
+    let hits = get_hits(response);
+    snapshot!(code, @"200 OK");
+    snapshot!(hits.len(), @"0");
+}

meilisearch/tests/search/errors.rs

@@ -1,8 +1,8 @@
 use meili_snap::*;
-use serde_json::json;
 
 use super::DOCUMENTS;
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn search_unexisting_index() {

meilisearch/tests/search/facet_search.rs

@@ -1,8 +1,8 @@
 use meili_snap::snapshot;
 use once_cell::sync::Lazy;
-use serde_json::{json, Value};
 
-use crate::common::Server;
+use crate::common::{Server, Value};
+use crate::json;
 
 pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([

meilisearch/tests/search/formatted.rs

@@ -1,8 +1,8 @@
 use insta::{allow_duplicates, assert_json_snapshot};
-use serde_json::json;
 
 use super::*;
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn formatted_contain_wildcard() {

meilisearch/tests/search/geo.rs

@@ -1,8 +1,8 @@
 use meili_snap::{json_string, snapshot};
 use once_cell::sync::Lazy;
-use serde_json::{json, Value};
 
-use crate::common::Server;
+use crate::common::{Server, Value};
+use crate::json;
 
 pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([

meilisearch/tests/search/mod.rs

@@ -1,6 +1,7 @@
 // This modules contains all the test concerning search. Each particular feature of the search
 // should be tested in its own module to isolate tests and keep the tests readable.
 
+mod distinct;
 mod errors;
 mod facet_search;
 mod formatted;
@@ -10,9 +11,9 @@ mod pagination;
 mod restrict_searchable;
 
 use once_cell::sync::Lazy;
-use serde_json::{json, Value};
 
-use crate::common::Server;
+use crate::common::{Server, Value};
+use crate::json;
 
 pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([
@@ -816,7 +817,7 @@ async fn experimental_feature_score_details() {
                       },
                       "proximity": {
                         "order": 2,
-                        "score": 0.875
+                        "score": 0.75
                       },
                       "attribute": {
                         "order": 3,

meilisearch/tests/search/multi.rs

@@ -1,8 +1,8 @@
 use meili_snap::{json_string, snapshot};
-use serde_json::json;
 
 use super::{DOCUMENTS, NESTED_DOCUMENTS};
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn search_empty_list() {

meilisearch/tests/search/pagination.rs

@@ -1,6 +1,5 @@
-use serde_json::json;
-
 use crate::common::Server;
+use crate::json;
 use crate::search::DOCUMENTS;
 
 #[actix_rt::test]

meilisearch/tests/search/restrict_searchable.rs

@@ -1,9 +1,9 @@
 use meili_snap::{json_string, snapshot};
 use once_cell::sync::Lazy;
-use serde_json::{json, Value};
 
 use crate::common::index::Index;
-use crate::common::Server;
+use crate::common::{Server, Value};
+use crate::json;
 
 async fn index_with_documents<'a>(server: &'a Server, documents: &Value) -> Index<'a> {
     let index = server.index("test");

meilisearch/tests/settings/distinct.rs

@@ -1,6 +1,5 @@
-use serde_json::json;
-
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn set_and_reset_distinct_attribute() {

meilisearch/tests/settings/errors.rs

@@ -1,7 +1,7 @@
 use meili_snap::*;
-use serde_json::json;
 
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn settings_bad_displayed_attributes() {

meilisearch/tests/settings/get_settings.rs

@@ -1,16 +1,16 @@
 use std::collections::HashMap;
 
 use once_cell::sync::Lazy;
-use serde_json::{json, Value};
 
-use crate::common::Server;
+use crate::common::{Server, Value};
+use crate::json;
 
 static DEFAULT_SETTINGS_VALUES: Lazy<HashMap<&'static str, Value>> = Lazy::new(|| {
     let mut map = HashMap::new();
     map.insert("displayed_attributes", json!(["*"]));
     map.insert("searchable_attributes", json!(["*"]));
     map.insert("filterable_attributes", json!([]));
-    map.insert("distinct_attribute", json!(Value::Null));
+    map.insert("distinct_attribute", json!(null));
     map.insert(
         "ranking_rules",
         json!(["words", "typo", "proximity", "attribute", "sort", "exactness"]),
@@ -229,7 +229,7 @@ macro_rules! test_setting_routes {
                         .chars()
                         .map(|c| if c == '_' { '-' } else { c })
                         .collect::<String>());
-                    let (response, code) = server.service.$write_method(url, serde_json::Value::Null).await;
+                    let (response, code) = server.service.$write_method(url, serde_json::Value::Null.into()).await;
                     assert_eq!(code, 202, "{}", response);
                     server.index("").wait_task(0).await;
                     let (response, code) = server.index("test").get().await;

meilisearch/tests/settings/tokenizer_customization.rs

@@ -1,7 +1,7 @@
 use meili_snap::{json_string, snapshot};
-use serde_json::json;
 
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn set_and_reset() {

meilisearch/tests/snapshot/mod.rs

@@ -1,11 +1,13 @@
 use std::time::Duration;
 
 use actix_rt::time::sleep;
+use meili_snap::{json_string, snapshot};
 use meilisearch::option::ScheduleSnapshot;
 use meilisearch::Opt;
 
 use crate::common::server::default_settings;
 use crate::common::{GetAllDocumentsOptions, Server};
+use crate::json;
 
 macro_rules! verify_snapshot {
     (
@@ -44,7 +46,7 @@ async fn perform_snapshot() {
 
     let index = server.index("test");
     index
-        .update_settings(serde_json::json! ({
+        .update_settings(json! ({
         "searchableAttributes": [],
         }))
         .await;
@@ -90,3 +92,95 @@ async fn perform_snapshot() {
         server.index("test1").settings(),
     );
 }
+
+#[actix_rt::test]
+async fn perform_on_demand_snapshot() {
+    let temp = tempfile::tempdir().unwrap();
+    let snapshot_dir = tempfile::tempdir().unwrap();
+
+    let options =
+        Opt { snapshot_dir: snapshot_dir.path().to_owned(), ..default_settings(temp.path()) };
+
+    let server = Server::new_with_options(options).await.unwrap();
+
+    let index = server.index("catto");
+    index
+        .update_settings(json! ({
+        "searchableAttributes": [],
+        }))
+        .await;
+
+    index.load_test_set().await;
+
+    server.index("doggo").create(Some("bone")).await;
+    index.wait_task(2).await;
+
+    server.index("doggo").create(Some("bone")).await;
+    index.wait_task(2).await;
+
+    let (task, code) = server.create_snapshot().await;
+    snapshot!(code, @"202 Accepted");
+    snapshot!(json_string!(task, { ".enqueuedAt" => "[date]" }), @r###"
+    {
+      "taskUid": 4,
+      "indexUid": null,
+      "status": "enqueued",
+      "type": "snapshotCreation",
+      "enqueuedAt": "[date]"
+    }
+    "###);
+    let task = index.wait_task(task.uid()).await;
+    snapshot!(json_string!(task, { ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" }), @r###"
+    {
+      "uid": 4,
+      "indexUid": null,
+      "status": "succeeded",
+      "type": "snapshotCreation",
+      "canceledBy": null,
+      "error": null,
+      "duration": "[duration]",
+      "enqueuedAt": "[date]",
+      "startedAt": "[date]",
+      "finishedAt": "[date]"
+    }
+    "###);
+
+    let temp = tempfile::tempdir().unwrap();
+
+    let snapshots: Vec<String> = std::fs::read_dir(&snapshot_dir)
+        .unwrap()
+        .map(|entry| entry.unwrap().path().file_name().unwrap().to_str().unwrap().to_string())
+        .collect();
+    meili_snap::snapshot!(format!("{snapshots:?}"), @r###"["db.snapshot"]"###);
+
+    let snapshot_path = snapshot_dir.path().to_owned().join("db.snapshot");
+    #[cfg_attr(windows, allow(unused))]
+    let snapshot_meta = std::fs::metadata(&snapshot_path).unwrap();
+
+    #[cfg(unix)]
+    {
+        use std::os::unix::fs::PermissionsExt;
+        let mode = snapshot_meta.permissions().mode();
+        //                                                    rwxrwxrwx
+        meili_snap::snapshot!(format!("{:b}", mode), @"1000000100100100");
+    }
+
+    let options = Opt { import_snapshot: Some(snapshot_path), ..default_settings(temp.path()) };
+
+    let snapshot_server = Server::new_with_options(options).await.unwrap();
+
+    verify_snapshot!(server, snapshot_server, |server| =>
+        server.list_indexes(None, None),
+        // for some reason the db sizes differ. this may be due to the compaction options we have
+        // set when performing the snapshot
+        //server.stats(),
+
+        // The original instance contains the snapshotCreation task, while the snapshotted-instance does not. For this reason we need to compare the task queue **after** the task 4
+        server.tasks_filter("?from=2"),
+
+        server.index("catto").get_all_documents(GetAllDocumentsOptions::default()),
+        server.index("catto").settings(),
+        server.index("doggo").get_all_documents(GetAllDocumentsOptions::default()),
+        server.index("doggo").settings(),
+    );
+}

meilisearch/tests/stats/mod.rs

@@ -1,8 +1,8 @@
-use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn get_settings_unexisting_index() {

meilisearch/tests/swap_indexes/errors.rs

@@ -1,7 +1,7 @@
 use meili_snap::*;
-use serde_json::json;
 
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn swap_indexes_bad_format() {

meilisearch/tests/swap_indexes/mod.rs

@@ -1,9 +1,9 @@
 mod errors;
 
 use meili_snap::{json_string, snapshot};
-use serde_json::json;
 
 use crate::common::{GetAllDocumentsOptions, Server};
+use crate::json;
 
 #[actix_rt::test]
 async fn swap_indexes() {

meilisearch/tests/tasks/mod.rs

@@ -1,11 +1,11 @@
 mod errors;
 
 use meili_snap::insta::assert_json_snapshot;
-use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::Server;
+use crate::json;
 
 #[actix_rt::test]
 async fn error_get_unexisting_task_status() {
@@ -33,7 +33,7 @@ async fn get_task_status() {
     index.create(None).await;
     index
         .add_documents(
-            serde_json::json!([{
+            json!([{
                 "id": 1,
                 "content": "foobar",
             }]),

milli/Cargo.toml

@@ -26,8 +26,8 @@ flatten-serde-json = { path = "../flatten-serde-json" }
 fst = "0.4.7"
 fxhash = "0.2.1"
 geoutils = "0.5.1"
-grenad = { version = "0.4.4", default-features = false, features = [
-    "tempfile",
+grenad = { version = "0.4.5", default-features = false, features = [
+    "rayon", "tempfile"
 ] }
 heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.7", default-features = false, features = [
     "lmdb", "read-txn-no-tls"
@@ -79,6 +79,7 @@ big_s = "1.0.2"
 insta = "1.29.0"
 maplit = "1.0.2"
 md5 = "0.7.0"
+meili-snap = { path = "../meili-snap" }
 rand = { version = "0.8.5", features = ["small_rng"] }
 
 [features]

milli/src/documents/enriched.rs

@@ -1,4 +1,5 @@
 use std::fs::File;
+use std::io::BufReader;
 use std::{io, str};
 
 use obkv::KvReader;
@@ -19,14 +20,14 @@ use crate::FieldId;
 pub struct EnrichedDocumentsBatchReader<R> {
     documents: DocumentsBatchReader<R>,
     primary_key: String,
-    external_ids: grenad::ReaderCursor<File>,
+    external_ids: grenad::ReaderCursor<BufReader<File>>,
 }
 
 impl<R: io::Read + io::Seek> EnrichedDocumentsBatchReader<R> {
     pub fn new(
         documents: DocumentsBatchReader<R>,
         primary_key: String,
-        external_ids: grenad::Reader<File>,
+        external_ids: grenad::Reader<BufReader<File>>,
     ) -> Result<Self, Error> {
         if documents.documents_count() as u64 == external_ids.len() {
             Ok(EnrichedDocumentsBatchReader {
@@ -75,7 +76,7 @@ pub struct EnrichedDocument<'a> {
 pub struct EnrichedDocumentsBatchCursor<R> {
     documents: DocumentsBatchCursor<R>,
     primary_key: String,
-    external_ids: grenad::ReaderCursor<File>,
+    external_ids: grenad::ReaderCursor<BufReader<File>>,
 }
 
 impl<R> EnrichedDocumentsBatchCursor<R> {

milli/src/error.rs

@@ -89,8 +89,6 @@ pub enum FieldIdMapMissingEntry {
 
 #[derive(Error, Debug)]
 pub enum UserError {
-    #[error("A soft deleted internal document id have been used: `{document_id}`.")]
-    AccessingSoftDeletedDocument { document_id: DocumentId },
     #[error("A document cannot contain more than 65,535 fields.")]
     AttributeLimitReached,
     #[error(transparent)]

milli/src/external_documents_ids.rs

@@ -1,159 +1,146 @@
-use std::borrow::Cow;
 use std::collections::HashMap;
-use std::convert::TryInto;
-use std::{fmt, str};
 
-use fst::map::IndexedValue;
-use fst::{IntoStreamer, Streamer};
+use heed::types::{OwnedType, Str};
+use heed::{Database, RoIter, RoTxn, RwTxn};
 use roaring::RoaringBitmap;
 
-const DELETED_ID: u64 = u64::MAX;
+use crate::{DocumentId, BEU32};
 
-pub struct ExternalDocumentsIds<'a> {
-    pub(crate) hard: fst::Map<Cow<'a, [u8]>>,
-    pub(crate) soft: fst::Map<Cow<'a, [u8]>>,
-    soft_deleted_docids: RoaringBitmap,
+pub enum DocumentOperationKind {
+    Create,
+    Delete,
 }
 
-impl<'a> ExternalDocumentsIds<'a> {
-    pub fn new(
-        hard: fst::Map<Cow<'a, [u8]>>,
-        soft: fst::Map<Cow<'a, [u8]>>,
-        soft_deleted_docids: RoaringBitmap,
-    ) -> ExternalDocumentsIds<'a> {
-        ExternalDocumentsIds { hard, soft, soft_deleted_docids }
-    }
+pub struct DocumentOperation {
+    pub external_id: String,
+    pub internal_id: DocumentId,
+    pub kind: DocumentOperationKind,
+}
 
-    pub fn into_static(self) -> ExternalDocumentsIds<'static> {
-        ExternalDocumentsIds {
-            hard: self.hard.map_data(|c| Cow::Owned(c.into_owned())).unwrap(),
-            soft: self.soft.map_data(|c| Cow::Owned(c.into_owned())).unwrap(),
-            soft_deleted_docids: self.soft_deleted_docids,
-        }
+pub struct ExternalDocumentsIds(Database<Str, OwnedType<BEU32>>);
+
+impl ExternalDocumentsIds {
+    pub fn new(db: Database<Str, OwnedType<BEU32>>) -> ExternalDocumentsIds {
+        ExternalDocumentsIds(db)
     }
 
     /// Returns `true` if hard and soft external documents lists are empty.
-    pub fn is_empty(&self) -> bool {
-        self.hard.is_empty() && self.soft.is_empty()
+    pub fn is_empty(&self, rtxn: &RoTxn) -> heed::Result<bool> {
+        self.0.is_empty(rtxn).map_err(Into::into)
     }
 
-    pub fn get<A: AsRef<[u8]>>(&self, external_id: A) -> Option<u32> {
-        let external_id = external_id.as_ref();
-        match self.soft.get(external_id).or_else(|| self.hard.get(external_id)) {
-            Some(id) if id != DELETED_ID && !self.soft_deleted_docids.contains(id as u32) => {
-                Some(id.try_into().unwrap())
-            }
-            _otherwise => None,
-        }
-    }
-
-    /// Rebuild the internal FSTs in the ExternalDocumentsIds structure such that they
-    /// don't contain any soft deleted document id.
-    pub fn delete_soft_deleted_documents_ids_from_fsts(&mut self) -> fst::Result<()> {
-        let mut new_hard_builder = fst::MapBuilder::memory();
-
-        let union_op = self.hard.op().add(&self.soft).r#union();
-        let mut iter = union_op.into_stream();
-        while let Some((external_id, docids)) = iter.next() {
-            // prefer selecting the ids from soft, always
-            let id = indexed_last_value(docids).unwrap();
-            if id != DELETED_ID && !self.soft_deleted_docids.contains(id as u32) {
-                new_hard_builder.insert(external_id, id)?;
-            }
-        }
-        drop(iter);
-
-        // Delete soft map completely
-        self.soft = fst::Map::default().map_data(Cow::Owned)?;
-        // We save the new map as the new hard map.
-        self.hard = new_hard_builder.into_map().map_data(Cow::Owned)?;
-
-        Ok(())
-    }
-
-    pub fn insert_ids<A: AsRef<[u8]>>(&mut self, other: &fst::Map<A>) -> fst::Result<()> {
-        let union_op = self.soft.op().add(other).r#union();
-
-        let mut new_soft_builder = fst::MapBuilder::memory();
-        let mut iter = union_op.into_stream();
-        while let Some((external_id, marked_docids)) = iter.next() {
-            let id = indexed_last_value(marked_docids).unwrap();
-            new_soft_builder.insert(external_id, id)?;
-        }
-
-        drop(iter);
-
-        // We save the new map as the new soft map.
-        self.soft = new_soft_builder.into_map().map_data(Cow::Owned)?;
-        self.merge_soft_into_hard()
+    pub fn get<A: AsRef<str>>(&self, rtxn: &RoTxn, external_id: A) -> heed::Result<Option<u32>> {
+        Ok(self.0.get(rtxn, external_id.as_ref())?.map(|x| x.get()))
     }
 
     /// An helper function to debug this type, returns an `HashMap` of both,
     /// soft and hard fst maps, combined.
-    pub fn to_hash_map(&self) -> HashMap<String, u32> {
-        let mut map = HashMap::new();
-
-        let union_op = self.hard.op().add(&self.soft).r#union();
-        let mut iter = union_op.into_stream();
-        while let Some((external_id, marked_docids)) = iter.next() {
-            let id = indexed_last_value(marked_docids).unwrap();
-            if id != DELETED_ID {
-                let external_id = str::from_utf8(external_id).unwrap();
-                map.insert(external_id.to_owned(), id.try_into().unwrap());
-            }
+    pub fn to_hash_map(&self, rtxn: &RoTxn) -> heed::Result<HashMap<String, u32>> {
+        let mut map = HashMap::default();
+        for result in self.0.iter(rtxn)? {
+            let (external, internal) = result?;
+            map.insert(external.to_owned(), internal.get());
         }
-
-        map
+        Ok(map)
     }
 
-    /// Return an fst of the combined hard and soft deleted ID.
-    pub fn to_fst<'b>(&'b self) -> fst::Result<Cow<'b, fst::Map<Cow<'a, [u8]>>>> {
-        if self.soft.is_empty() {
-            return Ok(Cow::Borrowed(&self.hard));
-        }
-        let union_op = self.hard.op().add(&self.soft).r#union();
-
-        let mut iter = union_op.into_stream();
-        let mut new_hard_builder = fst::MapBuilder::memory();
-        while let Some((external_id, marked_docids)) = iter.next() {
-            let value = indexed_last_value(marked_docids).unwrap();
-            if value != DELETED_ID {
-                new_hard_builder.insert(external_id, value)?;
-            }
-        }
-
-        drop(iter);
-
-        Ok(Cow::Owned(new_hard_builder.into_map().map_data(Cow::Owned)?))
+    /// Looks for the internal ids in the passed bitmap, and returns an iterator over the mapping between
+    /// these internal ids and their external id.
+    ///
+    /// The returned iterator has `Result<(String, DocumentId), RoaringBitmap>` as `Item`,
+    /// where the returned values can be:
+    /// - `Ok((external_id, internal_id))`: if a mapping was found
+    /// - `Err(remaining_ids)`: if the external ids for some of the requested internal ids weren't found.
+    ///   In that case the returned bitmap contains the internal ids whose external ids were not found after traversing
+    ///   the entire fst.
+    pub fn find_external_id_of<'t>(
+        &self,
+        rtxn: &'t RoTxn,
+        internal_ids: RoaringBitmap,
+    ) -> heed::Result<ExternalToInternalOwnedIterator<'t>> {
+        self.0.iter(rtxn).map(|iter| ExternalToInternalOwnedIterator { iter, internal_ids })
     }
 
-    fn merge_soft_into_hard(&mut self) -> fst::Result<()> {
-        if self.soft.len() >= self.hard.len() / 2 {
-            self.hard = self.to_fst()?.into_owned();
-            self.soft = fst::Map::default().map_data(Cow::Owned)?;
+    /// Applies the list of operations passed as argument, modifying the current external to internal id mapping.
+    ///
+    /// If the list contains multiple operations on the same external id, then the result is unspecified.
+    ///
+    /// # Panics
+    ///
+    /// - If attempting to delete a document that doesn't exist
+    /// - If attempting to create a document that already exists
+    pub fn apply(&self, wtxn: &mut RwTxn, operations: Vec<DocumentOperation>) -> heed::Result<()> {
+        for DocumentOperation { external_id, internal_id, kind } in operations {
+            match kind {
+                DocumentOperationKind::Create => {
+                    self.0.put(wtxn, &external_id, &BEU32::new(internal_id))?;
+                }
+                DocumentOperationKind::Delete => {
+                    if !self.0.delete(wtxn, &external_id)? {
+                        panic!("Attempting to delete a non-existing document")
+                    }
+                }
+            }
         }
 
         Ok(())
     }
-}
 
-impl fmt::Debug for ExternalDocumentsIds<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        f.debug_tuple("ExternalDocumentsIds").field(&self.to_hash_map()).finish()
+    /// Returns an iterator over all the external ids.
+    pub fn iter<'t>(&self, rtxn: &'t RoTxn) -> heed::Result<RoIter<'t, Str, OwnedType<BEU32>>> {
+        self.0.iter(rtxn)
     }
 }
 
-impl Default for ExternalDocumentsIds<'static> {
-    fn default() -> Self {
-        ExternalDocumentsIds {
-            hard: fst::Map::default().map_data(Cow::Owned).unwrap(),
-            soft: fst::Map::default().map_data(Cow::Owned).unwrap(),
-            soft_deleted_docids: RoaringBitmap::new(),
+/// An iterator over mappings between requested internal ids and external ids.
+///
+/// See [`ExternalDocumentsIds::find_external_id_of`] for details.
+pub struct ExternalToInternalOwnedIterator<'t> {
+    iter: RoIter<'t, Str, OwnedType<BEU32>>,
+    internal_ids: RoaringBitmap,
+}
+
+impl<'t> Iterator for ExternalToInternalOwnedIterator<'t> {
+    /// A result indicating if a mapping was found, or if the stream was exhausted without finding all internal ids.
+    type Item = Result<(&'t str, DocumentId), RoaringBitmap>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        // if all requested ids were found, we won't find any other, so short-circuit
+        if self.internal_ids.is_empty() {
+            return None;
+        }
+        loop {
+            let (external, internal) = match self.iter.next() {
+                Some(Ok((external, internal))) => (external, internal),
+                // TODO manage this better, remove panic
+                Some(Err(e)) => panic!("{}", e),
+                _ => {
+                    // we exhausted the stream but we still have some internal ids to find
+                    let remaining_ids = std::mem::take(&mut self.internal_ids);
+                    return Some(Err(remaining_ids));
+                    // note: next calls to `next` will return `None` since we replaced the internal_ids
+                    // with the default empty bitmap
+                }
+            };
+            let internal = internal.get();
+            let was_contained = self.internal_ids.remove(internal);
+            if was_contained {
+                return Some(Ok((external, internal)));
+            }
         }
     }
 }
 
-/// Returns the value of the `IndexedValue` with the highest _index_.
-fn indexed_last_value(indexed_values: &[IndexedValue]) -> Option<u64> {
-    indexed_values.iter().copied().max_by_key(|iv| iv.index).map(|iv| iv.value)
+impl<'t> ExternalToInternalOwnedIterator<'t> {
+    /// Returns the bitmap of internal ids whose external id are yet to be found
+    pub fn remaining_internal_ids(&self) -> &RoaringBitmap {
+        &self.internal_ids
+    }
+
+    /// Consumes this iterator and returns an iterator over only the external ids, ignoring the internal ids.
+    ///
+    /// Use this when you don't need the mapping between the external and the internal ids.
+    pub fn only_external_ids(self) -> impl Iterator<Item = Result<String, RoaringBitmap>> + 't {
+        self.map(|res| res.map(|(external, _internal)| external.to_owned()))
+    }
 }

milli/src/heed_codec/roaring_bitmap/cbo_roaring_bitmap_codec.rs

@@ -6,6 +6,7 @@ use byteorder::{NativeEndian, ReadBytesExt, WriteBytesExt};
 use roaring::RoaringBitmap;
 
 use crate::heed_codec::BytesDecodeOwned;
+use crate::update::del_add::{DelAdd, KvReaderDelAdd};
 
 /// This is the limit where using a byteorder became less size efficient
 /// than using a direct roaring encoding, it is also the point where we are able
@@ -60,12 +61,16 @@ impl CboRoaringBitmapCodec {
     /// if the merged values length is under the threshold, values are directly
     /// serialized in the buffer else a RoaringBitmap is created from the
     /// values and is serialized in the buffer.
-    pub fn merge_into(slices: &[Cow<[u8]>], buffer: &mut Vec<u8>) -> io::Result<()> {
+    pub fn merge_into<I, A>(slices: I, buffer: &mut Vec<u8>) -> io::Result<()>
+    where
+        I: IntoIterator<Item = A>,
+        A: AsRef<[u8]>,
+    {
         let mut roaring = RoaringBitmap::new();
         let mut vec = Vec::new();
 
         for bytes in slices {
-            if bytes.len() <= THRESHOLD * size_of::<u32>() {
+            if bytes.as_ref().len() <= THRESHOLD * size_of::<u32>() {
                 let mut reader = bytes.as_ref();
                 while let Ok(integer) = reader.read_u32::<NativeEndian>() {
                     vec.push(integer);
@@ -85,7 +90,7 @@ impl CboRoaringBitmapCodec {
                 }
             } else {
                 // We can unwrap safely because the vector is sorted upper.
-                let roaring = RoaringBitmap::from_sorted_iter(vec.into_iter()).unwrap();
+                let roaring = RoaringBitmap::from_sorted_iter(vec).unwrap();
                 roaring.serialize_into(buffer)?;
             }
         } else {
@@ -95,6 +100,33 @@ impl CboRoaringBitmapCodec {
 
         Ok(())
     }
+
+    /// Merges a DelAdd delta into a CboRoaringBitmap.
+    pub fn merge_deladd_into<'a>(
+        deladd: KvReaderDelAdd<'_>,
+        previous: &[u8],
+        buffer: &'a mut Vec<u8>,
+    ) -> io::Result<Option<&'a [u8]>> {
+        // Deserialize the bitmap that is already there
+        let mut previous = Self::deserialize_from(previous)?;
+
+        // Remove integers we no more want in the previous bitmap
+        if let Some(value) = deladd.get(DelAdd::Deletion) {
+            previous -= Self::deserialize_from(value)?;
+        }
+
+        // Insert the new integers we want in the previous bitmap
+        if let Some(value) = deladd.get(DelAdd::Addition) {
+            previous |= Self::deserialize_from(value)?;
+        }
+
+        if previous.is_empty() {
+            return Ok(None);
+        }
+
+        Self::serialize_into(&previous, buffer);
+        Ok(Some(&buffer[..]))
+    }
 }
 
 impl heed::BytesDecode<'_> for CboRoaringBitmapCodec {

milli/src/proximity.rs

@@ -2,7 +2,7 @@ use std::cmp;
 
 use crate::{relative_from_absolute_position, Position};
 
-pub const MAX_DISTANCE: u32 = 8;
+pub const MAX_DISTANCE: u32 = 4;
 
 pub fn index_proximity(lhs: u32, rhs: u32) -> u32 {
     if lhs <= rhs {

milli/src/search/facet/facet_sort_ascending.rs

@@ -13,7 +13,7 @@ use crate::heed_codec::ByteSliceRefCodec;
 /// The documents returned by the iterator are grouped by the facet values that
 /// determined their rank. For example, given the documents:
 ///
-/// ```ignore
+/// ```text
 /// 0: { "colour": ["blue", "green"] }
 /// 1: { "colour": ["blue", "red"] }
 /// 2: { "colour": ["orange", "red"] }
@@ -22,7 +22,7 @@ use crate::heed_codec::ByteSliceRefCodec;
 /// ```
 /// Then calling the function on the candidates `[0, 2, 3, 4]` will return an iterator
 /// over the following elements:
-/// ```ignore
+/// ```text
 /// [0, 4]  // corresponds to all the documents within the candidates that have the facet value "blue"
 /// [3]     // same for "green"
 /// [2]     // same for "orange"

milli/src/search/facet/filter.rs

@@ -223,12 +223,9 @@ impl<'a> Filter<'a> {
 impl<'a> Filter<'a> {
     pub fn evaluate(&self, rtxn: &heed::RoTxn, index: &Index) -> Result<RoaringBitmap> {
         // to avoid doing this for each recursive call we're going to do it ONCE ahead of time
-        let soft_deleted_documents = index.soft_deleted_documents_ids(rtxn)?;
         let filterable_fields = index.filterable_fields(rtxn)?;
 
-        // and finally we delete all the soft_deleted_documents, again, only once at the very end
         self.inner_evaluate(rtxn, index, &filterable_fields)
-            .map(|result| result - soft_deleted_documents)
     }
 
     fn evaluate_operator(

milli/src/search/new/bucket_sort.rs

@@ -46,18 +46,27 @@ pub fn bucket_sort<'ctx, Q: RankingRuleQueryTrait>(
         if let Some(distinct_fid) = distinct_fid {
             let mut excluded = RoaringBitmap::new();
             let mut results = vec![];
+            let mut skip = 0;
             for docid in universe.iter() {
-                if results.len() >= from + length {
+                if results.len() >= length {
                     break;
                 }
                 if excluded.contains(docid) {
                     continue;
                 }
+
                 distinct_single_docid(ctx.index, ctx.txn, distinct_fid, docid, &mut excluded)?;
+                skip += 1;
+                if skip <= from {
+                    continue;
+                }
+
                 results.push(docid);
             }
+
             let mut all_candidates = universe - excluded;
             all_candidates.extend(results.iter().copied());
+
             return Ok(BucketSortOutput {
                 scores: vec![Default::default(); results.len()],
                 docids: results,

milli/src/search/new/db_cache.rs

@@ -11,9 +11,7 @@ use super::interner::Interned;
 use super::Word;
 use crate::heed_codec::{BytesDecodeOwned, StrBEU16Codec};
 use crate::update::{merge_cbo_roaring_bitmaps, MergeFn};
-use crate::{
-    CboRoaringBitmapCodec, CboRoaringBitmapLenCodec, Result, RoaringBitmapCodec, SearchContext,
-};
+use crate::{CboRoaringBitmapCodec, CboRoaringBitmapLenCodec, Result, SearchContext};
 
 /// A cache storing pointers to values in the LMDB databases.
 ///
@@ -168,7 +166,7 @@ impl<'ctx> SearchContext<'ctx> {
                     merge_cbo_roaring_bitmaps,
                 )
             }
-            None => DatabaseCache::get_value::<_, _, RoaringBitmapCodec>(
+            None => DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
                 self.txn,
                 word,
                 self.word_interner.get(word).as_str(),
@@ -182,7 +180,7 @@ impl<'ctx> SearchContext<'ctx> {
         &mut self,
         word: Interned<String>,
     ) -> Result<Option<RoaringBitmap>> {
-        DatabaseCache::get_value::<_, _, RoaringBitmapCodec>(
+        DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
             self.txn,
             word,
             self.word_interner.get(word).as_str(),
@@ -230,7 +228,7 @@ impl<'ctx> SearchContext<'ctx> {
                     merge_cbo_roaring_bitmaps,
                 )
             }
-            None => DatabaseCache::get_value::<_, _, RoaringBitmapCodec>(
+            None => DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
                 self.txn,
                 prefix,
                 self.word_interner.get(prefix).as_str(),
@@ -244,7 +242,7 @@ impl<'ctx> SearchContext<'ctx> {
         &mut self,
         prefix: Interned<String>,
     ) -> Result<Option<RoaringBitmap>> {
-        DatabaseCache::get_value::<_, _, RoaringBitmapCodec>(
+        DatabaseCache::get_value::<_, _, CboRoaringBitmapCodec>(
             self.txn,
             prefix,
             self.word_interner.get(prefix).as_str(),

milli/src/search/new/matches/mod.rs

@@ -418,19 +418,11 @@ impl<'t> Matcher<'t, '_> {
         } else {
             match &self.matches {
                 Some((tokens, matches)) => {
-                    // If the text has to be cropped,
-                    // compute the best interval to crop around.
-                    let matches = match format_options.crop {
-                        Some(crop_size) if crop_size > 0 => {
-                            self.find_best_match_interval(matches, crop_size)
-                        }
-                        _ => matches,
-                    };
-
                     // If the text has to be cropped,
                     // crop around the best interval.
                     let (byte_start, byte_end) = match format_options.crop {
                         Some(crop_size) if crop_size > 0 => {
+                            let matches = self.find_best_match_interval(matches, crop_size);
                             self.crop_bounds(tokens, matches, crop_size)
                         }
                         _ => (0, self.text.len()),
@@ -450,6 +442,11 @@ impl<'t> Matcher<'t, '_> {
                         for m in matches {
                             let token = &tokens[m.token_position];
 
+                            // skip matches out of the crop window.
+                            if token.byte_start < byte_start || token.byte_end > byte_end {
+                                continue;
+                            }
+
                             if byte_index < token.byte_start {
                                 formatted.push(&self.text[byte_index..token.byte_start]);
                             }
@@ -800,6 +797,37 @@ mod tests {
         );
     }
 
+    #[test]
+    fn format_highlight_crop_phrase_query() {
+        //! testing: https://github.com/meilisearch/meilisearch/issues/3975
+        let temp_index = TempIndex::new();
+        temp_index
+            .add_documents(documents!([
+                { "id": 1, "text": "The groundbreaking invention had the power to split the world between those who embraced progress and those who resisted change!" }
+            ]))
+            .unwrap();
+        let rtxn = temp_index.read_txn().unwrap();
+
+        let format_options = FormatOptions { highlight: true, crop: Some(10) };
+        let text = "The groundbreaking invention had the power to split the world between those who embraced progress and those who resisted change!";
+
+        let builder = MatcherBuilder::new_test(&rtxn, &temp_index, "\"the world\"");
+        let mut matcher = builder.build(text);
+        // should return 10 words with a marker at the start as well the end, and the highlighted matches.
+        insta::assert_snapshot!(
+            matcher.format(format_options),
+            @"…had the power to split <em>the</em> <em>world</em> between those who…"
+        );
+
+        let builder = MatcherBuilder::new_test(&rtxn, &temp_index, "those \"and those\"");
+        let mut matcher = builder.build(text);
+        // should highlight "those" and the phrase "and those".
+        insta::assert_snapshot!(
+            matcher.format(format_options),
+            @"…world between <em>those</em> who embraced progress <em>and</em> <em>those</em> who resisted…"
+        );
+    }
+
     #[test]
     fn smaller_crop_size() {
         //! testing: https://github.com/meilisearch/specifications/pull/120#discussion_r836536295

milli/src/search/new/ranking_rule_graph/proximity/build.rs

@@ -1,6 +1,7 @@
 #![allow(clippy::too_many_arguments)]
 
 use super::ProximityCondition;
+use crate::proximity::MAX_DISTANCE;
 use crate::search::new::interner::{DedupInterner, Interned};
 use crate::search::new::query_term::LocatedQueryTermSubset;
 use crate::search::new::SearchContext;
@@ -35,7 +36,7 @@ pub fn build_edges(
     }
 
     let mut conditions = vec![];
-    for cost in right_ngram_max..(7 + right_ngram_max) {
+    for cost in right_ngram_max..(((MAX_DISTANCE as usize) - 1) + right_ngram_max) {
         conditions.push((
             cost as u32,
             conditions_interner.insert(ProximityCondition::Uninit {
@@ -47,7 +48,7 @@ pub fn build_edges(
     }
 
     conditions.push((
-        (7 + right_ngram_max) as u32,
+        ((MAX_DISTANCE - 1) + (right_ngram_max as u32)),
         conditions_interner.insert(ProximityCondition::Term { term: right_term.clone() }),
     ));
 

milli/src/search/new/tests/proximity.rs

@@ -273,7 +273,7 @@ fn test_proximity_simple() {
     s.terms_matching_strategy(TermsMatchingStrategy::All);
     s.query("the quick brown fox jumps over the lazy dog");
     let SearchResult { documents_ids, .. } = s.execute().unwrap();
-    insta::assert_snapshot!(format!("{documents_ids:?}"), @"[9, 10, 4, 7, 6, 5, 2, 3, 0, 1]");
+    insta::assert_snapshot!(format!("{documents_ids:?}"), @"[9, 10, 4, 7, 6, 2, 3, 5, 1, 0]");
     let texts = collect_field_values(&index, &txn, "text", &documents_ids);
     insta::assert_debug_snapshot!(texts, @r###"
     [
@@ -282,11 +282,11 @@ fn test_proximity_simple() {
         "\"the quickbrown fox jumps over the lazy dog\"",
         "\"the really quick brown fox jumps over the lazy dog\"",
         "\"the really quick brown fox jumps over the very lazy dog\"",
-        "\"brown quick fox jumps over the lazy dog\"",
         "\"the quick brown fox jumps over the lazy. dog\"",
         "\"dog the quick brown fox jumps over the lazy\"",
-        "\"the very quick dark brown and smart fox did jump over the terribly lazy and small dog\"",
+        "\"brown quick fox jumps over the lazy dog\"",
         "\"the. quick brown fox jumps over the lazy. dog\"",
+        "\"the very quick dark brown and smart fox did jump over the terribly lazy and small dog\"",
     ]
     "###);
 }
@@ -371,7 +371,7 @@ fn test_proximity_prefix_db() {
     s.scoring_strategy(crate::score_details::ScoringStrategy::Detailed);
     s.query("best s");
     let SearchResult { documents_ids, document_scores, .. } = s.execute().unwrap();
-    insta::assert_snapshot!(format!("{documents_ids:?}"), @"[10, 13, 9, 12, 8, 6, 7, 11, 15]");
+    insta::assert_snapshot!(format!("{documents_ids:?}"), @"[10, 13, 9, 12, 6, 7, 8, 11, 15]");
     insta::assert_snapshot!(format!("{document_scores:#?}"));
     let texts = collect_field_values(&index, &txn, "text", &documents_ids);
 
@@ -382,9 +382,9 @@ fn test_proximity_prefix_db() {
         "\"summer best\"",
         "\"this is the best meal of summer\"",
         "\"summer x best\"",
-        "\"this is the best meal of the summer\"",
         "\"this is the best meal I have ever had in such a beautiful summer day\"",
         "\"this is the best cooked meal of the summer\"",
+        "\"this is the best meal of the summer\"",
         "\"summer x y best\"",
         "\"this is the best meal I have ever had in such a beautiful winter day\"",
     ]
@@ -396,7 +396,7 @@ fn test_proximity_prefix_db() {
     s.scoring_strategy(crate::score_details::ScoringStrategy::Detailed);
     s.query("best su");
     let SearchResult { documents_ids, document_scores, .. } = s.execute().unwrap();
-    insta::assert_snapshot!(format!("{documents_ids:?}"), @"[10, 13, 9, 12, 8, 11, 7, 6, 15]");
+    insta::assert_snapshot!(format!("{documents_ids:?}"), @"[10, 13, 9, 12, 6, 7, 8, 11, 15]");
     insta::assert_snapshot!(format!("{document_scores:#?}"));
     let texts = collect_field_values(&index, &txn, "text", &documents_ids);
 
@@ -406,10 +406,10 @@ fn test_proximity_prefix_db() {
         "\"summer best\"",
         "\"this is the best meal of summer\"",
         "\"summer x best\"",
+        "\"this is the best meal I have ever had in such a beautiful summer day\"",
+        "\"this is the best cooked meal of the summer\"",
         "\"this is the best meal of the summer\"",
         "\"summer x y best\"",
-        "\"this is the best cooked meal of the summer\"",
-        "\"this is the best meal I have ever had in such a beautiful summer day\"",
         "\"this is the best meal I have ever had in such a beautiful winter day\"",
     ]
     "###);
@@ -423,20 +423,20 @@ fn test_proximity_prefix_db() {
     s.scoring_strategy(crate::score_details::ScoringStrategy::Detailed);
     s.query("best win");
     let SearchResult { documents_ids, document_scores, .. } = s.execute().unwrap();
-    insta::assert_snapshot!(format!("{documents_ids:?}"), @"[15, 16, 17, 18, 19, 20, 21, 22]");
+    insta::assert_snapshot!(format!("{documents_ids:?}"), @"[19, 22, 18, 21, 15, 16, 17, 20]");
     insta::assert_snapshot!(format!("{document_scores:#?}"));
     let texts = collect_field_values(&index, &txn, "text", &documents_ids);
 
     insta::assert_debug_snapshot!(texts, @r###"
     [
+        "\"this is the best winter meal\"",
+        "\"winter best\"",
+        "\"this is the best meal of winter\"",
+        "\"winter x best\"",
         "\"this is the best meal I have ever had in such a beautiful winter day\"",
         "\"this is the best cooked meal of the winter\"",
         "\"this is the best meal of the winter\"",
-        "\"this is the best meal of winter\"",
-        "\"this is the best winter meal\"",
         "\"winter x y best\"",
-        "\"winter x best\"",
-        "\"winter best\"",
     ]
     "###);
 
@@ -447,7 +447,7 @@ fn test_proximity_prefix_db() {
     s.scoring_strategy(crate::score_details::ScoringStrategy::Detailed);
     s.query("best wint");
     let SearchResult { documents_ids, document_scores, .. } = s.execute().unwrap();
-    insta::assert_snapshot!(format!("{documents_ids:?}"), @"[19, 22, 18, 21, 17, 20, 16, 15]");
+    insta::assert_snapshot!(format!("{documents_ids:?}"), @"[19, 22, 18, 21, 15, 16, 17, 20]");
     insta::assert_snapshot!(format!("{document_scores:#?}"));
     let texts = collect_field_values(&index, &txn, "text", &documents_ids);
 
@@ -457,10 +457,10 @@ fn test_proximity_prefix_db() {
         "\"winter best\"",
         "\"this is the best meal of winter\"",
         "\"winter x best\"",
+        "\"this is the best meal I have ever had in such a beautiful winter day\"",
+        "\"this is the best cooked meal of the winter\"",
         "\"this is the best meal of the winter\"",
         "\"winter x y best\"",
-        "\"this is the best cooked meal of the winter\"",
-        "\"this is the best meal I have ever had in such a beautiful winter day\"",
     ]
     "###);
 
@@ -471,7 +471,7 @@ fn test_proximity_prefix_db() {
     s.scoring_strategy(crate::score_details::ScoringStrategy::Detailed);
     s.query("best wi");
     let SearchResult { documents_ids, document_scores, .. } = s.execute().unwrap();
-    insta::assert_snapshot!(format!("{documents_ids:?}"), @"[19, 22, 18, 21, 17, 15, 16, 20]");
+    insta::assert_snapshot!(format!("{documents_ids:?}"), @"[19, 22, 18, 21, 15, 16, 17, 20]");
     insta::assert_snapshot!(format!("{document_scores:#?}"));
     let texts = collect_field_values(&index, &txn, "text", &documents_ids);
 
@@ -481,9 +481,9 @@ fn test_proximity_prefix_db() {
         "\"winter best\"",
         "\"this is the best meal of winter\"",
         "\"winter x best\"",
-        "\"this is the best meal of the winter\"",
         "\"this is the best meal I have ever had in such a beautiful winter day\"",
         "\"this is the best cooked meal of the winter\"",
+        "\"this is the best meal of the winter\"",
         "\"winter x y best\"",
     ]
     "###);