Fix the tests

4057: Fix stats delete by filter for v1.3.4 r=irevoire a=curquiza

Fixes https://github.com/meilisearch/meilisearch/issues/4018 for v1.3.4

Co-authored-by: Tamo <tamo@meilisearch.com>

.github/ISSUE_TEMPLATE/sprint_issue.md

@@ -7,17 +7,19 @@ assignees: ''
 
 ---
 
-Related product team resources: [PRD]() (_internal only_)
+Related product team resources: [roadmap card]() (_internal only_) and [PRD]() (_internal only_)
 Related product discussion:
 Related spec: WIP
 
 ## Motivation
 
-<!---Copy/paste the information in PRD or briefly detail the product motivation. Ask product team if any hesitation.-->
+<!---Copy/paste the information in the roadmap resources or briefly detail the product motivation. Ask product team if any hesitation.-->
 
 ## Usage
 
-<!---Link to the public part of the PRD, or to the related product discussion for experimental features-->
+<!---Write a quick description of the usage if the usage has already been defined-->
+
+Refer to the final spec to know the details and the final decisions about the usage.
 
 ## TODO
 

.github/workflows/benchmarks-manual.yml

@@ -74,4 +74,4 @@ jobs:
           echo "${{ steps.file.outputs.basename }}.json has just been pushed."
           echo 'How to compare this benchmark with another one?'
           echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scripts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"
+          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/benchmarks-pr.yml

@@ -1,97 +0,0 @@
-name: Benchmarks (PR)
-on: issue_comment
-permissions:
-  issues: write
-
-env:
-  GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-
-jobs:
-  run-benchmarks-on-comment:
-    if: startsWith(github.event.comment.body, '/benchmark')
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      - name: Check for Command
-        id: command
-        uses: xt0rted/slash-command-action@v2
-        with:
-          command: benchmark
-          reaction-type: "eyes"
-          repo-token: ${{ env.GH_TOKEN }}
-
-      - uses: xt0rted/pull-request-comment-branch@v2
-        id: comment-branch
-        with:
-          repo_token: ${{ env.GH_TOKEN }}
-
-      - uses: actions/checkout@v3
-        if: success()
-        with:
-          fetch-depth: 0 # fetch full history to be able to get main commit sha
-          ref: ${{ steps.comment-branch.outputs.head_ref }}
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(git rev-parse --abbrev-ref HEAD | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${{ steps.command.outputs.command-arguments }}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${{ steps.command.outputs.command-arguments }} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${{ steps.command.outputs.command-arguments }} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Compute the diff of the benchmarks and send a message on the GitHub PR
-      - name: Compute and send a message in the PR
-        env:
-          GITHUB_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-        run: |
-          set -x
-          export base_ref=$(git merge-base origin/main ${{ steps.comment-branch.outputs.head_ref }} | head -c8)
-          export base_filename=$(echo ${{ steps.command.outputs.command-arguments }}_main_${base_ref}.json)
-          echo 'Here are your benchmarks diff 👊' >> body.txt
-          echo '```' >> body.txt
-          ./benchmarks/scripts/compare.sh $base_filename ${{ steps.file.outputs.basename }}.json >> body.txt
-          echo '```' >> body.txt
-          gh pr comment ${{ steps.current_branch.outputs.name }} --body-file body.txt

.github/workflows/dependency-issue.yml

@@ -2,8 +2,8 @@ name: Create issue to upgrade dependencies
 
 on:
   schedule:
-    # Run the first of the month, every 6 month
-    - cron: '0 0 1 */6 *'
+    # Run the first of the month, every 3 month
+    - cron: '0 0 1 */3 *'
   workflow_dispatch:
 
 jobs:

.github/workflows/publish-apt-brew-pkg.yml

@@ -35,7 +35,7 @@ jobs:
     - name: Build deb package
       run: cargo deb -p meilisearch -o target/debian/meilisearch.deb
     - name: Upload debian pkg to release
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.6.1
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/debian/meilisearch.deb
@@ -53,6 +53,5 @@ jobs:
         uses: mislav/bump-homebrew-formula-action@v2
         with:
           formula-name: meilisearch
-          formula-path: Formula/m/meilisearch.rb
         env:
           COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }}

.github/workflows/publish-binaries.yml

@@ -54,7 +54,7 @@ jobs:
     # No need to upload binaries for dry run (cron)
     - name: Upload binaries to release
       if: github.event_name == 'release'
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.6.1
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/release/meilisearch
@@ -87,7 +87,7 @@ jobs:
     # No need to upload binaries for dry run (cron)
     - name: Upload binaries to release
       if: github.event_name == 'release'
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.6.1
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/release/${{ matrix.artifact_name }}
@@ -121,7 +121,7 @@ jobs:
       - name: Upload the binary to release
         # No need to upload binaries for dry run (cron)
         if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.6.1
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/${{ matrix.target }}/release/meilisearch
@@ -183,7 +183,7 @@ jobs:
       - name: Upload the binary to release
         # No need to upload binaries for dry run (cron)
         if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.6.1
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/${{ matrix.target }}/release/meilisearch

.github/workflows/publish-docker-images.yml

@@ -57,10 +57,10 @@ jobs:
           echo "date=$commit_date" >> $GITHUB_OUTPUT
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -70,7 +70,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v4
         with:
           images: getmeili/meilisearch
           # Prevent `latest` to be updated for each new tag pushed.
@@ -83,7 +83,7 @@ jobs:
             type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v4
         with:
           push: true
           platforms: linux/amd64,linux/arm64

.github/workflows/sdks-tests.yml

@@ -14,7 +14,6 @@ on:
 env:
   MEILI_MASTER_KEY: 'masterKey'
   MEILI_NO_ANALYTICS: 'true'
-  DISABLE_COVERAGE: 'true'
 
 jobs:
   define-docker-image:
@@ -31,117 +30,6 @@ jobs:
           if [[ $event == 'workflow_dispatch' ]]; then
             echo "docker-image=${{ github.event.inputs.docker_image }}" >> $GITHUB_OUTPUT
           fi
-      - name: Docker image is ${{ steps.define-image.outputs.docker-image }}
-        run: echo "Docker image is ${{ steps.define-image.outputs.docker-image }}"
-
-##########
-## SDKs ##
-##########
-
-  meilisearch-dotnet-tests:
-    needs: define-docker-image
-    name: .NET SDK tests
-    runs-on: ubuntu-latest
-    env:
-      MEILISEARCH_VERSION: ${{ needs.define-docker-image.outputs.docker-image }}
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-dotnet
-      - name: Setup .NET Core
-        uses: actions/setup-dotnet@v3
-        with:
-          dotnet-version: "6.0.x"
-      - name: Install dependencies
-        run: dotnet restore
-      - name: Build
-        run: dotnet build --configuration Release --no-restore
-      - name: Meilisearch (latest version) setup with Docker
-        run: docker compose up -d
-      - name: Run tests
-        run: dotnet test --no-restore --verbosity normal
-
-  meilisearch-dart-tests:
-    needs: define-docker-image
-    name: Dart SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-dart
-      - uses: dart-lang/setup-dart@v1
-        with:
-          sdk: 3.1.1
-      - name: Install dependencies
-        run: dart pub get
-      - name: Run integration tests
-        run: dart test --concurrency=4
-
-  meilisearch-go-tests:
-    needs: define-docker-image
-    name: Go SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-go
-      - name: Get dependencies
-        run: |
-          go get -v -t -d ./...
-          if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-          fi
-      - name: Run integration tests
-        run: go test -v ./...
-
-  meilisearch-java-tests:
-    needs: define-docker-image
-    name: Java SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-java
-      - name: Set up Java
-        uses: actions/setup-java@v3
-        with:
-          java-version: 8
-          distribution: 'zulu'
-          cache: gradle
-      - name: Grant execute permission for gradlew
-        run: chmod +x gradlew
-      - name: Build and run unit and integration tests
-        run: ./gradlew build integrationTest
 
   meilisearch-js-tests:
     needs: define-docker-image
@@ -178,6 +66,33 @@ jobs:
       - name: Run Browser env
         run: yarn test:env:browser
 
+  instant-meilisearch-tests:
+    needs: define-docker-image
+    name: instant-meilisearch tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/instant-meilisearch
+      - name: Setup node
+        uses: actions/setup-node@v3
+        with:
+          cache: yarn
+      - name: Install dependencies
+        run: yarn install
+      - name: Run tests
+        run: yarn test
+      - name: Build all the playgrounds and the packages
+        run: yarn build
+
   meilisearch-php-tests:
     needs: define-docker-image
     name: PHP SDK tests
@@ -196,6 +111,8 @@ jobs:
           repository: meilisearch/meilisearch-php
       - name: Install PHP
         uses: shivammathur/setup-php@v2
+        with:
+          coverage: none
       - name: Validate composer.json and composer.lock
         run: composer validate
       - name: Install dependencies
@@ -232,6 +149,36 @@ jobs:
       - name: Test with pytest
         run: pipenv run pytest
 
+  meilisearch-go-tests:
+    needs: define-docker-image
+    name: Go SDK tests
+    runs-on: ubuntu-latest
+    services:
+      meilisearch:
+        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
+        env:
+          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
+          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
+        ports:
+          - '7700:7700'
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version: stable
+      - uses: actions/checkout@v3
+        with:
+          repository: meilisearch/meilisearch-go
+      - name: Get dependencies
+        run: |
+          go get -v -t -d ./...
+          if [ -f Gopkg.toml ]; then
+            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+            dep ensure
+          fi
+      - name: Run integration tests
+        run: go test -v ./...
+
   meilisearch-ruby-tests:
     needs: define-docker-image
     name: Ruby SDK tests
@@ -277,110 +224,3 @@ jobs:
         run: cargo build --verbose
       - name: Run tests
         run: cargo test --verbose
-
-  meilisearch-swift-tests:
-    needs: define-docker-image
-    name: Swift SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-swift
-      - name: Run tests
-        run: swift test
-
-########################
-## FRONT-END PLUGINS ##
-########################
-
-  meilisearch-js-plugins-tests:
-    needs: define-docker-image
-    name: meilisearch-js-plugins tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-js-plugins
-      - name: Setup node
-        uses: actions/setup-node@v3
-        with:
-          cache: yarn
-      - name: Install dependencies
-        run: yarn install
-      - name: Run tests
-        run: yarn test
-      - name: Build all the playgrounds and the packages
-        run: yarn build
-
-########################
-## BACK-END PLUGINS ###
-########################
-
-  meilisearch-rails-tests:
-    needs: define-docker-image
-    name: meilisearch-rails tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-rails
-      - name: Set up Ruby 3
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3
-          bundler-cache: true
-      - name: Run tests
-        run: bundle exec rspec
-
-  meilisearch-symfony-tests:
-    needs: define-docker-image
-    name: meilisearch-symfony tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-symfony
-      - name: Install PHP
-        uses: shivammathur/setup-php@v2
-        with:
-          tools: composer:v2, flex
-      - name: Validate composer.json and composer.lock
-        run: composer validate
-      - name: Install dependencies
-        run: composer install --prefer-dist --no-progress --quiet
-      - name: Remove doctrine/annotations
-        run: composer remove --dev doctrine/annotations
-      - name: Run test suite
-        run: composer test:unit

.github/workflows/test-suite.yml

@@ -30,20 +30,20 @@ jobs:
         run: |
           apt-get update && apt-get install -y curl
           apt-get install build-essential -y
-      - name: Setup test with Rust stable
+      - name: Run test with Rust stable
         if: github.event_name != 'schedule'
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
           override: true
-      - name: Setup test with Rust nightly
-        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+      - name: Run test with Rust nightly
+        if: github.event_name == 'schedule'
         uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly
           override: true
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
+        uses: Swatinem/rust-cache@v2.4.0
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -65,7 +65,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
+        uses: Swatinem/rust-cache@v2.4.0
       - name: Run cargo check without any default features
         uses: actions-rs/cargo@v1
         with:
@@ -78,12 +78,12 @@ jobs:
           args: --locked --release --all
 
   test-all-features:
-    name: Tests all features
+    name: Tests all features on cron schedule only
     runs-on: ubuntu-latest
     container:
       # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
       image: ubuntu:18.04
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    if: github.event_name == 'schedule'
     steps:
       - uses: actions/checkout@v3
       - name: Install needed dependencies
@@ -110,27 +110,24 @@ jobs:
     runs-on: ubuntu-latest
     container:
       image: ubuntu:18.04
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    if: github.event_name == 'schedule'
     steps:
       - uses: actions/checkout@v3
       - name: Install needed dependencies
         run: |
           apt-get update
           apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
+      - uses: actions-rs/toolchain@v1 
+        with: 
+          toolchain: stable 
+          override: true 
       - name: Run cargo tree without default features and check lindera is not present
         run: |
-          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -vqz lindera; then
-            echo "lindera has been found in the sources and it shouldn't"
-            exit 1
-          fi
+          cargo tree -f '{p} {f}' -e normal --no-default-features | grep lindera -vqz
       - name: Run cargo tree with default features and check lindera is pressent
         run: |
           cargo tree -f '{p} {f}' -e normal | grep lindera -qz
-
+                
   # We run tests in debug also, to make sure that the debug_assertions are hit
   test-debug:
     name: Run tests in debug
@@ -149,7 +146,7 @@ jobs:
           toolchain: stable
           override: true
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
+        uses: Swatinem/rust-cache@v2.4.0
       - name: Run tests in debug
         uses: actions-rs/cargo@v1
         with:
@@ -164,11 +161,11 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: 1.71.1
+          toolchain: 1.69.0
           override: true
           components: clippy
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
+        uses: Swatinem/rust-cache@v2.4.0
       - name: Run cargo clippy
         uses: actions-rs/cargo@v1
         with:
@@ -187,7 +184,7 @@ jobs:
           override: true
           components: rustfmt
       - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
+        uses: Swatinem/rust-cache@v2.4.0
       - name: Run cargo fmt
         # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
         # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate

Cargo.toml

@@ -2,7 +2,6 @@
 resolver = "2"
 members = [
     "meilisearch",
-    "meilitool",
     "meilisearch-types",
     "meilisearch-auth",
     "meili-snap",
@@ -19,7 +18,7 @@ members = [
 ]
 
 [workspace.package]
-version = "1.5.0"
+version = "1.3.4"
 authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
 description = "Meilisearch HTTP server"
 homepage = "https://meilisearch.com"

Dockerfile

@@ -3,7 +3,7 @@ FROM    rust:alpine3.16 AS compiler
 
 RUN     apk add -q --update-cache --no-cache build-base openssl-dev
 
-WORKDIR /
+WORKDIR /meilisearch
 
 ARG     COMMIT_SHA
 ARG     COMMIT_DATE
@@ -17,7 +17,7 @@ RUN     set -eux; \
         if [ "$apkArch" = "aarch64" ]; then \
             export JEMALLOC_SYS_WITH_LG_PAGE=16; \
         fi && \
-        cargo build --release -p meilisearch -p meilitool
+        cargo build --release
 
 # Run
 FROM    alpine:3.16
@@ -28,10 +28,9 @@ ENV     MEILI_SERVER_PROVIDER docker
 RUN     apk update --quiet \
         && apk add -q --no-cache libgcc tini curl
 
-# add meilisearch and meilitool to the `/bin` so you can run it from anywhere
-# and it's easy to find.
-COPY    --from=compiler /target/release/meilisearch /bin/meilisearch
-COPY    --from=compiler /target/release/meilitool /bin/meilitool
+# add meilisearch to the `/bin` so you can run it from anywhere and it's easy
+# to find.
+COPY    --from=compiler /meilisearch/target/release/meilisearch /bin/meilisearch
 # To stay compatible with the older version of the container (pre v0.27.0) we're
 # going to symlink the meilisearch binary in the path to `/meilisearch`
 RUN     ln -s /bin/meilisearch /meilisearch

PROFILING.md

@@ -1,19 +0,0 @@
-# Profiling Meilisearch
-
-Search engine technologies are complex pieces of software that require thorough profiling tools. We chose to use [Puffin](https://github.com/EmbarkStudios/puffin), which the Rust gaming industry uses extensively. You can export and import the profiling reports using the top bar's _File_ menu options [in Puffin Viewer](https://github.com/embarkstudios/puffin#ui).
-
-![An example profiling with Puffin viewer](assets/profiling-example.png)
-
-## Profiling the Indexing Process
-
-When you enable [the `exportPuffinReports` experimental feature](https://www.meilisearch.com/docs/learn/experimental/overview) of Meilisearch, Puffin reports with the `.puffin` extension will be automatically exported to disk. When this option is enabled, the engine will automatically create a "frame" whenever it executes the `IndexScheduler::tick` method.
-
-[Puffin Viewer](https://github.com/EmbarkStudios/puffin/tree/main/puffin_viewer) is used to analyze the reports. Those reports show areas where Meilisearch spent time during indexing.
-
-Another piece of advice on the Puffin viewer UI interface is to consider the _Merge children with same ID_ option. It can hide the exact actual timings at which events were sent. Please turn it off when you see strange gaps on the Flamegraph. It can help.
-
-## Profiling the Search Process
-
-We still need to take the time to profile the search side of the engine with Puffin. It would require time to profile the filtering phase, query parsing, creation, and execution. We could even profile the Actix HTTP server.
-
-The only issue we see is the framing system. Puffin requires a global frame-based profiling phase, which collides with Meilisearch's ability to accept and answer multiple requests on different threads simultaneously.

README.md

@@ -1,20 +1,16 @@
 <p align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-light-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
-  </a>
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-dark-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
-  </a>
+  <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
+  <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
 </p>
 
 <h4 align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Website</a> |
+  <a href="https://www.meilisearch.com">Website</a> |
   <a href="https://roadmap.meilisearch.com/tabs/1-under-consideration">Roadmap</a> |
-  <a href="https://www.meilisearch.com/pricing?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Meilisearch Cloud</a> |
-  <a href="https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Blog</a> |
-  <a href="https://www.meilisearch.com/docs?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Documentation</a> |
-  <a href="https://www.meilisearch.com/docs/faq?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">FAQ</a> |
-  <a href="https://discord.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Discord</a>
+  <a href="https://www.meilisearch.com/pricing?utm_campaign=oss&utm_source=engine&utm_medium=meilisearch">Meilisearch Cloud</a> |
+  <a href="https://blog.meilisearch.com">Blog</a> |
+  <a href="https://www.meilisearch.com/docs">Documentation</a> |
+  <a href="https://www.meilisearch.com/docs/faq">FAQ</a> |
+  <a href="https://discord.meilisearch.com">Discord</a>
 </h4>
 
 <p align="center">
@@ -25,49 +21,43 @@
 
 <p align="center">⚡ A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow 🔍</p>
 
----
-
-### 🔥 On November 2nd, we are hosting our first-ever live demo and product updates for [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). Make sure to [register here](https://us06web.zoom.us/meeting/register/tZMlc-mqrjIsH912-HTRe-AaT-pp41bDe81a#/registration) and bring your questions for live Q&A!
-
----
-
 Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow.
 
 <p align="center" name="demo">
-  <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-light-mode-only" target="_blank">
+  <a href="https://where2watch.meilisearch.com/#gh-light-mode-only" target="_blank">
     <img src="assets/demo-light.gif#gh-light-mode-only" alt="A bright colored application for finding movies screening near the user">
   </a>
-  <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-dark-mode-only" target="_blank">
+  <a href="https://where2watch.meilisearch.com/#gh-dark-mode-only" target="_blank">
     <img src="assets/demo-dark.gif#gh-dark-mode-only" alt="A dark colored application for finding movies screening near the user">
   </a>
 </p>
 
-🔥 [**Try it!**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-link) 🔥
+🔥 [**Try it!**](https://where2watch.meilisearch.com/) 🔥
 
 ## ✨ Features
 
 - **Search-as-you-type:** find search results in less than 50 milliseconds
-- **[Typo tolerance](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#typo-tolerance):** get relevant matches even when queries contain typos and misspellings
-- **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code
-- **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** sort results based on price, date, or pretty much anything else your users need
-- **[Synonym support](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#synonyms):** configure synonyms to include more relevant content in your search results
-- **[Geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** filter and sort documents based on geographic data
-- **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
-- **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** control which users can access what data with API keys that allow fine-grained permissions handling
-- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** personalize search results for any number of application tenants
+- **[Typo tolerance](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy#typo-tolerance):** get relevant matches even when queries contain typos and misspellings
+- **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code
+- **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting):** sort results based on price, date, or pretty much anything else your users need
+- **[Synonym support](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy#synonyms):** configure synonyms to include more relevant content in your search results
+- **[Geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch):** filter and sort documents based on geographic data
+- **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
+- **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys):** control which users can access what data with API keys that allow fine-grained permissions handling
+- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/tenant_tokens):** personalize search results for any number of application tenants
 - **Highly Customizable:** customize Meilisearch to your specific needs or use our out-of-the-box and hassle-free presets
-- **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** integrate Meilisearch in your technical stack with our plugins and SDKs
+- **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview):** integrate Meilisearch in your technical stack with our plugins and SDKs
 - **Easy to install, deploy, and maintain**
 
 ## 📖 Documentation
 
-You can consult Meilisearch's documentation at [https://www.meilisearch.com/docs](https://www.meilisearch.com/docs/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=docs).
+You can consult Meilisearch's documentation at [https://www.meilisearch.com/docs](https://www.meilisearch.com/docs/).
 
 ## 🚀 Getting started
 
-For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
+For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start) guide.
 
-You may also want to check out [Meilisearch 101](https://www.meilisearch.com/docs/learn/getting_started/filtering_and_sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) for an introduction to some of Meilisearch's most popular features.
+You may also want to check out [Meilisearch 101](https://www.meilisearch.com/docs/learn/getting_started/filtering_and_sorting) for an introduction to some of Meilisearch's most popular features.
 
 ## ⚡ Supercharge your Meilisearch experience
 
@@ -77,29 +67,29 @@ Say goodbye to server deployment and manual updates with [Meilisearch Cloud](htt
 
 Install one of our SDKs in your project for seamless integration between Meilisearch and your favorite language or framework!
 
-Take a look at the complete [Meilisearch integration list](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-link).
+Take a look at the complete [Meilisearch integration list](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks).
 
-[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-logos)
+[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks)
 
 ## ⚙️ Advanced usage
 
-Experienced users will want to keep our [API Reference](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) close at hand.
+Experienced users will want to keep our [API Reference](https://www.meilisearch.com/docs/reference/api/overview) close at hand.
 
-We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [API keys](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), and [tenant tokens](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
+We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering), [sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting), [geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch), [API keys](https://www.meilisearch.com/docs/learn/security/master_api_keys), and [tenant tokens](https://www.meilisearch.com/docs/learn/security/tenant_tokens).
 
-Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
+Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes).
 
 ## 📊 Telemetry
 
-Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
+Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry#how-to-disable-data-collection) whenever you want.
 
-To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
+To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
 
-If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) of our documentation.
+If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry) of our documentation.
 
 ## 📫 Get in touch!
 
-Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
+Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/)
 
 🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
 

benchmarks/Cargo.toml

@@ -14,7 +14,7 @@ license.workspace = true
 anyhow = "1.0.70"
 csv = "1.2.1"
 milli = { path = "../milli" }
-mimalloc = { version = "0.1.37", default-features = false }
+mimalloc = { version = "0.1.36", default-features = false }
 serde_json = { version = "1.0.95", features = ["preserve_order"] }
 
 [dev-dependencies]

dump/src/lib.rs

@@ -262,9 +262,6 @@ pub(crate) mod test {
             sortable_attributes: Setting::Set(btreeset! { S("age") }),
             ranking_rules: Setting::NotSet,
             stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
             synonyms: Setting::NotSet,
             distinct_attribute: Setting::NotSet,
             typo_tolerance: Setting::NotSet,

dump/src/reader/compat/v5_to_v6.rs

@@ -340,9 +340,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                 }
             },
             stop_words: settings.stop_words.into(),
-            non_separator_tokens: v6::Setting::NotSet,
-            separator_tokens: v6::Setting::NotSet,
-            dictionary: v6::Setting::NotSet,
             synonyms: settings.synonyms.into(),
             distinct_attribute: settings.distinct_attribute.into(),
             typo_tolerance: match settings.typo_tolerance {

fuzzers/Cargo.toml

@@ -13,7 +13,7 @@ license.workspace = true
 [dependencies]
 arbitrary = { version = "1.3.0", features = ["derive"] }
 clap = { version = "4.3.0", features = ["derive"] }
-fastrand = "2.0.0"
+fastrand = "1.9.0"
 milli = { path = "../milli" }
 serde = { version = "1.0.160", features = ["derive"] }
 serde_json = { version = "1.0.95", features = ["preserve_order"] }

index-scheduler/Cargo.toml

@@ -22,7 +22,6 @@ log = "0.4.17"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
 page_size = "0.5.0"
-puffin = "0.16.0"
 roaring = { version = "0.10.1", features = ["serde"] }
 serde = { version = "1.0.160", features = ["derive"] }
 serde_json = { version = "1.0.95", features = ["preserve_order"] }

index-scheduler/src/batch.rs

@@ -19,7 +19,6 @@ one indexing operation.
 
 use std::collections::{BTreeSet, HashSet};
 use std::ffi::OsStr;
-use std::fmt;
 use std::fs::{self, File};
 use std::io::BufWriter;
 
@@ -200,29 +199,6 @@ impl Batch {
     }
 }
 
-impl fmt::Display for Batch {
-    /// A text used when we debug the profiling reports.
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let index_uid = self.index_uid();
-        let tasks = self.ids();
-        match self {
-            Batch::TaskCancelation { .. } => f.write_str("TaskCancelation")?,
-            Batch::TaskDeletion(_) => f.write_str("TaskDeletion")?,
-            Batch::SnapshotCreation(_) => f.write_str("SnapshotCreation")?,
-            Batch::Dump(_) => f.write_str("Dump")?,
-            Batch::IndexOperation { op, .. } => write!(f, "{op}")?,
-            Batch::IndexCreation { .. } => f.write_str("IndexCreation")?,
-            Batch::IndexUpdate { .. } => f.write_str("IndexUpdate")?,
-            Batch::IndexDeletion { .. } => f.write_str("IndexDeletion")?,
-            Batch::IndexSwap { .. } => f.write_str("IndexSwap")?,
-        };
-        match index_uid {
-            Some(name) => f.write_fmt(format_args!(" on {name:?} from tasks: {tasks:?}")),
-            None => f.write_fmt(format_args!(" from tasks: {tasks:?}")),
-        }
-    }
-}
-
 impl IndexOperation {
     pub fn index_uid(&self) -> &str {
         match self {
@@ -237,30 +213,6 @@ impl IndexOperation {
     }
 }
 
-impl fmt::Display for IndexOperation {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            IndexOperation::DocumentOperation { .. } => {
-                f.write_str("IndexOperation::DocumentOperation")
-            }
-            IndexOperation::DocumentDeletion { .. } => {
-                f.write_str("IndexOperation::DocumentDeletion")
-            }
-            IndexOperation::IndexDocumentDeletionByFilter { .. } => {
-                f.write_str("IndexOperation::IndexDocumentDeletionByFilter")
-            }
-            IndexOperation::DocumentClear { .. } => f.write_str("IndexOperation::DocumentClear"),
-            IndexOperation::Settings { .. } => f.write_str("IndexOperation::Settings"),
-            IndexOperation::DocumentClearAndSetting { .. } => {
-                f.write_str("IndexOperation::DocumentClearAndSetting")
-            }
-            IndexOperation::SettingsAndDocumentOperation { .. } => {
-                f.write_str("IndexOperation::SettingsAndDocumentOperation")
-            }
-        }
-    }
-}
-
 impl IndexScheduler {
     /// Convert an [`BatchKind`](crate::autobatcher::BatchKind) into a [`Batch`].
     ///
@@ -522,8 +474,6 @@ impl IndexScheduler {
         #[cfg(test)]
         self.maybe_fail(crate::tests::FailureLocation::InsideCreateBatch)?;
 
-        puffin::profile_function!();
-
         let enqueued = &self.get_status(rtxn, Status::Enqueued)?;
         let to_cancel = self.get_kind(rtxn, Kind::TaskCancelation)? & enqueued;
 
@@ -587,7 +537,9 @@ impl IndexScheduler {
         let index_tasks = self.index_tasks(rtxn, index_name)? & enqueued;
 
         // If autobatching is disabled we only take one task at a time.
-        let tasks_limit = if self.autobatching_enabled { usize::MAX } else { 1 };
+        // Otherwise, we take only a maximum of tasks to create batches.
+        let tasks_limit =
+            if self.autobatching_enabled { self.maximum_number_of_batched_tasks } else { 1 };
 
         let enqueued = index_tasks
             .into_iter()
@@ -628,9 +580,6 @@ impl IndexScheduler {
             self.maybe_fail(crate::tests::FailureLocation::PanicInsideProcessBatch)?;
             self.breakpoint(crate::Breakpoint::InsideProcessBatch);
         }
-
-        puffin::profile_function!(batch.to_string());
-
         match batch {
             Batch::TaskCancelation { mut task, previous_started_at, previous_processing_tasks } => {
                 // 1. Retrieve the tasks that matched the query at enqueue-time.
@@ -896,7 +845,7 @@ impl IndexScheduler {
                 })?;
 
                 // 4. Dump experimental feature settings
-                let features = self.features().runtime_features();
+                let features = self.features()?.runtime_features();
                 dump.create_experimental_features(features)?;
 
                 let dump_uid = started_at.format(format_description!(
@@ -923,10 +872,6 @@ impl IndexScheduler {
                     self.index_mapper.index(&rtxn, &index_uid)?
                 };
 
-                // the index operation can take a long time, so save this handle to make it available to the search for the duration of the tick
-                *self.currently_updating_index.write().unwrap() =
-                    Some((index_uid.clone(), index.clone()));
-
                 let mut index_wtxn = index.write_txn()?;
                 let tasks = self.apply_index_operation(&mut index_wtxn, &index, op)?;
                 index_wtxn.commit()?;
@@ -1126,8 +1071,6 @@ impl IndexScheduler {
         index: &'i Index,
         operation: IndexOperation,
     ) -> Result<Vec<Task>> {
-        puffin::profile_function!();
-
         match operation {
             IndexOperation::DocumentClear { mut tasks, .. } => {
                 let count = milli::update::ClearDocuments::new(index_wtxn, index).execute()?;

index-scheduler/src/features.rs

@@ -1,8 +1,6 @@
-use std::sync::{Arc, RwLock};
-
 use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures};
 use meilisearch_types::heed::types::{SerdeJson, Str};
-use meilisearch_types::heed::{Database, Env, RwTxn};
+use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn};
 
 use crate::error::FeatureNotEnabledError;
 use crate::Result;
@@ -11,19 +9,20 @@ const EXPERIMENTAL_FEATURES: &str = "experimental-features";
 
 #[derive(Clone)]
 pub(crate) struct FeatureData {
-    persisted: Database<Str, SerdeJson<RuntimeTogglableFeatures>>,
-    runtime: Arc<RwLock<RuntimeTogglableFeatures>>,
+    runtime: Database<Str, SerdeJson<RuntimeTogglableFeatures>>,
+    instance: InstanceTogglableFeatures,
 }
 
 #[derive(Debug, Clone, Copy)]
 pub struct RoFeatures {
     runtime: RuntimeTogglableFeatures,
+    instance: InstanceTogglableFeatures,
 }
 
 impl RoFeatures {
-    fn new(data: &FeatureData) -> Self {
-        let runtime = data.runtime_features();
-        Self { runtime }
+    fn new(txn: RoTxn<'_>, data: &FeatureData) -> Result<Self> {
+        let runtime = data.runtime_features(txn)?;
+        Ok(Self { runtime, instance: data.instance })
     }
 
     pub fn runtime_features(&self) -> RuntimeTogglableFeatures {
@@ -44,13 +43,13 @@ impl RoFeatures {
     }
 
     pub fn check_metrics(&self) -> Result<()> {
-        if self.runtime.metrics {
+        if self.instance.metrics {
             Ok(())
         } else {
             Err(FeatureNotEnabledError {
                 disabled_action: "Getting metrics",
                 feature: "metrics",
-                issue_link: "https://github.com/meilisearch/product/discussions/625",
+                issue_link: "https://github.com/meilisearch/meilisearch/discussions/3518",
             }
             .into())
         }
@@ -68,36 +67,15 @@ impl RoFeatures {
             .into())
         }
     }
-
-    pub fn check_puffin(&self) -> Result<()> {
-        if self.runtime.export_puffin_reports {
-            Ok(())
-        } else {
-            Err(FeatureNotEnabledError {
-                disabled_action: "Outputting Puffin reports to disk",
-                feature: "export puffin reports",
-                issue_link: "https://github.com/meilisearch/product/discussions/693",
-            }
-            .into())
-        }
-    }
 }
 
 impl FeatureData {
     pub fn new(env: &Env, instance_features: InstanceTogglableFeatures) -> Result<Self> {
         let mut wtxn = env.write_txn()?;
-        let runtime_features_db = env.create_database(&mut wtxn, Some(EXPERIMENTAL_FEATURES))?;
+        let runtime_features = env.create_database(&mut wtxn, Some(EXPERIMENTAL_FEATURES))?;
         wtxn.commit()?;
 
-        let txn = env.read_txn()?;
-        let persisted_features: RuntimeTogglableFeatures =
-            runtime_features_db.get(&txn, EXPERIMENTAL_FEATURES)?.unwrap_or_default();
-        let runtime = Arc::new(RwLock::new(RuntimeTogglableFeatures {
-            metrics: instance_features.metrics || persisted_features.metrics,
-            ..persisted_features
-        }));
-
-        Ok(Self { persisted: runtime_features_db, runtime })
+        Ok(Self { runtime: runtime_features, instance: instance_features })
     }
 
     pub fn put_runtime_features(
@@ -105,25 +83,16 @@ impl FeatureData {
         mut wtxn: RwTxn,
         features: RuntimeTogglableFeatures,
     ) -> Result<()> {
-        self.persisted.put(&mut wtxn, EXPERIMENTAL_FEATURES, &features)?;
+        self.runtime.put(&mut wtxn, EXPERIMENTAL_FEATURES, &features)?;
         wtxn.commit()?;
-
-        // safe to unwrap, the lock will only fail if:
-        // 1. requested by the same thread concurrently -> it is called and released in methods that don't call each other
-        // 2. there's a panic while the thread is held -> it is only used for an assignment here.
-        let mut toggled_features = self.runtime.write().unwrap();
-        *toggled_features = features;
         Ok(())
     }
 
-    fn runtime_features(&self) -> RuntimeTogglableFeatures {
-        // sound to unwrap, the lock will only fail if:
-        // 1. requested by the same thread concurrently -> it is called and released in methods that don't call each other
-        // 2. there's a panic while the thread is held -> it is only used for copying the data here
-        *self.runtime.read().unwrap()
+    fn runtime_features(&self, txn: RoTxn) -> Result<RuntimeTogglableFeatures> {
+        Ok(self.runtime.get(&txn, EXPERIMENTAL_FEATURES)?.unwrap_or_default())
     }
 
-    pub fn features(&self) -> RoFeatures {
-        RoFeatures::new(self)
+    pub fn features(&self, txn: RoTxn) -> Result<RoFeatures> {
+        RoFeatures::new(txn, self)
     }
 }

index-scheduler/src/insta_snapshot.rs

@@ -15,6 +15,7 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
 
     let IndexScheduler {
         autobatching_enabled,
+        maximum_number_of_batched_tasks: _,
         must_stop_processing: _,
         processing_tasks,
         file_store,
@@ -30,7 +31,6 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         index_mapper,
         features: _,
         max_number_of_tasks: _,
-        puffin_frame: _,
         wake_up: _,
         dumps_path: _,
         snapshots_path: _,
@@ -39,7 +39,6 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         test_breakpoint_sdr: _,
         planned_failures: _,
         run_loop_iteration: _,
-        currently_updating_index: _,
     } = scheduler;
 
     let rtxn = env.read_txn().unwrap();

index-scheduler/src/lib.rs

@@ -27,13 +27,12 @@ mod index_mapper;
 mod insta_snapshot;
 mod lru;
 mod utils;
-pub mod uuid_codec;
+mod uuid_codec;
 
 pub type Result<T> = std::result::Result<T, Error>;
 pub type TaskId = u32;
 
 use std::collections::{BTreeMap, HashMap};
-use std::fs::File;
 use std::ops::{Bound, RangeBounds};
 use std::path::{Path, PathBuf};
 use std::sync::atomic::AtomicBool;
@@ -53,7 +52,6 @@ use meilisearch_types::milli::documents::DocumentsBatchBuilder;
 use meilisearch_types::milli::update::IndexerConfig;
 use meilisearch_types::milli::{self, CboRoaringBitmapCodec, Index, RoaringBitmapCodec, BEU32};
 use meilisearch_types::tasks::{Kind, KindWithContent, Status, Task};
-use puffin::FrameView;
 use roaring::RoaringBitmap;
 use synchronoise::SignalEvent;
 use time::format_description::well_known::Rfc3339;
@@ -255,6 +253,9 @@ pub struct IndexSchedulerOptions {
     /// Set to `true` iff the index scheduler is allowed to automatically
     /// batch tasks together, to process multiple tasks at once.
     pub autobatching_enabled: bool,
+    /// If the autobatcher is allowed to automatically batch tasks
+    /// it will only batch this defined number of tasks at once.
+    pub maximum_number_of_batched_tasks: usize,
     /// The maximum number of tasks stored in the task queue before starting
     /// to auto schedule task deletions.
     pub max_number_of_tasks: usize,
@@ -312,13 +313,13 @@ pub struct IndexScheduler {
     /// Whether auto-batching is enabled or not.
     pub(crate) autobatching_enabled: bool,
 
+    /// The maximum number of tasks that will be batched together.
+    pub(crate) maximum_number_of_batched_tasks: usize,
+
     /// The max number of tasks allowed before the scheduler starts to delete
     /// the finished tasks automatically.
     pub(crate) max_number_of_tasks: usize,
 
-    /// A frame to output the indexation profiling files to disk.
-    pub(crate) puffin_frame: Arc<puffin::GlobalFrameView>,
-
     /// The path used to create the dumps.
     pub(crate) dumps_path: PathBuf,
 
@@ -331,10 +332,6 @@ pub struct IndexScheduler {
     /// The path to the version file of Meilisearch.
     pub(crate) version_file_path: PathBuf,
 
-    /// A few types of long running batches of tasks that act on a single index set this field
-    /// so that a handle to the index is available from other threads (search) in an optimized manner.
-    currently_updating_index: Arc<RwLock<Option<(String, Index)>>>,
-
     // ================= test
     // The next entry is dedicated to the tests.
     /// Provide a way to set a breakpoint in multiple part of the scheduler.
@@ -372,13 +369,12 @@ impl IndexScheduler {
             index_mapper: self.index_mapper.clone(),
             wake_up: self.wake_up.clone(),
             autobatching_enabled: self.autobatching_enabled,
+            maximum_number_of_batched_tasks: self.maximum_number_of_batched_tasks,
             max_number_of_tasks: self.max_number_of_tasks,
-            puffin_frame: self.puffin_frame.clone(),
             snapshots_path: self.snapshots_path.clone(),
             dumps_path: self.dumps_path.clone(),
             auth_path: self.auth_path.clone(),
             version_file_path: self.version_file_path.clone(),
-            currently_updating_index: self.currently_updating_index.clone(),
             #[cfg(test)]
             test_breakpoint_sdr: self.test_breakpoint_sdr.clone(),
             #[cfg(test)]
@@ -468,14 +464,13 @@ impl IndexScheduler {
             env,
             // we want to start the loop right away in case meilisearch was ctrl+Ced while processing things
             wake_up: Arc::new(SignalEvent::auto(true)),
-            puffin_frame: Arc::new(puffin::GlobalFrameView::default()),
             autobatching_enabled: options.autobatching_enabled,
+            maximum_number_of_batched_tasks: options.maximum_number_of_batched_tasks,
             max_number_of_tasks: options.max_number_of_tasks,
             dumps_path: options.dumps_path,
             snapshots_path: options.snapshots_path,
             auth_path: options.auth_path,
             version_file_path: options.version_file_path,
-            currently_updating_index: Arc::new(RwLock::new(None)),
 
             #[cfg(test)]
             test_breakpoint_sdr,
@@ -585,46 +580,17 @@ impl IndexScheduler {
                 run.wake_up.wait();
 
                 loop {
-                    let puffin_enabled = run.features().check_puffin().is_ok();
-                    puffin::set_scopes_on(puffin_enabled);
-                    puffin::GlobalProfiler::lock().new_frame();
-
                     match run.tick() {
                         Ok(TickOutcome::TickAgain(_)) => (),
                         Ok(TickOutcome::WaitForSignal) => run.wake_up.wait(),
                         Err(e) => {
-                            log::error!("{e}");
+                            log::error!("{}", e);
                             // Wait one second when an irrecoverable error occurs.
                             if !e.is_recoverable() {
                                 std::thread::sleep(Duration::from_secs(1));
                             }
                         }
                     }
-
-                    // Let's write the previous frame to disk but only if
-                    // the user wanted to profile with puffin.
-                    if puffin_enabled {
-                        let mut frame_view = run.puffin_frame.lock();
-                        if !frame_view.is_empty() {
-                            let now = OffsetDateTime::now_utc();
-                            let mut file = match File::create(format!("{}.puffin", now)) {
-                                Ok(file) => file,
-                                Err(e) => {
-                                    log::error!("{e}");
-                                    continue;
-                                }
-                            };
-                            if let Err(e) = frame_view.save_to_writer(&mut file) {
-                                log::error!("{e}");
-                            }
-                            if let Err(e) = file.sync_all() {
-                                log::error!("{e}");
-                            }
-                            // We erase this frame view as it is no more useful. We want to
-                            // measure the new frames now that we exported the previous ones.
-                            *frame_view = FrameView::default();
-                        }
-                    }
                 }
             })
             .unwrap();
@@ -658,13 +624,6 @@ impl IndexScheduler {
     /// If you need to fetch information from or perform an action on all indexes,
     /// see the `try_for_each_index` function.
     pub fn index(&self, name: &str) -> Result<Index> {
-        if let Some((current_name, current_index)) =
-            self.currently_updating_index.read().unwrap().as_ref()
-        {
-            if current_name == name {
-                return Ok(current_index.clone());
-            }
-        }
         let rtxn = self.env.read_txn()?;
         self.index_mapper.index(&rtxn, name)
     }
@@ -1146,9 +1105,6 @@ impl IndexScheduler {
             handle.join().unwrap_or(Err(Error::ProcessBatchPanicked))
         };
 
-        // Reset the currently updating index to relinquish the index handle
-        *self.currently_updating_index.write().unwrap() = None;
-
         #[cfg(test)]
         self.maybe_fail(tests::FailureLocation::AcquiringWtxn)?;
 
@@ -1309,8 +1265,9 @@ impl IndexScheduler {
         Ok(IndexStats { is_indexing, inner_stats: index_stats })
     }
 
-    pub fn features(&self) -> RoFeatures {
-        self.features.features()
+    pub fn features(&self) -> Result<RoFeatures> {
+        let rtxn = self.read_txn()?;
+        self.features.features(rtxn)
     }
 
     pub fn put_runtime_features(&self, features: RuntimeTogglableFeatures) -> Result<()> {
@@ -1638,6 +1595,7 @@ mod tests {
                 index_count: 5,
                 indexer_config,
                 autobatching_enabled: true,
+                maximum_number_of_batched_tasks: usize::MAX,
                 max_number_of_tasks: 1_000_000,
                 instance_features: Default::default(),
             };

meilisearch-auth/src/store.rs

@@ -129,9 +129,6 @@ impl HeedAuthStore {
                 Action::DumpsAll => {
                     actions.insert(Action::DumpsCreate);
                 }
-                Action::SnapshotsAll => {
-                    actions.insert(Action::SnapshotsCreate);
-                }
                 Action::TasksAll => {
                     actions.extend([Action::TasksGet, Action::TasksDelete, Action::TasksCancel]);
                 }

meilisearch-types/Cargo.toml

@@ -15,13 +15,13 @@ actix-web = { version = "4.3.1", default-features = false }
 anyhow = "1.0.70"
 convert_case = "0.6.0"
 csv = "1.2.1"
-deserr = { version = "0.6.0", features = ["actix-web"]}
+deserr = "0.5.0"
 either = { version = "1.8.1", features = ["serde"] }
 enum-iterator = "1.4.0"
 file-store = { path = "../file-store" }
 flate2 = "1.0.25"
 fst = "0.4.7"
-memmap2 = "0.7.1"
+memmap2 = "0.5.10"
 milli = { path = "../milli" }
 roaring = { version = "0.10.1", features = ["serde"] }
 serde = { version = "1.0.160", features = ["derive"] }
@@ -50,7 +50,6 @@ hebrew = ["milli/hebrew"]
 japanese = ["milli/japanese"]
 # thai specialized tokenization
 thai = ["milli/thai"]
+
 # allow greek specialized tokenization
 greek = ["milli/greek"]
-# allow khmer specialized tokenization
-khmer = ["milli/khmer"]

meilisearch-types/src/document_formats.rs

@@ -1,3 +1,4 @@
+use std::borrow::Borrow;
 use std::fmt::{self, Debug, Display};
 use std::fs::File;
 use std::io::{self, Seek, Write};
@@ -41,7 +42,7 @@ impl Display for DocumentFormatError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             Self::Io(e) => write!(f, "{e}"),
-            Self::MalformedPayload(me, b) => match me {
+            Self::MalformedPayload(me, b) => match me.borrow() {
                 Error::Json(se) => {
                     let mut message = match se.classify() {
                         Category::Data => {

meilisearch-types/src/error.rs

@@ -259,9 +259,6 @@ InvalidSettingsRankingRules           , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSearchableAttributes   , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSortableAttributes     , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsStopWords              , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsNonSeparatorTokens     , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsSeparatorTokens        , InvalidRequest       , BAD_REQUEST ;
-InvalidSettingsDictionary             , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsSynonyms               , InvalidRequest       , BAD_REQUEST ;
 InvalidSettingsTypoTolerance          , InvalidRequest       , BAD_REQUEST ;
 InvalidState                          , Internal             , INTERNAL_SERVER_ERROR ;

meilisearch-types/src/features.rs

@@ -5,8 +5,6 @@ use serde::{Deserialize, Serialize};
 pub struct RuntimeTogglableFeatures {
     pub score_details: bool,
     pub vector_store: bool,
-    pub metrics: bool,
-    pub export_puffin_reports: bool,
 }
 
 #[derive(Default, Debug, Clone, Copy)]

meilisearch-types/src/keys.rs

@@ -257,12 +257,6 @@ pub enum Action {
     #[serde(rename = "dumps.create")]
     #[deserr(rename = "dumps.create")]
     DumpsCreate,
-    #[serde(rename = "snapshots.*")]
-    #[deserr(rename = "snapshots.*")]
-    SnapshotsAll,
-    #[serde(rename = "snapshots.create")]
-    #[deserr(rename = "snapshots.create")]
-    SnapshotsCreate,
     #[serde(rename = "version")]
     #[deserr(rename = "version")]
     Version,
@@ -315,7 +309,6 @@ impl Action {
             METRICS_GET => Some(Self::MetricsGet),
             DUMPS_ALL => Some(Self::DumpsAll),
             DUMPS_CREATE => Some(Self::DumpsCreate),
-            SNAPSHOTS_CREATE => Some(Self::SnapshotsCreate),
             VERSION => Some(Self::Version),
             KEYS_CREATE => Some(Self::KeysAdd),
             KEYS_GET => Some(Self::KeysGet),
@@ -360,7 +353,6 @@ pub mod actions {
     pub const METRICS_GET: u8 = MetricsGet.repr();
     pub const DUMPS_ALL: u8 = DumpsAll.repr();
     pub const DUMPS_CREATE: u8 = DumpsCreate.repr();
-    pub const SNAPSHOTS_CREATE: u8 = SnapshotsCreate.repr();
     pub const VERSION: u8 = Version.repr();
     pub const KEYS_CREATE: u8 = KeysAdd.repr();
     pub const KEYS_GET: u8 = KeysGet.repr();

meilisearch-types/src/settings.rs

@@ -171,15 +171,6 @@ pub struct Settings<T> {
     #[deserr(default, error = DeserrJsonError<InvalidSettingsStopWords>)]
     pub stop_words: Setting<BTreeSet<String>>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    #[deserr(default, error = DeserrJsonError<InvalidSettingsNonSeparatorTokens>)]
-    pub non_separator_tokens: Setting<BTreeSet<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    #[deserr(default, error = DeserrJsonError<InvalidSettingsSeparatorTokens>)]
-    pub separator_tokens: Setting<BTreeSet<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    #[deserr(default, error = DeserrJsonError<InvalidSettingsDictionary>)]
-    pub dictionary: Setting<BTreeSet<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
     #[deserr(default, error = DeserrJsonError<InvalidSettingsSynonyms>)]
     pub synonyms: Setting<BTreeMap<String, Vec<String>>>,
     #[serde(default, skip_serializing_if = "Setting::is_not_set")]
@@ -210,9 +201,6 @@ impl Settings<Checked> {
             ranking_rules: Setting::Reset,
             stop_words: Setting::Reset,
             synonyms: Setting::Reset,
-            non_separator_tokens: Setting::Reset,
-            separator_tokens: Setting::Reset,
-            dictionary: Setting::Reset,
             distinct_attribute: Setting::Reset,
             typo_tolerance: Setting::Reset,
             faceting: Setting::Reset,
@@ -229,9 +217,6 @@ impl Settings<Checked> {
             sortable_attributes,
             ranking_rules,
             stop_words,
-            non_separator_tokens,
-            separator_tokens,
-            dictionary,
             synonyms,
             distinct_attribute,
             typo_tolerance,
@@ -247,9 +232,6 @@ impl Settings<Checked> {
             sortable_attributes,
             ranking_rules,
             stop_words,
-            non_separator_tokens,
-            separator_tokens,
-            dictionary,
             synonyms,
             distinct_attribute,
             typo_tolerance,
@@ -292,9 +274,6 @@ impl Settings<Unchecked> {
             ranking_rules: self.ranking_rules,
             stop_words: self.stop_words,
             synonyms: self.synonyms,
-            non_separator_tokens: self.non_separator_tokens,
-            separator_tokens: self.separator_tokens,
-            dictionary: self.dictionary,
             distinct_attribute: self.distinct_attribute,
             typo_tolerance: self.typo_tolerance,
             faceting: self.faceting,
@@ -356,28 +335,6 @@ pub fn apply_settings_to_builder(
         Setting::NotSet => (),
     }
 
-    match settings.non_separator_tokens {
-        Setting::Set(ref non_separator_tokens) => {
-            builder.set_non_separator_tokens(non_separator_tokens.clone())
-        }
-        Setting::Reset => builder.reset_non_separator_tokens(),
-        Setting::NotSet => (),
-    }
-
-    match settings.separator_tokens {
-        Setting::Set(ref separator_tokens) => {
-            builder.set_separator_tokens(separator_tokens.clone())
-        }
-        Setting::Reset => builder.reset_separator_tokens(),
-        Setting::NotSet => (),
-    }
-
-    match settings.dictionary {
-        Setting::Set(ref dictionary) => builder.set_dictionary(dictionary.clone()),
-        Setting::Reset => builder.reset_dictionary(),
-        Setting::NotSet => (),
-    }
-
     match settings.synonyms {
         Setting::Set(ref synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()),
         Setting::Reset => builder.reset_synonyms(),
@@ -502,14 +459,15 @@ pub fn settings(
         })
         .transpose()?
         .unwrap_or_default();
-
-    let non_separator_tokens = index.non_separator_tokens(rtxn)?.unwrap_or_default();
-    let separator_tokens = index.separator_tokens(rtxn)?.unwrap_or_default();
-    let dictionary = index.dictionary(rtxn)?.unwrap_or_default();
-
     let distinct_field = index.distinct_field(rtxn)?.map(String::from);
 
-    let synonyms = index.user_defined_synonyms(rtxn)?;
+    // in milli each word in the synonyms map were split on their separator. Since we lost
+    // this information we are going to put space between words.
+    let synonyms = index
+        .synonyms(rtxn)?
+        .iter()
+        .map(|(key, values)| (key.join(" "), values.iter().map(|value| value.join(" ")).collect()))
+        .collect();
 
     let min_typo_word_len = MinWordSizeTyposSetting {
         one_typo: Setting::Set(index.min_word_len_one_typo(rtxn)?),
@@ -562,9 +520,6 @@ pub fn settings(
         sortable_attributes: Setting::Set(sortable_attributes),
         ranking_rules: Setting::Set(criteria.iter().map(|c| c.clone().into()).collect()),
         stop_words: Setting::Set(stop_words),
-        non_separator_tokens: Setting::Set(non_separator_tokens),
-        separator_tokens: Setting::Set(separator_tokens),
-        dictionary: Setting::Set(dictionary),
         distinct_attribute: match distinct_field {
             Some(field) => Setting::Set(field),
             None => Setting::Reset,
@@ -687,9 +642,6 @@ pub(crate) mod test {
             sortable_attributes: Setting::NotSet,
             ranking_rules: Setting::NotSet,
             stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
             synonyms: Setting::NotSet,
             distinct_attribute: Setting::NotSet,
             typo_tolerance: Setting::NotSet,
@@ -711,9 +663,6 @@ pub(crate) mod test {
             sortable_attributes: Setting::NotSet,
             ranking_rules: Setting::NotSet,
             stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
             synonyms: Setting::NotSet,
             distinct_attribute: Setting::NotSet,
             typo_tolerance: Setting::NotSet,

meilisearch/Cargo.toml

@@ -39,7 +39,7 @@ byte-unit = { version = "4.0.19", default-features = false, features = [
 bytes = "1.4.0"
 clap = { version = "4.2.1", features = ["derive", "env"] }
 crossbeam-channel = "0.5.8"
-deserr = { version = "0.6.0", features = ["actix-web"]}
+deserr = "0.5.0"
 dump = { path = "../dump" }
 either = "1.8.1"
 env_logger = "0.10.0"
@@ -50,15 +50,15 @@ futures = "0.3.28"
 futures-util = "0.3.28"
 http = "0.2.9"
 index-scheduler = { path = "../index-scheduler" }
-indexmap = { version = "2.0.0", features = ["serde"] }
+indexmap = { version = "1.9.3", features = ["serde-1"] }
 is-terminal = "0.4.8"
-itertools = "0.11.0"
+itertools = "0.10.5"
 jsonwebtoken = "8.3.0"
 lazy_static = "1.4.0"
 log = "0.4.17"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-mimalloc = { version = "0.1.37", default-features = false }
+mimalloc = { version = "0.1.36", default-features = false }
 mime = "0.3.17"
 num_cpus = "1.15.0"
 obkv = "0.2.0"
@@ -69,7 +69,6 @@ permissive-json-pointer = { path = "../permissive-json-pointer" }
 pin-project-lite = "0.2.9"
 platform-dirs = "0.3.0"
 prometheus = { version = "0.13.3", features = ["process"] }
-puffin = { version = "0.16.0", features = ["serialization"] }
 rand = "0.8.5"
 rayon = "1.7.0"
 regex = "1.7.3"
@@ -86,7 +85,7 @@ sha2 = "0.10.6"
 siphasher = "0.3.10"
 slice-group-by = "0.3.0"
 static-files = { version = "0.2.3", optional = true }
-sysinfo = "0.29.7"
+sysinfo = "0.28.4"
 tar = "0.4.38"
 tempfile = "3.5.0"
 thiserror = "1.0.40"
@@ -134,23 +133,12 @@ zip = { version = "0.6.4", optional = true }
 [features]
 default = ["analytics", "meilisearch-types/all-tokenizations", "mini-dashboard"]
 analytics = ["segment"]
-mini-dashboard = [
-    "actix-web-static-files",
-    "static-files",
-    "anyhow",
-    "cargo_toml",
-    "hex",
-    "reqwest",
-    "sha-1",
-    "tempfile",
-    "zip",
-]
+mini-dashboard = ["actix-web-static-files", "static-files", "anyhow", "cargo_toml", "hex", "reqwest", "sha-1", "tempfile", "zip"]
 chinese = ["meilisearch-types/chinese"]
 hebrew = ["meilisearch-types/hebrew"]
 japanese = ["meilisearch-types/japanese"]
 thai = ["meilisearch-types/thai"]
 greek = ["meilisearch-types/greek"]
-khmer = ["meilisearch-types/khmer"]
 
 [package.metadata.mini-dashboard]
 assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.11/build.zip"

meilisearch/src/analytics/mock_analytics.rs

@@ -20,7 +20,7 @@ pub struct SearchAggregator;
 #[allow(dead_code)]
 impl SearchAggregator {
     pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self {
-        Self
+        Self::default()
     }
 
     pub fn succeed(&mut self, _: &dyn Any) {}
@@ -32,7 +32,7 @@ pub struct MultiSearchAggregator;
 #[allow(dead_code)]
 impl MultiSearchAggregator {
     pub fn from_queries(_: &dyn Any, _: &dyn Any) -> Self {
-        Self
+        Self::default()
     }
 
     pub fn succeed(&mut self) {}
@@ -44,7 +44,7 @@ pub struct FacetSearchAggregator;
 #[allow(dead_code)]
 impl FacetSearchAggregator {
     pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self {
-        Self
+        Self::default()
     }
 
     pub fn succeed(&mut self, _: &dyn Any) {}

meilisearch/src/analytics/mod.rs

@@ -1,5 +1,6 @@
 mod mock_analytics;
-#[cfg(feature = "analytics")]
+// if we are in release mode and the feature analytics was enabled
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 mod segment_analytics;
 
 use std::fs;
@@ -16,25 +17,26 @@ use serde_json::Value;
 use crate::routes::indexes::documents::UpdateDocumentsQuery;
 use crate::routes::tasks::TasksFilterQuery;
 
-// if the analytics feature is disabled
+// if we are in debug mode OR the analytics feature is disabled
 // the `SegmentAnalytics` point to the mock instead of the real analytics
-#[cfg(not(feature = "analytics"))]
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type SegmentAnalytics = mock_analytics::MockAnalytics;
-#[cfg(not(feature = "analytics"))]
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type SearchAggregator = mock_analytics::SearchAggregator;
-#[cfg(not(feature = "analytics"))]
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type MultiSearchAggregator = mock_analytics::MultiSearchAggregator;
-#[cfg(not(feature = "analytics"))]
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
 pub type FacetSearchAggregator = mock_analytics::FacetSearchAggregator;
 
-// if the feature analytics is enabled we use the real analytics
-#[cfg(feature = "analytics")]
+// if we are in release mode and the feature analytics was enabled
+// we use the real analytics
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type SegmentAnalytics = segment_analytics::SegmentAnalytics;
-#[cfg(feature = "analytics")]
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type SearchAggregator = segment_analytics::SearchAggregator;
-#[cfg(feature = "analytics")]
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type MultiSearchAggregator = segment_analytics::MultiSearchAggregator;
-#[cfg(feature = "analytics")]
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 pub type FacetSearchAggregator = segment_analytics::FacetSearchAggregator;
 
 /// The Meilisearch config dir:

meilisearch/src/lib.rs

@@ -114,7 +114,10 @@ pub fn create_app(
         .configure(routes::configure)
         .configure(|s| dashboard(s, enable_dashboard));
 
-    let app = app.wrap(middleware::RouteMetrics);
+    let app = app.wrap(actix_web::middleware::Condition::new(
+        opt.experimental_enable_metrics,
+        middleware::RouteMetrics,
+    ));
     app.wrap(
         Cors::default()
             .send_wildcard()
@@ -233,6 +236,7 @@ fn open_or_create_database_unchecked(
             enable_mdb_writemap: opt.experimental_reduce_indexing_memory_usage,
             indexer_config: (&opt.indexer_options).try_into()?,
             autobatching_enabled: true,
+            maximum_number_of_batched_tasks: opt.experimental_limit_batched_tasks,
             max_number_of_tasks: 1_000_000,
             index_growth_amount: byte_unit::Byte::from_str("10GiB").unwrap().get_bytes() as usize,
             index_count: DEFAULT_INDEX_COUNT,

meilisearch/src/middleware.rs

@@ -3,10 +3,8 @@
 use std::future::{ready, Ready};
 
 use actix_web::dev::{self, Service, ServiceRequest, ServiceResponse, Transform};
-use actix_web::web::Data;
 use actix_web::Error;
 use futures_util::future::LocalBoxFuture;
-use index_scheduler::IndexScheduler;
 use prometheus::HistogramTimer;
 
 pub struct RouteMetrics;
@@ -49,27 +47,19 @@ where
 
     fn call(&self, req: ServiceRequest) -> Self::Future {
         let mut histogram_timer: Option<HistogramTimer> = None;
-
-        // calling unwrap here is safe because index scheduler is added to app data while creating actix app.
-        // also, the tests will fail if this is not present.
-        let index_scheduler = req.app_data::<Data<IndexScheduler>>().unwrap();
-        let features = index_scheduler.features();
-
-        if features.check_metrics().is_ok() {
-            let request_path = req.path();
-            let is_registered_resource = req.resource_map().has_resource(request_path);
-            if is_registered_resource {
-                let request_method = req.method().to_string();
-                histogram_timer = Some(
-                    crate::metrics::MEILISEARCH_HTTP_RESPONSE_TIME_SECONDS
-                        .with_label_values(&[&request_method, request_path])
-                        .start_timer(),
-                );
-                crate::metrics::MEILISEARCH_HTTP_REQUESTS_TOTAL
+        let request_path = req.path();
+        let is_registered_resource = req.resource_map().has_resource(request_path);
+        if is_registered_resource {
+            let request_method = req.method().to_string();
+            histogram_timer = Some(
+                crate::metrics::MEILISEARCH_HTTP_RESPONSE_TIME_SECONDS
                     .with_label_values(&[&request_method, request_path])
-                    .inc();
-            }
-        };
+                    .start_timer(),
+            );
+            crate::metrics::MEILISEARCH_HTTP_REQUESTS_TOTAL
+                .with_label_values(&[&request_method, request_path])
+                .inc();
+        }
 
         let fut = self.service.call(req);
 

meilisearch/src/option.rs

@@ -28,7 +28,7 @@ const MEILI_DB_PATH: &str = "MEILI_DB_PATH";
 const MEILI_HTTP_ADDR: &str = "MEILI_HTTP_ADDR";
 const MEILI_MASTER_KEY: &str = "MEILI_MASTER_KEY";
 const MEILI_ENV: &str = "MEILI_ENV";
-#[cfg(feature = "analytics")]
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
 const MEILI_NO_ANALYTICS: &str = "MEILI_NO_ANALYTICS";
 const MEILI_HTTP_PAYLOAD_SIZE_LIMIT: &str = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT";
 const MEILI_SSL_CERT_PATH: &str = "MEILI_SSL_CERT_PATH";
@@ -51,6 +51,7 @@ const MEILI_LOG_LEVEL: &str = "MEILI_LOG_LEVEL";
 const MEILI_EXPERIMENTAL_ENABLE_METRICS: &str = "MEILI_EXPERIMENTAL_ENABLE_METRICS";
 const MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE: &str =
     "MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE";
+const MEILI_EXPERIMENTAL_LIMIT_BATCHED_TASKS: &str = "MEILI_EXPERIMENTAL_LIMIT_BATCHED_TASKS";
 
 const DEFAULT_CONFIG_FILE_PATH: &str = "./config.toml";
 const DEFAULT_DB_PATH: &str = "./data.ms";
@@ -159,7 +160,7 @@ pub struct Opt {
     /// Meilisearch automatically collects data from all instances that do not opt out using this flag.
     /// All gathered data is used solely for the purpose of improving Meilisearch, and can be deleted
     /// at any time.
-    #[cfg(feature = "analytics")]
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
     #[serde(default)] // we can't send true
     #[clap(long, env = MEILI_NO_ANALYTICS)]
     pub no_analytics: bool,
@@ -301,6 +302,11 @@ pub struct Opt {
     #[serde(default)]
     pub experimental_reduce_indexing_memory_usage: bool,
 
+    /// Experimental limit to the number of tasks per batch
+    #[clap(long, env = MEILI_EXPERIMENTAL_LIMIT_BATCHED_TASKS, default_value_t = default_limit_batched_tasks())]
+    #[serde(default = "default_limit_batched_tasks")]
+    pub experimental_limit_batched_tasks: usize,
+
     #[serde(flatten)]
     #[clap(flatten)]
     pub indexer_options: IndexerOpts,
@@ -390,10 +396,11 @@ impl Opt {
             ignore_missing_dump: _,
             ignore_dump_if_db_exists: _,
             config_file_path: _,
-            #[cfg(feature = "analytics")]
+            #[cfg(all(not(debug_assertions), feature = "analytics"))]
             no_analytics,
             experimental_enable_metrics: enable_metrics_route,
-            experimental_reduce_indexing_memory_usage: reduce_indexing_memory_usage,
+            experimental_reduce_indexing_memory_usage,
+            experimental_limit_batched_tasks,
         } = self;
         export_to_env_if_not_present(MEILI_DB_PATH, db_path);
         export_to_env_if_not_present(MEILI_HTTP_ADDR, http_addr);
@@ -401,7 +408,7 @@ impl Opt {
             export_to_env_if_not_present(MEILI_MASTER_KEY, master_key);
         }
         export_to_env_if_not_present(MEILI_ENV, env);
-        #[cfg(feature = "analytics")]
+        #[cfg(all(not(debug_assertions), feature = "analytics"))]
         {
             export_to_env_if_not_present(MEILI_NO_ANALYTICS, no_analytics.to_string());
         }
@@ -437,7 +444,11 @@ impl Opt {
         );
         export_to_env_if_not_present(
             MEILI_EXPERIMENTAL_REDUCE_INDEXING_MEMORY_USAGE,
-            reduce_indexing_memory_usage.to_string(),
+            experimental_reduce_indexing_memory_usage.to_string(),
+        );
+        export_to_env_if_not_present(
+            MEILI_EXPERIMENTAL_LIMIT_BATCHED_TASKS,
+            experimental_limit_batched_tasks.to_string(),
         );
         indexer_options.export_to_env();
     }
@@ -739,6 +750,10 @@ fn default_dump_dir() -> PathBuf {
     PathBuf::from(DEFAULT_DUMP_DIR)
 }
 
+fn default_limit_batched_tasks() -> usize {
+    usize::MAX
+}
+
 /// Indicates if a snapshot was scheduled, and if yes with which interval.
 #[derive(Debug, Default, Copy, Clone, Deserialize, Serialize)]
 pub enum ScheduleSnapshot {

meilisearch/src/routes/features.rs

@@ -29,12 +29,12 @@ async fn get_features(
     >,
     req: HttpRequest,
     analytics: Data<dyn Analytics>,
-) -> HttpResponse {
-    let features = index_scheduler.features();
+) -> Result<HttpResponse, ResponseError> {
+    let features = index_scheduler.features()?;
 
     analytics.publish("Experimental features Seen".to_string(), json!(null), Some(&req));
     debug!("returns: {:?}", features.runtime_features());
-    HttpResponse::Ok().json(features.runtime_features())
+    Ok(HttpResponse::Ok().json(features.runtime_features()))
 }
 
 #[derive(Debug, Deserr)]
@@ -44,10 +44,6 @@ pub struct RuntimeTogglableFeatures {
     pub score_details: Option<bool>,
     #[deserr(default)]
     pub vector_store: Option<bool>,
-    #[deserr(default)]
-    pub metrics: Option<bool>,
-    #[deserr(default)]
-    pub export_puffin_reports: Option<bool>,
 }
 
 async fn patch_features(
@@ -59,36 +55,26 @@ async fn patch_features(
     req: HttpRequest,
     analytics: Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    let features = index_scheduler.features();
+    let features = index_scheduler.features()?;
 
     let old_features = features.runtime_features();
+
     let new_features = meilisearch_types::features::RuntimeTogglableFeatures {
         score_details: new_features.0.score_details.unwrap_or(old_features.score_details),
         vector_store: new_features.0.vector_store.unwrap_or(old_features.vector_store),
-        metrics: new_features.0.metrics.unwrap_or(old_features.metrics),
-        export_puffin_reports: new_features
-            .0
-            .export_puffin_reports
-            .unwrap_or(old_features.export_puffin_reports),
     };
 
     // explicitly destructure for analytics rather than using the `Serialize` implementation, because
     // the it renames to camelCase, which we don't want for analytics.
     // **Do not** ignore fields with `..` or `_` here, because we want to add them in the future.
-    let meilisearch_types::features::RuntimeTogglableFeatures {
-        score_details,
-        vector_store,
-        metrics,
-        export_puffin_reports,
-    } = new_features;
+    let meilisearch_types::features::RuntimeTogglableFeatures { score_details, vector_store } =
+        new_features;
 
     analytics.publish(
         "Experimental features Updated".to_string(),
         json!({
             "score_details": score_details,
             "vector_store": vector_store,
-            "metrics": metrics,
-            "export_puffin_reports": export_puffin_reports,
         }),
         Some(&req),
     );

meilisearch/src/routes/indexes/facet_search.rs

@@ -68,7 +68,7 @@ pub async fn search(
     }
 
     let index = index_scheduler.index(&index_uid)?;
-    let features = index_scheduler.features();
+    let features = index_scheduler.features()?;
     let search_result = tokio::task::spawn_blocking(move || {
         perform_facet_search(&index, search_query, facet_query, facet_name, features)
     })

meilisearch/src/routes/indexes/search.rs

@@ -157,7 +157,7 @@ pub async fn search_with_url_query(
     let mut aggregate = SearchAggregator::from_query(&query, &req);
 
     let index = index_scheduler.index(&index_uid)?;
-    let features = index_scheduler.features();
+    let features = index_scheduler.features()?;
     let search_result =
         tokio::task::spawn_blocking(move || perform_search(&index, query, features)).await?;
     if let Ok(ref search_result) = search_result {
@@ -192,7 +192,7 @@ pub async fn search_with_post(
 
     let index = index_scheduler.index(&index_uid)?;
 
-    let features = index_scheduler.features();
+    let features = index_scheduler.features()?;
     let search_result =
         tokio::task::spawn_blocking(move || perform_search(&index, query, features)).await?;
     if let Ok(ref search_result) = search_result {

meilisearch/src/routes/indexes/settings.rs

@@ -310,81 +310,6 @@ make_setting_route!(
     }
 );
 
-make_setting_route!(
-    "/non-separator-tokens",
-    put,
-    std::collections::BTreeSet<String>,
-    meilisearch_types::deserr::DeserrJsonError<
-        meilisearch_types::error::deserr_codes::InvalidSettingsNonSeparatorTokens,
-    >,
-    non_separator_tokens,
-    "nonSeparatorTokens",
-    analytics,
-    |non_separator_tokens: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
-        use serde_json::json;
-
-        analytics.publish(
-            "nonSeparatorTokens Updated".to_string(),
-            json!({
-                "non_separator_tokens": {
-                    "total": non_separator_tokens.as_ref().map(|non_separator_tokens| non_separator_tokens.len()),
-                },
-            }),
-            Some(req),
-        );
-    }
-);
-
-make_setting_route!(
-    "/separator-tokens",
-    put,
-    std::collections::BTreeSet<String>,
-    meilisearch_types::deserr::DeserrJsonError<
-        meilisearch_types::error::deserr_codes::InvalidSettingsSeparatorTokens,
-    >,
-    separator_tokens,
-    "separatorTokens",
-    analytics,
-    |separator_tokens: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
-        use serde_json::json;
-
-        analytics.publish(
-            "separatorTokens Updated".to_string(),
-            json!({
-                "separator_tokens": {
-                    "total": separator_tokens.as_ref().map(|separator_tokens| separator_tokens.len()),
-                },
-            }),
-            Some(req),
-        );
-    }
-);
-
-make_setting_route!(
-    "/dictionary",
-    put,
-    std::collections::BTreeSet<String>,
-    meilisearch_types::deserr::DeserrJsonError<
-        meilisearch_types::error::deserr_codes::InvalidSettingsDictionary,
-    >,
-    dictionary,
-    "dictionary",
-    analytics,
-    |dictionary: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
-        use serde_json::json;
-
-        analytics.publish(
-            "dictionary Updated".to_string(),
-            json!({
-                "dictionary": {
-                    "total": dictionary.as_ref().map(|dictionary| dictionary.len()),
-                },
-            }),
-            Some(req),
-        );
-    }
-);
-
 make_setting_route!(
     "/synonyms",
     put,
@@ -541,9 +466,6 @@ generate_configure!(
     searchable_attributes,
     distinct_attribute,
     stop_words,
-    separator_tokens,
-    non_separator_tokens,
-    dictionary,
     synonyms,
     ranking_rules,
     typo_tolerance,

meilisearch/src/routes/metrics.rs

@@ -19,7 +19,7 @@ pub async fn get_metrics(
     index_scheduler: GuardedData<ActionPolicy<{ actions::METRICS_GET }>, Data<IndexScheduler>>,
     auth_controller: Data<AuthController>,
 ) -> Result<HttpResponse, ResponseError> {
-    index_scheduler.features().check_metrics()?;
+    index_scheduler.features()?.check_metrics()?;
     let auth_filters = index_scheduler.filters();
     if !auth_filters.all_indexes_authorized() {
         let mut error = ResponseError::from(AuthenticationError::InvalidToken);

meilisearch/src/routes/mod.rs

@@ -24,7 +24,6 @@ pub mod features;
 pub mod indexes;
 mod metrics;
 mod multi_search;
-mod snapshot;
 mod swap_indexes;
 pub mod tasks;
 
@@ -33,7 +32,6 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
         .service(web::resource("/health").route(web::get().to(get_health)))
         .service(web::scope("/keys").configure(api_key::configure))
         .service(web::scope("/dumps").configure(dump::configure))
-        .service(web::scope("/snapshots").configure(snapshot::configure))
         .service(web::resource("/stats").route(web::get().to(get_stats)))
         .service(web::resource("/version").route(web::get().to(get_version)))
         .service(web::scope("/indexes").configure(indexes::configure))

meilisearch/src/routes/multi_search.rs

@@ -41,7 +41,7 @@ pub async fn multi_search_with_post(
     let queries = params.into_inner().queries;
 
     let mut multi_aggregate = MultiSearchAggregator::from_queries(&queries, &req);
-    let features = index_scheduler.features();
+    let features = index_scheduler.features()?;
 
     // Explicitly expect a `(ResponseError, usize)` for the error type rather than `ResponseError` only,
     // so that `?` doesn't work if it doesn't use `with_index`, ensuring that it is not forgotten in case of code

meilisearch/src/routes/snapshot.rs

@@ -1,32 +0,0 @@
-use actix_web::web::Data;
-use actix_web::{web, HttpRequest, HttpResponse};
-use index_scheduler::IndexScheduler;
-use log::debug;
-use meilisearch_types::error::ResponseError;
-use meilisearch_types::tasks::KindWithContent;
-use serde_json::json;
-
-use crate::analytics::Analytics;
-use crate::extractors::authentication::policies::*;
-use crate::extractors::authentication::GuardedData;
-use crate::extractors::sequential_extractor::SeqHandler;
-use crate::routes::SummarizedTaskView;
-
-pub fn configure(cfg: &mut web::ServiceConfig) {
-    cfg.service(web::resource("").route(web::post().to(SeqHandler(create_snapshot))));
-}
-
-pub async fn create_snapshot(
-    index_scheduler: GuardedData<ActionPolicy<{ actions::SNAPSHOTS_CREATE }>, Data<IndexScheduler>>,
-    req: HttpRequest,
-    analytics: web::Data<dyn Analytics>,
-) -> Result<HttpResponse, ResponseError> {
-    analytics.publish("Snapshot Created".to_string(), json!({}), Some(&req));
-
-    let task = KindWithContent::SnapshotCreation;
-    let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
-
-    debug!("returns: {:?}", task);
-    Ok(HttpResponse::Accepted().json(task))
-}

meilisearch/src/routes/swap_indexes.rs

@@ -60,7 +60,8 @@ pub async fn swap_indexes(
     }
 
     let task = KindWithContent::IndexSwap { swaps };
-    let task: SummarizedTaskView =
-        tokio::task::spawn_blocking(move || index_scheduler.register(task)).await??.into();
+
+    let task = index_scheduler.register(task)?;
+    let task: SummarizedTaskView = task.into();
     Ok(HttpResponse::Accepted().json(task))
 }

meilisearch/src/search.rs

@@ -491,20 +491,6 @@ pub fn perform_search(
         tokenizer_builder.allow_list(&script_lang_map);
     }
 
-    let separators = index.allowed_separators(&rtxn)?;
-    let separators: Option<Vec<_>> =
-        separators.as_ref().map(|x| x.iter().map(String::as_str).collect());
-    if let Some(ref separators) = separators {
-        tokenizer_builder.separators(separators);
-    }
-
-    let dictionary = index.dictionary(&rtxn)?;
-    let dictionary: Option<Vec<_>> =
-        dictionary.as_ref().map(|x| x.iter().map(String::as_str).collect());
-    if let Some(ref dictionary) = dictionary {
-        tokenizer_builder.words_dict(dictionary);
-    }
-
     let mut formatter_builder = MatcherBuilder::new(matching_words, tokenizer_builder.build());
     formatter_builder.crop_marker(query.crop_marker);
     formatter_builder.highlight_prefix(query.highlight_pre_tag);

meilisearch/tests/auth/api_keys.rs

@@ -1,7 +1,8 @@
 use std::{thread, time};
 
-use crate::common::{Server, Value};
-use crate::json;
+use serde_json::{json, Value};
+
+use crate::common::Server;
 
 #[actix_rt::test]
 async fn add_valid_api_key() {
@@ -161,7 +162,7 @@ async fn add_valid_api_key_null_description() {
     server.use_api_key("MASTER_KEY");
 
     let content = json!({
-        "description": json!(null),
+        "description": Value::Null,
         "indexes": ["products"],
         "actions": ["documents.add"],
         "expiresAt": "2050-11-13T00:00:00"
@@ -364,7 +365,7 @@ async fn error_add_api_key_invalid_index_uids() {
     server.use_api_key("MASTER_KEY");
 
     let content = json!({
-        "description": json!(null),
+        "description": Value::Null,
         "indexes": ["invalid index # / \\name with spaces"],
         "actions": [
             "documents.add"
@@ -421,7 +422,7 @@ async fn error_add_api_key_invalid_parameters_actions() {
     meili_snap::snapshot!(code, @"400 Bad Request");
     meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]" }), @r###"
     {
-      "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
+      "message": "Unknown value `doc.add` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
       "code": "invalid_api_key_actions",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions"
@@ -506,7 +507,7 @@ async fn error_add_api_key_invalid_parameters_uid() {
 async fn error_add_api_key_parameters_uid_already_exist() {
     let mut server = Server::new_auth().await;
     server.use_api_key("MASTER_KEY");
-    let content: Value = json!({
+    let content = json!({
         "uid": "4bc0887a-0e41-4f3b-935d-0c451dcee9c8",
         "indexes": ["products"],
         "actions": ["search"],
@@ -1145,7 +1146,7 @@ async fn patch_api_key_description() {
     meili_snap::snapshot!(code, @"200 OK");
 
     // Remove the description
-    let content = json!({ "description": null });
+    let content = json!({ "description": serde_json::Value::Null });
 
     let (response, code) = server.patch_api_key(&uid, content).await;
     meili_snap::snapshot!(meili_snap::json_string!(response, { ".createdAt" => "[ignored]", ".updatedAt" => "[ignored]", ".uid" => "[ignored]", ".key" => "[ignored]" }), @r###"

meilisearch/tests/auth/authorization.rs

@@ -2,13 +2,11 @@ use std::collections::{HashMap, HashSet};
 
 use ::time::format_description::well_known::Rfc3339;
 use maplit::{hashmap, hashset};
-use meilisearch::Opt;
 use once_cell::sync::Lazy;
-use tempfile::TempDir;
+use serde_json::{json, Value};
 use time::{Duration, OffsetDateTime};
 
-use crate::common::{default_settings, Server, Value};
-use crate::json;
+use crate::common::Server;
 
 pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'static str>>> =
     Lazy::new(|| {
@@ -56,7 +54,6 @@ pub static AUTHORIZATIONS: Lazy<HashMap<(&'static str, &'static str), HashSet<&'
             ("GET",     "/indexes/products/stats") =>                          hashset!{"stats.get", "stats.*", "*"},
             ("GET",     "/stats") =>                                           hashset!{"stats.get", "stats.*", "*"},
             ("POST",    "/dumps") =>                                           hashset!{"dumps.create", "dumps.*", "*"},
-            ("POST",    "/snapshots") =>                                       hashset!{"snapshots.create", "snapshots.*", "*"},
             ("GET",     "/version") =>                                         hashset!{"version", "*"},
             ("GET",     "/metrics") =>                                         hashset!{"metrics.get", "metrics.*", "*"},
             ("PATCH",   "/keys/mykey/") =>                                     hashset!{"keys.update", "*"},
@@ -197,9 +194,7 @@ async fn access_authorized_master_key() {
 
 #[actix_rt::test]
 async fn access_authorized_restricted_index() {
-    let dir = TempDir::new().unwrap();
-    let enable_metrics = Opt { experimental_enable_metrics: true, ..default_settings(dir.path()) };
-    let mut server = Server::new_auth_with_options(enable_metrics, dir).await;
+    let mut server = Server::new_auth().await;
     for ((method, route), actions) in AUTHORIZATIONS.iter() {
         for action in actions {
             // create a new API key letting only the needed action.

meilisearch/tests/auth/errors.rs

@@ -1,8 +1,8 @@
 use meili_snap::*;
+use serde_json::json;
 use uuid::Uuid;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn create_api_key_bad_description() {
@@ -90,7 +90,7 @@ async fn create_api_key_bad_actions() {
     snapshot!(code, @"400 Bad Request");
     snapshot!(json_string!(response), @r###"
     {
-      "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `snapshots.*`, `snapshots.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
+      "message": "Unknown value `doggo` at `.actions[0]`: expected one of `*`, `search`, `documents.*`, `documents.add`, `documents.get`, `documents.delete`, `indexes.*`, `indexes.create`, `indexes.get`, `indexes.update`, `indexes.delete`, `indexes.swap`, `tasks.*`, `tasks.cancel`, `tasks.delete`, `tasks.get`, `settings.*`, `settings.get`, `settings.update`, `stats.*`, `stats.get`, `metrics.*`, `metrics.get`, `dumps.*`, `dumps.create`, `version`, `keys.create`, `keys.get`, `keys.update`, `keys.delete`, `experimental.get`, `experimental.update`",
       "code": "invalid_api_key_actions",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#invalid_api_key_actions"

meilisearch/tests/auth/mod.rs

@@ -7,9 +7,9 @@ mod tenant_token;
 mod tenant_token_multi_search;
 
 use actix_web::http::StatusCode;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 impl Server {
     pub fn use_api_key(&mut self, api_key: impl AsRef<str>) {

meilisearch/tests/auth/tenant_token.rs

@@ -3,11 +3,11 @@ use std::collections::HashMap;
 use ::time::format_description::well_known::Rfc3339;
 use maplit::hashmap;
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 use time::{Duration, OffsetDateTime};
 
 use super::authorization::{ALL_ACTIONS, AUTHORIZATIONS};
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 fn generate_tenant_token(
     parent_uid: impl AsRef<str>,
@@ -233,31 +233,31 @@ async fn search_authorized_simple_token() {
         },
         hashmap! {
             "searchRules" => json!({"*": {}}),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
-            "exp" => json!(null)
+            "searchRules" => json!({"*": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["*"]),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
-            "exp" => json!(null)
+            "searchRules" => json!({"sales": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sales"]),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sa*"]),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
     ];
 
@@ -386,7 +386,7 @@ async fn error_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -398,7 +398,7 @@ async fn error_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
+            "searchRules" => json!({"sales": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -428,15 +428,15 @@ async fn error_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"products": {}}),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"products": null}),
-            "exp" => json!(null)
+            "searchRules" => json!({"products": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["products"]),
-            "exp" => json!(null)
+            "exp" => Value::Null
         },
         // expired token
         hashmap! {
@@ -444,7 +444,7 @@ async fn error_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -456,7 +456,7 @@ async fn error_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
+            "searchRules" => json!({"sales": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {

meilisearch/tests/auth/tenant_token_multi_search.rs

@@ -3,11 +3,11 @@ use std::collections::HashMap;
 use ::time::format_description::well_known::Rfc3339;
 use maplit::hashmap;
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 use time::{Duration, OffsetDateTime};
 
 use super::authorization::ALL_ACTIONS;
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 fn generate_tenant_token(
     parent_uid: impl AsRef<str>,
@@ -512,31 +512,31 @@ async fn single_search_authorized_simple_token() {
         },
         hashmap! {
             "searchRules" => json!({"*": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"*": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["*"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"sales": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sales"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sa*"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
     ];
 
@@ -564,31 +564,31 @@ async fn multi_search_authorized_simple_token() {
         },
         hashmap! {
             "searchRules" => json!({"*": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"*": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["*"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!({"sales": {}, "products": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"sales": null, "products": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"sales": Value::Null, "products": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sales", "products"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sa*", "pro*"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
     ];
 
@@ -823,7 +823,7 @@ async fn error_single_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -835,7 +835,7 @@ async fn error_single_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
+            "searchRules" => json!({"sales": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -864,7 +864,7 @@ async fn error_multi_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -876,7 +876,7 @@ async fn error_multi_search_token_forbidden_parent_key() {
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null, "products": null}),
+            "searchRules" => json!({"sales": Value::Null, "products": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() + Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -919,15 +919,15 @@ async fn error_single_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"products": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"products": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"products": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["products"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         // expired token
         hashmap! {
@@ -935,7 +935,7 @@ async fn error_single_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -947,7 +947,7 @@ async fn error_single_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
+            "searchRules" => json!({"sales": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -978,15 +978,15 @@ async fn error_multi_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"products": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"products": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"products": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["products"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
@@ -998,15 +998,15 @@ async fn error_multi_search_forbidden_token() {
         },
         hashmap! {
             "searchRules" => json!({"sales": {}}),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         hashmap! {
-            "searchRules" => json!({"sales": null}),
-            "exp" => json!(null),
+            "searchRules" => json!({"sales": Value::Null}),
+            "exp" => Value::Null
         },
         hashmap! {
             "searchRules" => json!(["sales"]),
-            "exp" => json!(null),
+            "exp" => Value::Null
         },
         // expired token
         hashmap! {
@@ -1014,7 +1014,7 @@ async fn error_multi_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"*": null}),
+            "searchRules" => json!({"*": Value::Null}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
@@ -1026,7 +1026,7 @@ async fn error_multi_search_forbidden_token() {
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {
-            "searchRules" => json!({"sales": null, "products": {}}),
+            "searchRules" => json!({"sales": Value::Null, "products": {}}),
             "exp" => json!((OffsetDateTime::now_utc() - Duration::hours(1)).unix_timestamp())
         },
         hashmap! {

meilisearch/tests/common/index.rs

@@ -3,13 +3,12 @@ use std::panic::{catch_unwind, resume_unwind, UnwindSafe};
 use std::time::Duration;
 
 use actix_web::http::StatusCode;
+use serde_json::{json, Value};
 use tokio::time::sleep;
 use urlencoding::encode as urlencode;
 
 use super::encoder::Encoder;
 use super::service::Service;
-use super::Value;
-use crate::json;
 
 pub struct Index<'a> {
     pub uid: String,
@@ -243,9 +242,7 @@ impl Index<'_> {
 
     pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
         let url = format!("/indexes/{}/documents/delete-batch", urlencode(self.uid.as_ref()));
-        self.service
-            .post_encoded(url, serde_json::to_value(&ids).unwrap().into(), self.encoder)
-            .await
+        self.service.post_encoded(url, serde_json::to_value(&ids).unwrap(), self.encoder).await
     }
 
     pub async fn delete_batch_raw(&self, body: Value) -> (Value, StatusCode) {

meilisearch/tests/common/mod.rs

@@ -3,85 +3,9 @@ pub mod index;
 pub mod server;
 pub mod service;
 
-use std::fmt::{self, Display};
-
-#[allow(unused)]
 pub use index::{GetAllDocumentsOptions, GetDocumentOptions};
-use meili_snap::json_string;
-use serde::{Deserialize, Serialize};
-#[allow(unused)]
 pub use server::{default_settings, Server};
 
-#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)]
-pub struct Value(pub serde_json::Value);
-
-impl Value {
-    pub fn uid(&self) -> u64 {
-        if let Some(uid) = self["uid"].as_u64() {
-            uid
-        } else if let Some(uid) = self["taskUid"].as_u64() {
-            uid
-        } else {
-            panic!("Didn't find any task id in: {self}");
-        }
-    }
-}
-
-impl From<serde_json::Value> for Value {
-    fn from(value: serde_json::Value) -> Self {
-        Value(value)
-    }
-}
-
-impl std::ops::Deref for Value {
-    type Target = serde_json::Value;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl PartialEq<serde_json::Value> for Value {
-    fn eq(&self, other: &serde_json::Value) -> bool {
-        &self.0 == other
-    }
-}
-
-impl PartialEq<Value> for serde_json::Value {
-    fn eq(&self, other: &Value) -> bool {
-        self == &other.0
-    }
-}
-
-impl PartialEq<&str> for Value {
-    fn eq(&self, other: &&str) -> bool {
-        self.0.eq(other)
-    }
-}
-
-impl Display for Value {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        write!(
-            f,
-            "{}",
-            json_string!(self, { ".enqueuedAt" => "[date]", ".processedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" })
-        )
-    }
-}
-
-impl From<Vec<Value>> for Value {
-    fn from(value: Vec<Value>) -> Self {
-        Self(value.into_iter().map(|value| value.0).collect::<serde_json::Value>())
-    }
-}
-
-#[macro_export]
-macro_rules! json {
-    ($($json:tt)+) => {
-        $crate::common::Value(serde_json::json!($($json)+))
-    };
-}
-
 /// Performs a search test on both post and get routes
 #[macro_export]
 macro_rules! test_post_get_search {

meilisearch/tests/common/server.rs

@@ -11,14 +11,13 @@ use clap::Parser;
 use meilisearch::option::{IndexerOpts, MaxMemory, Opt};
 use meilisearch::{analytics, create_app, setup_meilisearch};
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 use tempfile::TempDir;
 use tokio::time::sleep;
 
 use super::index::Index;
 use super::service::Service;
 use crate::common::encoder::Encoder;
-use crate::common::Value;
-use crate::json;
 
 pub struct Server {
     pub service: Service,
@@ -157,10 +156,6 @@ impl Server {
         self.service.post("/dumps", json!(null)).await
     }
 
-    pub async fn create_snapshot(&self) -> (Value, StatusCode) {
-        self.service.post("/snapshots", json!(null)).await
-    }
-
     pub async fn index_swap(&self, value: Value) -> (Value, StatusCode) {
         self.service.post("/swap-indexes", value).await
     }
@@ -202,10 +197,6 @@ impl Server {
     pub async fn set_features(&self, value: Value) -> (Value, StatusCode) {
         self.service.patch("/experimental-features", value).await
     }
-
-    pub async fn get_metrics(&self) -> (Value, StatusCode) {
-        self.service.get("/metrics").await
-    }
 }
 
 pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
@@ -213,7 +204,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
         db_path: dir.as_ref().join("db"),
         dump_dir: dir.as_ref().join("dumps"),
         env: "development".to_owned(),
-        #[cfg(feature = "analytics")]
+        #[cfg(all(not(debug_assertions), feature = "analytics"))]
         no_analytics: true,
         max_index_size: Byte::from_unit(100.0, ByteUnit::MiB).unwrap(),
         max_task_db_size: Byte::from_unit(1.0, ByteUnit::GiB).unwrap(),
@@ -225,7 +216,7 @@ pub fn default_settings(dir: impl AsRef<Path>) -> Opt {
             skip_index_budget: true,
             ..Parser::parse_from(None as Option<&str>)
         },
-        experimental_enable_metrics: false,
+        experimental_enable_metrics: true,
         ..Parser::parse_from(None as Option<&str>)
     }
 }

meilisearch/tests/common/service.rs

@@ -7,9 +7,9 @@ use actix_web::test::TestRequest;
 use index_scheduler::IndexScheduler;
 use meilisearch::{analytics, create_app, Opt};
 use meilisearch_auth::AuthController;
+use serde_json::Value;
 
 use crate::common::encoder::Encoder;
-use crate::common::Value;
 
 pub struct Service {
     pub index_scheduler: Arc<IndexScheduler>,

meilisearch/tests/content_type.rs

@@ -3,8 +3,9 @@
 mod common;
 
 use actix_web::test;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
+use crate::common::Server;
 
 enum HttpVerb {
     Put,

meilisearch/tests/documents/add_documents.rs

@@ -1,11 +1,11 @@
 use actix_web::test;
 use meili_snap::{json_string, snapshot};
+use serde_json::{json, Value};
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::encoder::Encoder;
-use crate::common::{GetAllDocumentsOptions, Server, Value};
-use crate::json;
+use crate::common::{GetAllDocumentsOptions, Server};
 
 /// This is the basic usage of our API and every other tests uses the content-type application/json
 #[actix_rt::test]

meilisearch/tests/documents/delete_documents.rs

@@ -1,7 +1,7 @@
 use meili_snap::{json_string, snapshot};
+use serde_json::json;
 
 use crate::common::{GetAllDocumentsOptions, Server};
-use crate::json;
 
 #[actix_rt::test]
 async fn delete_one_document_unexisting_index() {

meilisearch/tests/documents/errors.rs

@@ -1,8 +1,8 @@
 use meili_snap::*;
+use serde_json::json;
 use urlencoding::encode;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn get_all_documents_bad_offset() {

meilisearch/tests/documents/get_documents.rs

@@ -1,11 +1,11 @@
 use actix_web::test;
 use http::header::ACCEPT_ENCODING;
 use meili_snap::*;
+use serde_json::{json, Value};
 use urlencoding::encode as urlencode;
 
 use crate::common::encoder::Encoder;
-use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server, Value};
-use crate::json;
+use crate::common::{GetAllDocumentsOptions, GetDocumentOptions, Server};
 
 // TODO: partial test since we are testing error, amd error is not yet fully implemented in
 // transplant
@@ -40,7 +40,7 @@ async fn get_document() {
     let server = Server::new().await;
     let index = server.index("test");
     index.create(None).await;
-    let documents = json!([
+    let documents = serde_json::json!([
         {
             "id": 0,
             "nested": { "content": "foobar" },
@@ -53,7 +53,7 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        json!({
+        serde_json::json!({
             "id": 0,
             "nested": { "content": "foobar" },
         })
@@ -64,7 +64,7 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        json!({
+        serde_json::json!({
             "id": 0,
         })
     );
@@ -75,7 +75,7 @@ async fn get_document() {
     assert_eq!(code, 200);
     assert_eq!(
         response,
-        json!({
+        serde_json::json!({
             "nested": { "content": "foobar" },
         })
     );
@@ -122,7 +122,7 @@ async fn get_all_documents_no_options() {
     assert_eq!(code, 200);
     let arr = response["results"].as_array().unwrap();
     assert_eq!(arr.len(), 20);
-    let first = json!({
+    let first = serde_json::json!({
         "id":0,
         "isActive":false,
         "balance":"$2,668.55",

meilisearch/tests/documents/update_documents.rs

@@ -1,8 +1,7 @@
-use meili_snap::snapshot;
+use serde_json::json;
 
 use crate::common::encoder::Encoder;
 use crate::common::{GetAllDocumentsOptions, Server};
-use crate::json;
 
 #[actix_rt::test]
 async fn error_document_update_create_index_bad_uid() {
@@ -85,13 +84,7 @@ async fn update_document() {
 
     let (response, code) = index.get_document(1, None).await;
     assert_eq!(code, 200);
-    snapshot!(response, @r###"
-    {
-      "doc_id": 1,
-      "content": "foo",
-      "other": "bar"
-    }
-    "###);
+    assert_eq!(response.to_string(), r##"{"doc_id":1,"content":"foo","other":"bar"}"##);
 }
 
 #[actix_rt::test]
@@ -129,13 +122,7 @@ async fn update_document_gzip_encoded() {
 
     let (response, code) = index.get_document(1, None).await;
     assert_eq!(code, 200);
-    snapshot!(response, @r###"
-    {
-      "doc_id": 1,
-      "content": "foo",
-      "other": "bar"
-    }
-    "###);
+    assert_eq!(response.to_string(), r##"{"doc_id":1,"content":"foo","other":"bar"}"##);
 }
 
 #[actix_rt::test]

meilisearch/tests/features/mod.rs

@@ -1,8 +1,6 @@
-use meilisearch::Opt;
-use tempfile::TempDir;
+use serde_json::json;
 
-use crate::common::{default_settings, Server};
-use crate::json;
+use crate::common::Server;
 
 /// Feature name to test against.
 /// This will have to be changed by a different one when that feature is stabilized.
@@ -19,9 +17,7 @@ async fn experimental_features() {
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
       "scoreDetails": false,
-      "vectorStore": false,
-      "metrics": false,
-      "exportPuffinReports": false
+      "vectorStore": false
     }
     "###);
 
@@ -31,9 +27,7 @@ async fn experimental_features() {
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
       "scoreDetails": false,
-      "vectorStore": true,
-      "metrics": false,
-      "exportPuffinReports": false
+      "vectorStore": true
     }
     "###);
 
@@ -43,9 +37,7 @@ async fn experimental_features() {
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
       "scoreDetails": false,
-      "vectorStore": true,
-      "metrics": false,
-      "exportPuffinReports": false
+      "vectorStore": true
     }
     "###);
 
@@ -56,9 +48,7 @@ async fn experimental_features() {
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
       "scoreDetails": false,
-      "vectorStore": true,
-      "metrics": false,
-      "exportPuffinReports": false
+      "vectorStore": true
     }
     "###);
 
@@ -69,73 +59,11 @@ async fn experimental_features() {
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
       "scoreDetails": false,
-      "vectorStore": true,
-      "metrics": false,
-      "exportPuffinReports": false
+      "vectorStore": true
     }
     "###);
 }
 
-#[actix_rt::test]
-async fn experimental_feature_metrics() {
-    // instance flag for metrics enables metrics at startup
-    let dir = TempDir::new().unwrap();
-    let enable_metrics = Opt { experimental_enable_metrics: true, ..default_settings(dir.path()) };
-    let server = Server::new_with_options(enable_metrics).await.unwrap();
-
-    let (response, code) = server.get_features().await;
-
-    meili_snap::snapshot!(code, @"200 OK");
-    meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
-    {
-      "scoreDetails": false,
-      "vectorStore": false,
-      "metrics": true,
-      "exportPuffinReports": false
-    }
-    "###);
-
-    let (response, code) = server.get_metrics().await;
-    meili_snap::snapshot!(code, @"200 OK");
-
-    // metrics are not returned in json format
-    // so the test server will return null
-    meili_snap::snapshot!(response, @"null");
-
-    // disabling metrics results in invalid request
-    let (response, code) = server.set_features(json!({"metrics": false})).await;
-    meili_snap::snapshot!(code, @"200 OK");
-    meili_snap::snapshot!(response["metrics"], @"false");
-
-    let (response, code) = server.get_metrics().await;
-    meili_snap::snapshot!(code, @"400 Bad Request");
-    meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
-    {
-      "message": "Getting metrics requires enabling the `metrics` experimental feature. See https://github.com/meilisearch/product/discussions/625",
-      "code": "feature_not_enabled",
-      "type": "invalid_request",
-      "link": "https://docs.meilisearch.com/errors#feature_not_enabled"
-    }
-    "###);
-
-    // enabling metrics via HTTP results in valid request
-    let (response, code) = server.set_features(json!({"metrics": true})).await;
-    meili_snap::snapshot!(code, @"200 OK");
-    meili_snap::snapshot!(response["metrics"], @"true");
-
-    let (response, code) = server.get_metrics().await;
-    meili_snap::snapshot!(code, @"200 OK");
-    meili_snap::snapshot!(response, @"null");
-
-    // startup without flag respects persisted metrics value
-    let disable_metrics =
-        Opt { experimental_enable_metrics: false, ..default_settings(dir.path()) };
-    let server_no_flag = Server::new_with_options(disable_metrics).await.unwrap();
-    let (response, code) = server_no_flag.get_metrics().await;
-    meili_snap::snapshot!(code, @"200 OK");
-    meili_snap::snapshot!(response, @"null");
-}
-
 #[actix_rt::test]
 async fn errors() {
     let server = Server::new().await;
@@ -146,7 +74,7 @@ async fn errors() {
     meili_snap::snapshot!(code, @"400 Bad Request");
     meili_snap::snapshot!(meili_snap::json_string!(response), @r###"
     {
-      "message": "Unknown field `NotAFeature`: expected one of `scoreDetails`, `vectorStore`, `metrics`, `exportPuffinReports`",
+      "message": "Unknown field `NotAFeature`: expected one of `scoreDetails`, `vectorStore`",
       "code": "bad_request",
       "type": "invalid_request",
       "link": "https://docs.meilisearch.com/errors#bad_request"

meilisearch/tests/index/create_index.rs

@@ -2,10 +2,10 @@ use actix_web::http::header::ContentType;
 use actix_web::test;
 use http::header::ACCEPT_ENCODING;
 use meili_snap::{json_string, snapshot};
+use serde_json::{json, Value};
 
 use crate::common::encoder::Encoder;
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 #[actix_rt::test]
 async fn create_index_no_primary_key() {
@@ -21,7 +21,7 @@ async fn create_index_no_primary_key() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], json!(null));
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -38,7 +38,7 @@ async fn create_index_with_gzip_encoded_request() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], json!(null));
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -86,7 +86,7 @@ async fn create_index_with_zlib_encoded_request() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], json!(null));
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -103,7 +103,7 @@ async fn create_index_with_brotli_encoded_request() {
 
     assert_eq!(response["status"], "succeeded");
     assert_eq!(response["type"], "indexCreation");
-    assert_eq!(response["details"]["primaryKey"], json!(null));
+    assert_eq!(response["details"]["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -136,7 +136,7 @@ async fn create_index_with_invalid_primary_key() {
 
     let (response, code) = index.get().await;
     assert_eq!(code, 200);
-    assert_eq!(response["primaryKey"], json!(null));
+    assert_eq!(response["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]

meilisearch/tests/index/delete_index.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn create_and_delete_index() {

meilisearch/tests/index/errors.rs

@@ -1,7 +1,7 @@
 use meili_snap::*;
+use serde_json::json;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn get_indexes_bad_offset() {

meilisearch/tests/index/stats.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn stats() {

meilisearch/tests/index/update_index.rs

@@ -1,9 +1,9 @@
+use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::encoder::Encoder;
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn update_primary_key() {

meilisearch/tests/search/distinct.rs

@@ -1,241 +0,0 @@
-use meili_snap::snapshot;
-use once_cell::sync::Lazy;
-
-use crate::common::{Server, Value};
-use crate::json;
-
-pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
-    json!([
-      {
-        "id": 1,
-        "description": "Leather Jacket",
-        "brand": "Lee Jeans",
-        "product_id": "123456",
-        "color": "Brown"
-      },
-      {
-        "id": 2,
-        "description": "Leather Jacket",
-        "brand": "Lee Jeans",
-        "product_id": "123456",
-        "color": "Black"
-      },
-      {
-        "id": 3,
-        "description": "Leather Jacket",
-        "brand": "Lee Jeans",
-        "product_id": "123456",
-        "color": "Blue"
-      },
-      {
-        "id": 4,
-        "description": "T-Shirt",
-        "brand": "Nike",
-        "product_id": "789012",
-        "color": "Red"
-      },
-      {
-        "id": 5,
-        "description": "T-Shirt",
-        "brand": "Nike",
-        "product_id": "789012",
-        "color": "Blue"
-      },
-      {
-        "id": 6,
-        "description": "Running Shoes",
-        "brand": "Adidas",
-        "product_id": "456789",
-        "color": "Black"
-      },
-      {
-        "id": 7,
-        "description": "Running Shoes",
-        "brand": "Adidas",
-        "product_id": "456789",
-        "color": "White"
-      },
-      {
-        "id": 8,
-        "description": "Hoodie",
-        "brand": "Puma",
-        "product_id": "987654",
-        "color": "Gray"
-      },
-      {
-        "id": 9,
-        "description": "Sweater",
-        "brand": "Gap",
-        "product_id": "234567",
-        "color": "Green"
-      },
-      {
-        "id": 10,
-        "description": "Sweater",
-        "brand": "Gap",
-        "product_id": "234567",
-        "color": "Red"
-      },
-      {
-        "id": 11,
-        "description": "Sweater",
-        "brand": "Gap",
-        "product_id": "234567",
-        "color": "Blue"
-      },
-      {
-        "id": 12,
-        "description": "Jeans",
-        "brand": "Levi's",
-        "product_id": "345678",
-        "color": "Indigo"
-      },
-      {
-        "id": 13,
-        "description": "Jeans",
-        "brand": "Levi's",
-        "product_id": "345678",
-        "color": "Black"
-      },
-      {
-        "id": 14,
-        "description": "Jeans",
-        "brand": "Levi's",
-        "product_id": "345678",
-        "color": "Stone Wash"
-      }
-    ])
-});
-
-pub(self) static DOCUMENT_PRIMARY_KEY: &str = "id";
-pub(self) static DOCUMENT_DISTINCT_KEY: &str = "product_id";
-
-/// testing: https://github.com/meilisearch/meilisearch/issues/4078
-#[actix_rt::test]
-async fn distinct_search_with_offset_no_ranking() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, Some(DOCUMENT_PRIMARY_KEY)).await;
-    index.update_distinct_attribute(json!(DOCUMENT_DISTINCT_KEY)).await;
-    index.wait_task(1).await;
-
-    fn get_hits(response: &Value) -> Vec<&str> {
-        let hits_array = response["hits"].as_array().unwrap();
-        hits_array.iter().map(|h| h[DOCUMENT_DISTINCT_KEY].as_str().unwrap()).collect::<Vec<_>>()
-    }
-
-    let (response, code) = index.search_post(json!({"offset": 0, "limit": 2})).await;
-    let hits = get_hits(&response);
-    snapshot!(code, @"200 OK");
-    snapshot!(hits.len(), @"2");
-    snapshot!(format!("{:?}", hits), @r#"["123456", "789012"]"#);
-    snapshot!(response["estimatedTotalHits"] , @"11");
-
-    let (response, code) = index.search_post(json!({"offset": 2, "limit": 2})).await;
-    let hits = get_hits(&response);
-    snapshot!(code, @"200 OK");
-    snapshot!(hits.len(), @"2");
-    snapshot!(format!("{:?}", hits), @r#"["456789", "987654"]"#);
-    snapshot!(response["estimatedTotalHits"], @"10");
-
-    let (response, code) = index.search_post(json!({"offset": 4, "limit": 2})).await;
-    let hits = get_hits(&response);
-    snapshot!(code, @"200 OK");
-    snapshot!(hits.len(), @"2");
-    snapshot!(format!("{:?}", hits), @r#"["234567", "345678"]"#);
-    snapshot!(response["estimatedTotalHits"], @"6");
-
-    let (response, code) = index.search_post(json!({"offset": 5, "limit": 2})).await;
-    let hits = get_hits(&response);
-    snapshot!(code, @"200 OK");
-    snapshot!(hits.len(), @"1");
-    snapshot!(format!("{:?}", hits), @r#"["345678"]"#);
-    snapshot!(response["estimatedTotalHits"], @"6");
-
-    let (response, code) = index.search_post(json!({"offset": 6, "limit": 2})).await;
-    let hits = get_hits(&response);
-    snapshot!(code, @"200 OK");
-    snapshot!(hits.len(), @"0");
-    snapshot!(format!("{:?}", hits), @r#"[]"#);
-    snapshot!(response["estimatedTotalHits"], @"6");
-
-    let (response, code) = index.search_post(json!({"offset": 7, "limit": 2})).await;
-    let hits = get_hits(&response);
-    snapshot!(code, @"200 OK");
-    snapshot!(hits.len(), @"0");
-    snapshot!(format!("{:?}", hits), @r#"[]"#);
-    snapshot!(response["estimatedTotalHits"], @"6");
-}
-
-/// testing: https://github.com/meilisearch/meilisearch/issues/4130
-#[actix_rt::test]
-async fn distinct_search_with_pagination_no_ranking() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, Some(DOCUMENT_PRIMARY_KEY)).await;
-    index.update_distinct_attribute(json!(DOCUMENT_DISTINCT_KEY)).await;
-    index.wait_task(1).await;
-
-    fn get_hits(response: &Value) -> Vec<&str> {
-        let hits_array = response["hits"].as_array().unwrap();
-        hits_array.iter().map(|h| h[DOCUMENT_DISTINCT_KEY].as_str().unwrap()).collect::<Vec<_>>()
-    }
-
-    let (response, code) = index.search_post(json!({"page": 0, "hitsPerPage": 2})).await;
-    let hits = get_hits(&response);
-    snapshot!(code, @"200 OK");
-    snapshot!(hits.len(), @"0");
-    snapshot!(format!("{:?}", hits), @r#"[]"#);
-    snapshot!(response["page"], @"0");
-    snapshot!(response["totalPages"], @"3");
-    snapshot!(response["totalHits"], @"6");
-
-    let (response, code) = index.search_post(json!({"page": 1, "hitsPerPage": 2})).await;
-    let hits = get_hits(&response);
-    snapshot!(code, @"200 OK");
-    snapshot!(hits.len(), @"2");
-    snapshot!(format!("{:?}", hits), @r#"["123456", "789012"]"#);
-    snapshot!(response["page"], @"1");
-    snapshot!(response["totalPages"], @"3");
-    snapshot!(response["totalHits"], @"6");
-
-    let (response, code) = index.search_post(json!({"page": 2, "hitsPerPage": 2})).await;
-    let hits = get_hits(&response);
-    snapshot!(code, @"200 OK");
-    snapshot!(hits.len(), @"2");
-    snapshot!(format!("{:?}", hits), @r#"["456789", "987654"]"#);
-    snapshot!(response["page"], @"2");
-    snapshot!(response["totalPages"], @"3");
-    snapshot!(response["totalHits"], @"6");
-
-    let (response, code) = index.search_post(json!({"page": 3, "hitsPerPage": 2})).await;
-    let hits = get_hits(&response);
-    snapshot!(code, @"200 OK");
-    snapshot!(hits.len(), @"2");
-    snapshot!(format!("{:?}", hits), @r#"["234567", "345678"]"#);
-    snapshot!(response["page"], @"3");
-    snapshot!(response["totalPages"], @"3");
-    snapshot!(response["totalHits"], @"6");
-
-    let (response, code) = index.search_post(json!({"page": 4, "hitsPerPage": 2})).await;
-    let hits = get_hits(&response);
-    snapshot!(code, @"200 OK");
-    snapshot!(hits.len(), @"0");
-    snapshot!(format!("{:?}", hits), @r#"[]"#);
-    snapshot!(response["page"], @"4");
-    snapshot!(response["totalPages"], @"3");
-    snapshot!(response["totalHits"], @"6");
-
-    let (response, code) = index.search_post(json!({"page": 2, "hitsPerPage": 3})).await;
-    let hits = get_hits(&response);
-    snapshot!(code, @"200 OK");
-    snapshot!(hits.len(), @"3");
-    snapshot!(format!("{:?}", hits), @r#"["987654", "234567", "345678"]"#);
-    snapshot!(response["page"], @"2");
-    snapshot!(response["totalPages"], @"2");
-    snapshot!(response["totalHits"], @"6");
-}

meilisearch/tests/search/errors.rs

@@ -1,8 +1,8 @@
 use meili_snap::*;
+use serde_json::json;
 
 use super::DOCUMENTS;
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn search_unexisting_index() {

meilisearch/tests/search/facet_search.rs

@@ -1,8 +1,8 @@
 use meili_snap::snapshot;
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([

meilisearch/tests/search/formatted.rs

@@ -1,8 +1,8 @@
 use insta::{allow_duplicates, assert_json_snapshot};
+use serde_json::json;
 
 use super::*;
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn formatted_contain_wildcard() {

meilisearch/tests/search/geo.rs

@@ -1,8 +1,8 @@
 use meili_snap::{json_string, snapshot};
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([

meilisearch/tests/search/mod.rs

@@ -1,7 +1,6 @@
 // This modules contains all the test concerning search. Each particular feature of the search
 // should be tested in its own module to isolate tests and keep the tests readable.
 
-mod distinct;
 mod errors;
 mod facet_search;
 mod formatted;
@@ -11,9 +10,9 @@ mod pagination;
 mod restrict_searchable;
 
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 pub(self) static DOCUMENTS: Lazy<Value> = Lazy::new(|| {
     json!([
@@ -817,7 +816,7 @@ async fn experimental_feature_score_details() {
                       },
                       "proximity": {
                         "order": 2,
-                        "score": 0.75
+                        "score": 0.875
                       },
                       "attribute": {
                         "order": 3,
@@ -1105,59 +1104,3 @@ async fn camelcased_words() {
         })
         .await;
 }
-
-#[actix_rt::test]
-async fn simple_search_with_strange_synonyms() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.update_settings(json!({ "synonyms": {"&": ["to"], "to": ["&"]} })).await;
-    let r = index.wait_task(0).await;
-    meili_snap::snapshot!(r["status"], @r###""succeeded""###);
-
-    let documents = DOCUMENTS.clone();
-    index.add_documents(documents, None).await;
-    index.wait_task(1).await;
-
-    index
-        .search(json!({"q": "How to train"}), |response, code| {
-            meili_snap::snapshot!(code, @"200 OK");
-            meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###"
-            [
-              {
-                "title": "How to Train Your Dragon: The Hidden World",
-                "id": "166428"
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "How & train"}), |response, code| {
-            meili_snap::snapshot!(code, @"200 OK");
-            meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###"
-            [
-              {
-                "title": "How to Train Your Dragon: The Hidden World",
-                "id": "166428"
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "to"}), |response, code| {
-            meili_snap::snapshot!(code, @"200 OK");
-            meili_snap::snapshot!(meili_snap::json_string!(response["hits"]), @r###"
-            [
-              {
-                "title": "How to Train Your Dragon: The Hidden World",
-                "id": "166428"
-              }
-            ]
-            "###);
-        })
-        .await;
-}

meilisearch/tests/search/multi.rs

@@ -1,8 +1,8 @@
 use meili_snap::{json_string, snapshot};
+use serde_json::json;
 
 use super::{DOCUMENTS, NESTED_DOCUMENTS};
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn search_empty_list() {

meilisearch/tests/search/pagination.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 use crate::search::DOCUMENTS;
 
 #[actix_rt::test]

meilisearch/tests/search/restrict_searchable.rs

@@ -1,9 +1,9 @@
 use meili_snap::{json_string, snapshot};
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
 use crate::common::index::Index;
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 async fn index_with_documents<'a>(server: &'a Server, documents: &Value) -> Index<'a> {
     let index = server.index("test");

meilisearch/tests/settings/distinct.rs

@@ -1,5 +1,6 @@
+use serde_json::json;
+
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn set_and_reset_distinct_attribute() {

meilisearch/tests/settings/errors.rs

@@ -1,7 +1,7 @@
 use meili_snap::*;
+use serde_json::json;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn settings_bad_displayed_attributes() {

meilisearch/tests/settings/get_settings.rs

@@ -1,24 +1,21 @@
 use std::collections::HashMap;
 
 use once_cell::sync::Lazy;
+use serde_json::{json, Value};
 
-use crate::common::{Server, Value};
-use crate::json;
+use crate::common::Server;
 
 static DEFAULT_SETTINGS_VALUES: Lazy<HashMap<&'static str, Value>> = Lazy::new(|| {
     let mut map = HashMap::new();
     map.insert("displayed_attributes", json!(["*"]));
     map.insert("searchable_attributes", json!(["*"]));
     map.insert("filterable_attributes", json!([]));
-    map.insert("distinct_attribute", json!(null));
+    map.insert("distinct_attribute", json!(Value::Null));
     map.insert(
         "ranking_rules",
         json!(["words", "typo", "proximity", "attribute", "sort", "exactness"]),
     );
     map.insert("stop_words", json!([]));
-    map.insert("non_separator_tokens", json!([]));
-    map.insert("separator_tokens", json!([]));
-    map.insert("dictionary", json!([]));
     map.insert("synonyms", json!({}));
     map.insert(
         "faceting",
@@ -54,7 +51,7 @@ async fn get_settings() {
     let (response, code) = index.settings().await;
     assert_eq!(code, 200);
     let settings = response.as_object().unwrap();
-    assert_eq!(settings.keys().len(), 14);
+    assert_eq!(settings.keys().len(), 11);
     assert_eq!(settings["displayedAttributes"], json!(["*"]));
     assert_eq!(settings["searchableAttributes"], json!(["*"]));
     assert_eq!(settings["filterableAttributes"], json!([]));
@@ -65,9 +62,6 @@ async fn get_settings() {
         json!(["words", "typo", "proximity", "attribute", "sort", "exactness"])
     );
     assert_eq!(settings["stopWords"], json!([]));
-    assert_eq!(settings["nonSeparatorTokens"], json!([]));
-    assert_eq!(settings["separatorTokens"], json!([]));
-    assert_eq!(settings["dictionary"], json!([]));
     assert_eq!(
         settings["faceting"],
         json!({
@@ -229,7 +223,7 @@ macro_rules! test_setting_routes {
                         .chars()
                         .map(|c| if c == '_' { '-' } else { c })
                         .collect::<String>());
-                    let (response, code) = server.service.$write_method(url, serde_json::Value::Null.into()).await;
+                    let (response, code) = server.service.$write_method(url, serde_json::Value::Null).await;
                     assert_eq!(code, 202, "{}", response);
                     server.index("").wait_task(0).await;
                     let (response, code) = server.index("test").get().await;
@@ -278,9 +272,6 @@ test_setting_routes!(
     searchable_attributes put,
     distinct_attribute put,
     stop_words put,
-    separator_tokens put,
-    non_separator_tokens put,
-    dictionary put,
     ranking_rules put,
     synonyms put,
     pagination patch,

meilisearch/tests/settings/mod.rs

@@ -1,4 +1,3 @@
 mod distinct;
 mod errors;
 mod get_settings;
-mod tokenizer_customization;

meilisearch/tests/settings/tokenizer_customization.rs

@@ -1,467 +0,0 @@
-use meili_snap::{json_string, snapshot};
-
-use crate::common::Server;
-use crate::json;
-
-#[actix_rt::test]
-async fn set_and_reset() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    let (_response, _code) = index
-        .update_settings(json!({
-            "nonSeparatorTokens": ["#", "&"],
-            "separatorTokens": ["&sep", "<br/>"],
-            "dictionary": ["J.R.R.", "J. R. R."],
-        }))
-        .await;
-    index.wait_task(0).await;
-
-    let (response, _) = index.settings().await;
-    snapshot!(json_string!(response["nonSeparatorTokens"]), @r###"
-    [
-      "#",
-      "&"
-    ]
-    "###);
-    snapshot!(json_string!(response["separatorTokens"]), @r###"
-    [
-      "&sep",
-      "<br/>"
-    ]
-    "###);
-    snapshot!(json_string!(response["dictionary"]), @r###"
-    [
-      "J. R. R.",
-      "J.R.R."
-    ]
-    "###);
-
-    index
-        .update_settings(json!({
-            "nonSeparatorTokens": null,
-            "separatorTokens": null,
-            "dictionary": null,
-        }))
-        .await;
-
-    index.wait_task(1).await;
-
-    let (response, _) = index.settings().await;
-    snapshot!(json_string!(response["nonSeparatorTokens"]), @"[]");
-    snapshot!(json_string!(response["separatorTokens"]), @"[]");
-    snapshot!(json_string!(response["dictionary"]), @"[]");
-}
-
-#[actix_rt::test]
-async fn set_and_search() {
-    let documents = json!([
-        {
-            "id": 1,
-            "content": "Mac & cheese",
-        },
-        {
-            "id": 2,
-            "content": "G#D#G#D#G#C#D#G#C#",
-        },
-        {
-            "id": 3,
-            "content": "Mac&sep&&sepcheese",
-        },
-    ]);
-
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
-    let (_response, _code) = index
-        .update_settings(json!({
-            "nonSeparatorTokens": ["#", "&"],
-            "separatorTokens": ["<br/>", "&sep"],
-            "dictionary": ["#", "A#", "B#", "C#", "D#", "E#", "F#", "G#"],
-        }))
-        .await;
-    index.wait_task(1).await;
-
-    index
-        .search(json!({"q": "&", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 1,
-                "content": "Mac & cheese",
-                "_formatted": {
-                  "id": "1",
-                  "content": "Mac <em>&</em> cheese"
-                }
-              },
-              {
-                "id": 3,
-                "content": "Mac&sep&&sepcheese",
-                "_formatted": {
-                  "id": "3",
-                  "content": "Mac&sep<em>&</em>&sepcheese"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(
-            json!({"q": "Mac & cheese", "attributesToHighlight": ["content"]}),
-            |response, code| {
-                snapshot!(code, @"200 OK");
-                snapshot!(json_string!(response["hits"]), @r###"
-                [
-                  {
-                    "id": 1,
-                    "content": "Mac & cheese",
-                    "_formatted": {
-                      "id": "1",
-                      "content": "<em>Mac</em> <em>&</em> <em>cheese</em>"
-                    }
-                  },
-                  {
-                    "id": 3,
-                    "content": "Mac&sep&&sepcheese",
-                    "_formatted": {
-                      "id": "3",
-                      "content": "<em>Mac</em>&sep<em>&</em>&sep<em>cheese</em>"
-                    }
-                  }
-                ]
-                "###);
-            },
-        )
-        .await;
-
-    index
-        .search(
-            json!({"q": "Mac&sep&&sepcheese", "attributesToHighlight": ["content"]}),
-            |response, code| {
-                snapshot!(code, @"200 OK");
-                snapshot!(json_string!(response["hits"]), @r###"
-                [
-                  {
-                    "id": 1,
-                    "content": "Mac & cheese",
-                    "_formatted": {
-                      "id": "1",
-                      "content": "<em>Mac</em> <em>&</em> <em>cheese</em>"
-                    }
-                  },
-                  {
-                    "id": 3,
-                    "content": "Mac&sep&&sepcheese",
-                    "_formatted": {
-                      "id": "3",
-                      "content": "<em>Mac</em>&sep<em>&</em>&sep<em>cheese</em>"
-                    }
-                  }
-                ]
-                "###);
-            },
-        )
-        .await;
-
-    index
-        .search(json!({"q": "C#D#G", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 2,
-                "content": "G#D#G#D#G#C#D#G#C#",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>G</em>#<em>D#</em><em>G</em>#<em>D#</em><em>G</em>#<em>C#</em><em>D#</em><em>G</em>#<em>C#</em>"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "#", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @"[]");
-        })
-        .await;
-}
-
-#[actix_rt::test]
-async fn advanced_synergies() {
-    let documents = json!([
-        {
-            "id": 1,
-            "content": "J.R.R. Tolkien",
-        },
-        {
-            "id": 2,
-            "content": "J. R. R. Tolkien",
-        },
-        {
-            "id": 3,
-            "content": "jrr Tolkien",
-        },
-        {
-            "id": 4,
-            "content": "J.K. Rowlings",
-        },
-        {
-            "id": 5,
-            "content": "J. K. Rowlings",
-        },
-        {
-            "id": 6,
-            "content": "jk Rowlings",
-        },
-    ]);
-
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.add_documents(documents, None).await;
-    index.wait_task(0).await;
-
-    let (_response, _code) = index
-        .update_settings(json!({
-            "dictionary": ["J.R.R.", "J. R. R."],
-            "synonyms": {
-                "J.R.R.": ["jrr", "J. R. R."],
-                "J. R. R.": ["jrr", "J.R.R."],
-                "jrr": ["J.R.R.", "J. R. R."],
-                "J.K.": ["jk", "J. K."],
-                "J. K.": ["jk", "J.K."],
-                "jk": ["J.K.", "J. K."],
-            }
-        }))
-        .await;
-    index.wait_task(1).await;
-
-    index
-        .search(json!({"q": "J.R.R.", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 1,
-                "content": "J.R.R. Tolkien",
-                "_formatted": {
-                  "id": "1",
-                  "content": "<em>J.R.R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 2,
-                "content": "J. R. R. Tolkien",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>J. R. R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 3,
-                "content": "jrr Tolkien",
-                "_formatted": {
-                  "id": "3",
-                  "content": "<em>jrr</em> Tolkien"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "jrr", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 3,
-                "content": "jrr Tolkien",
-                "_formatted": {
-                  "id": "3",
-                  "content": "<em>jrr</em> Tolkien"
-                }
-              },
-              {
-                "id": 1,
-                "content": "J.R.R. Tolkien",
-                "_formatted": {
-                  "id": "1",
-                  "content": "<em>J.R.R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 2,
-                "content": "J. R. R. Tolkien",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>J. R. R.</em> Tolkien"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "J. R. R.", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 2,
-                "content": "J. R. R. Tolkien",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>J. R. R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 1,
-                "content": "J.R.R. Tolkien",
-                "_formatted": {
-                  "id": "1",
-                  "content": "<em>J.R.R.</em> Tolkien"
-                }
-              },
-              {
-                "id": 3,
-                "content": "jrr Tolkien",
-                "_formatted": {
-                  "id": "3",
-                  "content": "<em>jrr</em> Tolkien"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    // Only update dictionary, the synonyms should be recomputed.
-    let (_response, _code) = index
-        .update_settings(json!({
-            "dictionary": ["J.R.R.", "J. R. R.", "J.K.", "J. K."],
-        }))
-        .await;
-    index.wait_task(2).await;
-
-    index
-        .search(json!({"q": "jk", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 6,
-                "content": "jk Rowlings",
-                "_formatted": {
-                  "id": "6",
-                  "content": "<em>jk</em> Rowlings"
-                }
-              },
-              {
-                "id": 4,
-                "content": "J.K. Rowlings",
-                "_formatted": {
-                  "id": "4",
-                  "content": "<em>J.K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 5,
-                "content": "J. K. Rowlings",
-                "_formatted": {
-                  "id": "5",
-                  "content": "<em>J. K.</em> Rowlings"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "J.K.", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 4,
-                "content": "J.K. Rowlings",
-                "_formatted": {
-                  "id": "4",
-                  "content": "<em>J.K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 5,
-                "content": "J. K. Rowlings",
-                "_formatted": {
-                  "id": "5",
-                  "content": "<em>J. K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 6,
-                "content": "jk Rowlings",
-                "_formatted": {
-                  "id": "6",
-                  "content": "<em>jk</em> Rowlings"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-
-    index
-        .search(json!({"q": "J. K.", "attributesToHighlight": ["content"]}), |response, code| {
-            snapshot!(code, @"200 OK");
-            snapshot!(json_string!(response["hits"]), @r###"
-            [
-              {
-                "id": 5,
-                "content": "J. K. Rowlings",
-                "_formatted": {
-                  "id": "5",
-                  "content": "<em>J. K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 4,
-                "content": "J.K. Rowlings",
-                "_formatted": {
-                  "id": "4",
-                  "content": "<em>J.K.</em> Rowlings"
-                }
-              },
-              {
-                "id": 6,
-                "content": "jk Rowlings",
-                "_formatted": {
-                  "id": "6",
-                  "content": "<em>jk</em> Rowlings"
-                }
-              },
-              {
-                "id": 2,
-                "content": "J. R. R. Tolkien",
-                "_formatted": {
-                  "id": "2",
-                  "content": "<em>J. R.</em> R. Tolkien"
-                }
-              }
-            ]
-            "###);
-        })
-        .await;
-}

meilisearch/tests/snapshot/mod.rs

@@ -1,13 +1,11 @@
 use std::time::Duration;
 
 use actix_rt::time::sleep;
-use meili_snap::{json_string, snapshot};
 use meilisearch::option::ScheduleSnapshot;
 use meilisearch::Opt;
 
 use crate::common::server::default_settings;
 use crate::common::{GetAllDocumentsOptions, Server};
-use crate::json;
 
 macro_rules! verify_snapshot {
     (
@@ -46,7 +44,7 @@ async fn perform_snapshot() {
 
     let index = server.index("test");
     index
-        .update_settings(json! ({
+        .update_settings(serde_json::json! ({
         "searchableAttributes": [],
         }))
         .await;
@@ -92,95 +90,3 @@ async fn perform_snapshot() {
         server.index("test1").settings(),
     );
 }
-
-#[actix_rt::test]
-async fn perform_on_demand_snapshot() {
-    let temp = tempfile::tempdir().unwrap();
-    let snapshot_dir = tempfile::tempdir().unwrap();
-
-    let options =
-        Opt { snapshot_dir: snapshot_dir.path().to_owned(), ..default_settings(temp.path()) };
-
-    let server = Server::new_with_options(options).await.unwrap();
-
-    let index = server.index("catto");
-    index
-        .update_settings(json! ({
-        "searchableAttributes": [],
-        }))
-        .await;
-
-    index.load_test_set().await;
-
-    server.index("doggo").create(Some("bone")).await;
-    index.wait_task(2).await;
-
-    server.index("doggo").create(Some("bone")).await;
-    index.wait_task(2).await;
-
-    let (task, code) = server.create_snapshot().await;
-    snapshot!(code, @"202 Accepted");
-    snapshot!(json_string!(task, { ".enqueuedAt" => "[date]" }), @r###"
-    {
-      "taskUid": 4,
-      "indexUid": null,
-      "status": "enqueued",
-      "type": "snapshotCreation",
-      "enqueuedAt": "[date]"
-    }
-    "###);
-    let task = index.wait_task(task.uid()).await;
-    snapshot!(json_string!(task, { ".enqueuedAt" => "[date]", ".startedAt" => "[date]", ".finishedAt" => "[date]", ".duration" => "[duration]" }), @r###"
-    {
-      "uid": 4,
-      "indexUid": null,
-      "status": "succeeded",
-      "type": "snapshotCreation",
-      "canceledBy": null,
-      "error": null,
-      "duration": "[duration]",
-      "enqueuedAt": "[date]",
-      "startedAt": "[date]",
-      "finishedAt": "[date]"
-    }
-    "###);
-
-    let temp = tempfile::tempdir().unwrap();
-
-    let snapshots: Vec<String> = std::fs::read_dir(&snapshot_dir)
-        .unwrap()
-        .map(|entry| entry.unwrap().path().file_name().unwrap().to_str().unwrap().to_string())
-        .collect();
-    meili_snap::snapshot!(format!("{snapshots:?}"), @r###"["db.snapshot"]"###);
-
-    let snapshot_path = snapshot_dir.path().to_owned().join("db.snapshot");
-    #[cfg_attr(windows, allow(unused))]
-    let snapshot_meta = std::fs::metadata(&snapshot_path).unwrap();
-
-    #[cfg(unix)]
-    {
-        use std::os::unix::fs::PermissionsExt;
-        let mode = snapshot_meta.permissions().mode();
-        //                                                    rwxrwxrwx
-        meili_snap::snapshot!(format!("{:b}", mode), @"1000000100100100");
-    }
-
-    let options = Opt { import_snapshot: Some(snapshot_path), ..default_settings(temp.path()) };
-
-    let snapshot_server = Server::new_with_options(options).await.unwrap();
-
-    verify_snapshot!(server, snapshot_server, |server| =>
-        server.list_indexes(None, None),
-        // for some reason the db sizes differ. this may be due to the compaction options we have
-        // set when performing the snapshot
-        //server.stats(),
-
-        // The original instance contains the snapshotCreation task, while the snapshotted-instance does not. For this reason we need to compare the task queue **after** the task 4
-        server.tasks_filter("?from=2"),
-
-        server.index("catto").get_all_documents(GetAllDocumentsOptions::default()),
-        server.index("catto").settings(),
-        server.index("doggo").get_all_documents(GetAllDocumentsOptions::default()),
-        server.index("doggo").settings(),
-    );
-}

meilisearch/tests/stats/mod.rs

@@ -1,8 +1,8 @@
+use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn get_settings_unexisting_index() {

meilisearch/tests/swap_indexes/errors.rs

@@ -1,7 +1,7 @@
 use meili_snap::*;
+use serde_json::json;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn swap_indexes_bad_format() {

meilisearch/tests/swap_indexes/mod.rs

@@ -1,9 +1,9 @@
 mod errors;
 
 use meili_snap::{json_string, snapshot};
+use serde_json::json;
 
 use crate::common::{GetAllDocumentsOptions, Server};
-use crate::json;
 
 #[actix_rt::test]
 async fn swap_indexes() {

meilisearch/tests/tasks/mod.rs

@@ -1,11 +1,11 @@
 mod errors;
 
 use meili_snap::insta::assert_json_snapshot;
+use serde_json::json;
 use time::format_description::well_known::Rfc3339;
 use time::OffsetDateTime;
 
 use crate::common::Server;
-use crate::json;
 
 #[actix_rt::test]
 async fn error_get_unexisting_task_status() {
@@ -33,7 +33,7 @@ async fn get_task_status() {
     index.create(None).await;
     index
         .add_documents(
-            json!([{
+            serde_json::json!([{
                 "id": 1,
                 "content": "foobar",
             }]),

meilitool/Cargo.toml

@@ -1,19 +0,0 @@
-[package]
-name = "meilitool"
-description = "A CLI to edit a Meilisearch database from the command line"
-version.workspace = true
-authors.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
-[dependencies]
-anyhow = "1.0.75"
-clap = { version = "4.2.1", features = ["derive"] }
-dump = { path = "../dump" }
-file-store = { path = "../file-store" }
-meilisearch-auth = { path = "../meilisearch-auth" }
-meilisearch-types = { path = "../meilisearch-types" }
-time = { version = "0.3.30", features = ["formatting"] }
-uuid = { version = "1.5.0", features = ["v4"], default-features = false }

meilitool/src/main.rs

@@ -1,312 +0,0 @@
-use std::fs::{read_dir, read_to_string, remove_file, File};
-use std::io::BufWriter;
-use std::path::PathBuf;
-
-use anyhow::Context;
-use clap::{Parser, Subcommand};
-use dump::{DumpWriter, IndexMetadata};
-use file_store::FileStore;
-use meilisearch_auth::AuthController;
-use meilisearch_types::heed::types::{OwnedType, SerdeJson, Str};
-use meilisearch_types::heed::{Database, Env, EnvOpenOptions, PolyDatabase, RoTxn, RwTxn};
-use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader};
-use meilisearch_types::milli::{obkv_to_json, BEU32};
-use meilisearch_types::tasks::{Status, Task};
-use meilisearch_types::versioning::check_version_file;
-use meilisearch_types::Index;
-use time::macros::format_description;
-use time::OffsetDateTime;
-use uuid_codec::UuidCodec;
-
-mod uuid_codec;
-
-#[derive(Parser)]
-#[command(author, version, about, long_about = None)]
-struct Cli {
-    /// The database path where the Meilisearch is running.
-    #[arg(long, default_value = "data.ms/")]
-    db_path: PathBuf,
-
-    #[command(subcommand)]
-    command: Command,
-}
-
-#[derive(Subcommand)]
-enum Command {
-    /// Clears the task queue and make it empty.
-    ///
-    /// This command can be safely executed even if Meilisearch is running and processing tasks.
-    /// Once the task queue is empty you can restart Meilisearch and no more tasks must be visible,
-    /// even the ones that were processing. However, it's highly possible that you see the processing
-    /// tasks in the queue again with an associated internal error message.
-    ClearTaskQueue,
-
-    /// Exports a dump from the Meilisearch database.
-    ///
-    /// Make sure to run this command when Meilisearch is not running or running but not processing tasks.
-    /// If tasks are being processed while a dump is being exported there are chances for the dump to be
-    /// malformed with missing tasks.
-    ///
-    /// TODO Verify this claim or make sure it cannot happen and we can export dumps
-    ///      without caring about killing Meilisearch first!
-    ExportADump {
-        /// The directory in which the dump will be created.
-        #[arg(long, default_value = "dumps/")]
-        dump_dir: PathBuf,
-
-        /// Skip dumping the enqueued or processing tasks.
-        ///
-        /// Can be useful when there are a lot of them and it is not particularly useful
-        /// to keep them. Note that only the enqueued tasks takes up space so skipping
-        /// the processed ones is not particularly interesting.
-        #[arg(long)]
-        skip_enqueued_tasks: bool,
-    },
-}
-
-fn main() -> anyhow::Result<()> {
-    let Cli { db_path, command } = Cli::parse();
-
-    check_version_file(&db_path).context("While checking the version file")?;
-
-    match command {
-        Command::ClearTaskQueue => clear_task_queue(db_path),
-        Command::ExportADump { dump_dir, skip_enqueued_tasks } => {
-            export_a_dump(db_path, dump_dir, skip_enqueued_tasks)
-        }
-    }
-}
-
-/// Clears the task queue located at `db_path`.
-fn clear_task_queue(db_path: PathBuf) -> anyhow::Result<()> {
-    let path = db_path.join("tasks");
-    let env = EnvOpenOptions::new()
-        .max_dbs(100)
-        .open(&path)
-        .with_context(|| format!("While trying to open {:?}", path.display()))?;
-
-    eprintln!("Deleting tasks from the database...");
-
-    let mut wtxn = env.write_txn()?;
-    let all_tasks = try_opening_poly_database(&env, &wtxn, "all-tasks")?;
-    let total = all_tasks.len(&wtxn)?;
-    let status = try_opening_poly_database(&env, &wtxn, "status")?;
-    let kind = try_opening_poly_database(&env, &wtxn, "kind")?;
-    let index_tasks = try_opening_poly_database(&env, &wtxn, "index-tasks")?;
-    let canceled_by = try_opening_poly_database(&env, &wtxn, "canceled_by")?;
-    let enqueued_at = try_opening_poly_database(&env, &wtxn, "enqueued-at")?;
-    let started_at = try_opening_poly_database(&env, &wtxn, "started-at")?;
-    let finished_at = try_opening_poly_database(&env, &wtxn, "finished-at")?;
-
-    try_clearing_poly_database(&mut wtxn, all_tasks, "all-tasks")?;
-    try_clearing_poly_database(&mut wtxn, status, "status")?;
-    try_clearing_poly_database(&mut wtxn, kind, "kind")?;
-    try_clearing_poly_database(&mut wtxn, index_tasks, "index-tasks")?;
-    try_clearing_poly_database(&mut wtxn, canceled_by, "canceled_by")?;
-    try_clearing_poly_database(&mut wtxn, enqueued_at, "enqueued-at")?;
-    try_clearing_poly_database(&mut wtxn, started_at, "started-at")?;
-    try_clearing_poly_database(&mut wtxn, finished_at, "finished-at")?;
-
-    wtxn.commit().context("While committing the transaction")?;
-
-    eprintln!("Successfully deleted {total} tasks from the tasks database!");
-    eprintln!("Deleting the content files from disk...");
-
-    let mut count = 0usize;
-    let update_files = db_path.join("update_files");
-    let entries = read_dir(&update_files).with_context(|| {
-        format!("While trying to read the content of {:?}", update_files.display())
-    })?;
-    for result in entries {
-        match result {
-            Ok(ent) => match remove_file(ent.path()) {
-                Ok(_) => count += 1,
-                Err(e) => eprintln!("Error while deleting {:?}: {}", ent.path().display(), e),
-            },
-            Err(e) => {
-                eprintln!("Error while reading a file in {:?}: {}", update_files.display(), e)
-            }
-        }
-    }
-
-    eprintln!("Sucessfully deleted {count} content files from disk!");
-
-    Ok(())
-}
-
-fn try_opening_database<KC: 'static, DC: 'static>(
-    env: &Env,
-    rtxn: &RoTxn,
-    db_name: &str,
-) -> anyhow::Result<Database<KC, DC>> {
-    env.open_database(rtxn, Some(db_name))
-        .with_context(|| format!("While opening the {db_name:?} database"))?
-        .with_context(|| format!("Missing the {db_name:?} database"))
-}
-
-fn try_opening_poly_database(
-    env: &Env,
-    rtxn: &RoTxn,
-    db_name: &str,
-) -> anyhow::Result<PolyDatabase> {
-    env.open_poly_database(rtxn, Some(db_name))
-        .with_context(|| format!("While opening the {db_name:?} poly database"))?
-        .with_context(|| format!("Missing the {db_name:?} poly database"))
-}
-
-fn try_clearing_poly_database(
-    wtxn: &mut RwTxn,
-    database: PolyDatabase,
-    db_name: &str,
-) -> anyhow::Result<()> {
-    database.clear(wtxn).with_context(|| format!("While clearing the {db_name:?} database"))
-}
-
-/// Exports a dump into the dump directory.
-fn export_a_dump(
-    db_path: PathBuf,
-    dump_dir: PathBuf,
-    skip_enqueued_tasks: bool,
-) -> Result<(), anyhow::Error> {
-    let started_at = OffsetDateTime::now_utc();
-
-    // 1. Extracts the instance UID from disk
-    let instance_uid_path = db_path.join("instance-uid");
-    let instance_uid = match read_to_string(&instance_uid_path) {
-        Ok(content) => match content.trim().parse() {
-            Ok(uuid) => Some(uuid),
-            Err(e) => {
-                eprintln!("Impossible to parse instance-uid: {e}");
-                None
-            }
-        },
-        Err(e) => {
-            eprintln!("Impossible to read {}: {}", instance_uid_path.display(), e);
-            None
-        }
-    };
-
-    let dump = DumpWriter::new(instance_uid).context("While creating a new dump")?;
-    let file_store =
-        FileStore::new(db_path.join("update_files")).context("While opening the FileStore")?;
-
-    let index_scheduler_path = db_path.join("tasks");
-    let env = EnvOpenOptions::new()
-        .max_dbs(100)
-        .open(&index_scheduler_path)
-        .with_context(|| format!("While trying to open {:?}", index_scheduler_path.display()))?;
-
-    eprintln!("Dumping the keys...");
-
-    // 2. dump the keys
-    let auth_store = AuthController::new(&db_path, &None)
-        .with_context(|| format!("While opening the auth store at {}", db_path.display()))?;
-    let mut dump_keys = dump.create_keys()?;
-    let mut count = 0;
-    for key in auth_store.list_keys()? {
-        dump_keys.push_key(&key)?;
-        count += 1;
-    }
-    dump_keys.flush()?;
-
-    eprintln!("Successfully dumped {count} keys!");
-
-    let rtxn = env.read_txn()?;
-    let all_tasks: Database<OwnedType<BEU32>, SerdeJson<Task>> =
-        try_opening_database(&env, &rtxn, "all-tasks")?;
-    let index_mapping: Database<Str, UuidCodec> =
-        try_opening_database(&env, &rtxn, "index-mapping")?;
-
-    if skip_enqueued_tasks {
-        eprintln!("Skip dumping the enqueued tasks...");
-    } else {
-        eprintln!("Dumping the enqueued tasks...");
-
-        // 3. dump the tasks
-        let mut dump_tasks = dump.create_tasks_queue()?;
-        let mut count = 0;
-        for ret in all_tasks.iter(&rtxn)? {
-            let (_, t) = ret?;
-            let status = t.status;
-            let content_file = t.content_uuid();
-            let mut dump_content_file = dump_tasks.push_task(&t.into())?;
-
-            // 3.1. Dump the `content_file` associated with the task if there is one and the task is not finished yet.
-            if let Some(content_file_uuid) = content_file {
-                if status == Status::Enqueued {
-                    let content_file = file_store.get_update(content_file_uuid)?;
-
-                    let reader =
-                        DocumentsBatchReader::from_reader(content_file).with_context(|| {
-                            format!("While reading content file {:?}", content_file_uuid)
-                        })?;
-
-                    let (mut cursor, documents_batch_index) = reader.into_cursor_and_fields_index();
-                    while let Some(doc) = cursor.next_document().with_context(|| {
-                        format!("While iterating on content file {:?}", content_file_uuid)
-                    })? {
-                        dump_content_file
-                            .push_document(&obkv_to_object(&doc, &documents_batch_index)?)?;
-                    }
-                    dump_content_file.flush()?;
-                    count += 1;
-                }
-            }
-        }
-        dump_tasks.flush()?;
-
-        eprintln!("Successfully dumped {count} enqueued tasks!");
-    }
-
-    eprintln!("Dumping the indexes...");
-
-    // 4. Dump the indexes
-    let mut count = 0;
-    for result in index_mapping.iter(&rtxn)? {
-        let (uid, uuid) = result?;
-        let index_path = db_path.join("indexes").join(uuid.to_string());
-        let index = Index::new(EnvOpenOptions::new(), &index_path).with_context(|| {
-            format!("While trying to open the index at path {:?}", index_path.display())
-        })?;
-
-        let rtxn = index.read_txn()?;
-        let metadata = IndexMetadata {
-            uid: uid.to_owned(),
-            primary_key: index.primary_key(&rtxn)?.map(String::from),
-            created_at: index.created_at(&rtxn)?,
-            updated_at: index.updated_at(&rtxn)?,
-        };
-        let mut index_dumper = dump.create_index(uid, &metadata)?;
-
-        let fields_ids_map = index.fields_ids_map(&rtxn)?;
-        let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
-
-        // 4.1. Dump the documents
-        for ret in index.all_documents(&rtxn)? {
-            let (_id, doc) = ret?;
-            let document = obkv_to_json(&all_fields, &fields_ids_map, doc)?;
-            index_dumper.push_document(&document)?;
-        }
-
-        // 4.2. Dump the settings
-        let settings = meilisearch_types::settings::settings(&index, &rtxn)?;
-        index_dumper.settings(&settings)?;
-        count += 1;
-    }
-
-    eprintln!("Successfully dumped {count} indexes!");
-    // We will not dump experimental feature settings
-    eprintln!("The tool is not dumping experimental features, please set them by hand afterward");
-
-    let dump_uid = started_at.format(format_description!(
-        "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]"
-    )).unwrap();
-
-    let path = dump_dir.join(format!("{}.dump", dump_uid));
-    let file = File::create(&path)?;
-    dump.persist_to(BufWriter::new(file))?;
-
-    eprintln!("Dump exported at path {:?}", path.display());
-
-    Ok(())
-}

meilitool/src/uuid_codec.rs

@@ -1,24 +0,0 @@
-use std::borrow::Cow;
-use std::convert::TryInto;
-
-use meilisearch_types::heed::{BytesDecode, BytesEncode};
-use uuid::Uuid;
-
-/// A heed codec for value of struct Uuid.
-pub struct UuidCodec;
-
-impl<'a> BytesDecode<'a> for UuidCodec {
-    type DItem = Uuid;
-
-    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
-        bytes.try_into().ok().map(Uuid::from_bytes)
-    }
-}
-
-impl BytesEncode<'_> for UuidCodec {
-    type EItem = Uuid;
-
-    fn bytes_encode(item: &Self::EItem) -> Option<Cow<[u8]>> {
-        Some(Cow::Borrowed(item.as_bytes()))
-    }
-}

milli/Cargo.toml

@@ -17,10 +17,10 @@ bincode = "1.3.3"
 bstr = "1.4.0"
 bytemuck = { version = "1.13.1", features = ["extern_crate_alloc"] }
 byteorder = "1.4.3"
-charabia = { version = "0.8.5", default-features = false }
+charabia = { version = "0.8.3", default-features = false }
 concat-arrays = "0.1.2"
 crossbeam-channel = "0.5.8"
-deserr = { version = "0.6.0", features = ["actix-web"]}
+deserr = "0.5.0"
 either = { version = "1.8.1", features = ["serde"] }
 flatten-serde-json = { path = "../flatten-serde-json" }
 fst = "0.4.7"
@@ -32,18 +32,18 @@ grenad = { version = "0.4.4", default-features = false, features = [
 heed = { git = "https://github.com/meilisearch/heed", tag = "v0.12.7", default-features = false, features = [
     "lmdb", "read-txn-no-tls"
 ] }
-indexmap = { version = "2.0.0", features = ["serde"] }
+indexmap = { version = "1.9.3", features = ["serde"] }
 instant-distance = { version = "0.6.1", features = ["with-serde"] }
 json-depth-checker = { path = "../json-depth-checker" }
 levenshtein_automata = { version = "0.2.1", features = ["fst_automaton"] }
-memmap2 = "0.7.1"
+memmap2 = "0.5.10"
 obkv = "0.2.0"
 once_cell = "1.17.1"
 ordered-float = "3.6.0"
 rand_pcg = { version = "0.3.1", features = ["serde1"] }
 rayon = "1.7.0"
 roaring = "0.10.1"
-rstar = { version = "0.11.0", features = ["serde"] }
+rstar = { version = "0.10.0", features = ["serde"] }
 serde = { version = "1.0.160", features = ["derive"] }
 serde_json = { version = "1.0.95", features = ["preserve_order"] }
 slice-group-by = "0.3.0"
@@ -63,10 +63,7 @@ uuid = { version = "1.3.1", features = ["v4"] }
 filter-parser = { path = "../filter-parser" }
 
 # documents words self-join
-itertools = "0.11.0"
-
-# profiling
-puffin = "0.16.0"
+itertools = "0.10.5"
 
 # logging
 log = "0.4.17"
@@ -74,7 +71,7 @@ logging_timer = "1.1.0"
 csv = "1.2.1"
 
 [dev-dependencies]
-mimalloc = { version = "0.1.37", default-features = false }
+mimalloc = { version = "0.1.29", default-features = false }
 big_s = "1.0.2"
 insta = "1.29.0"
 maplit = "1.0.2"
@@ -82,7 +79,7 @@ md5 = "0.7.0"
 rand = { version = "0.8.5", features = ["small_rng"] }
 
 [features]
-all-tokenizations = ["charabia/chinese", "charabia/hebrew", "charabia/japanese", "charabia/thai", "charabia/korean", "charabia/greek", "charabia/khmer"]
+all-tokenizations = ["charabia/chinese", "charabia/hebrew", "charabia/japanese", "charabia/thai", "charabia/korean", "charabia/greek"]
 
 # Use POSIX semaphores instead of SysV semaphores in LMDB
 # For more information on this feature, see heed's Cargo.toml
@@ -106,6 +103,3 @@ thai = ["charabia/thai"]
 
 # allow greek specialized tokenization
 greek = ["charabia/greek"]
-
-# allow khmer specialized tokenization
-khmer = ["charabia/khmer"]

milli/src/documents/enriched.rs

@@ -1,5 +1,4 @@
 use std::fs::File;
-use std::io::BufReader;
 use std::{io, str};
 
 use obkv::KvReader;
@@ -20,14 +19,14 @@ use crate::FieldId;
 pub struct EnrichedDocumentsBatchReader<R> {
     documents: DocumentsBatchReader<R>,
     primary_key: String,
-    external_ids: grenad::ReaderCursor<BufReader<File>>,
+    external_ids: grenad::ReaderCursor<File>,
 }
 
 impl<R: io::Read + io::Seek> EnrichedDocumentsBatchReader<R> {
     pub fn new(
         documents: DocumentsBatchReader<R>,
         primary_key: String,
-        external_ids: grenad::Reader<BufReader<File>>,
+        external_ids: grenad::Reader<File>,
     ) -> Result<Self, Error> {
         if documents.documents_count() as u64 == external_ids.len() {
             Ok(EnrichedDocumentsBatchReader {
@@ -76,7 +75,7 @@ pub struct EnrichedDocument<'a> {
 pub struct EnrichedDocumentsBatchCursor<R> {
     documents: DocumentsBatchCursor<R>,
     primary_key: String,
-    external_ids: grenad::ReaderCursor<BufReader<File>>,
+    external_ids: grenad::ReaderCursor<File>,
 }
 
 impl<R> EnrichedDocumentsBatchCursor<R> {

milli/src/error.rs

@@ -122,28 +122,22 @@ only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and undersco
         .field,
         match .valid_fields.is_empty() {
             true => "This index does not have configured sortable attributes.".to_string(),
-            false => format!("Available sortable attributes are: `{}{}`.",
-                    valid_fields.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(", "),
-                    .hidden_fields.then_some(", <..hidden-attributes>").unwrap_or(""),
+            false => format!("Available sortable attributes are: `{}`.",
+                    valid_fields.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(", ")
                 ),
         }
     )]
-    InvalidSortableAttribute { field: String, valid_fields: BTreeSet<String>, hidden_fields: bool },
+    InvalidSortableAttribute { field: String, valid_fields: BTreeSet<String> },
     #[error("Attribute `{}` is not facet-searchable. {}",
         .field,
         match .valid_fields.is_empty() {
             true => "This index does not have configured facet-searchable attributes. To make it facet-searchable add it to the `filterableAttributes` index settings.".to_string(),
-            false => format!("Available facet-searchable attributes are: `{}{}`. To make it facet-searchable add it to the `filterableAttributes` index settings.",
-                    valid_fields.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(", "),
-                    .hidden_fields.then_some(", <..hidden-attributes>").unwrap_or(""),
+            false => format!("Available facet-searchable attributes are: `{}`. To make it facet-searchable add it to the `filterableAttributes` index settings.",
+                    valid_fields.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(", ")
                 ),
         }
     )]
-    InvalidFacetSearchFacetName {
-        field: String,
-        valid_fields: BTreeSet<String>,
-        hidden_fields: bool,
-    },
+    InvalidFacetSearchFacetName { field: String, valid_fields: BTreeSet<String> },
     #[error("Attribute `{}` is not searchable. Available searchable attributes are: `{}{}`.",
         .field,
         .valid_fields.iter().map(AsRef::as_ref).collect::<Vec<&str>>().join(", "),
@@ -346,11 +340,8 @@ fn conditionally_lookup_for_error_message() {
     ];
 
     for (list, suffix) in messages {
-        let err = UserError::InvalidSortableAttribute {
-            field: "name".to_string(),
-            valid_fields: list,
-            hidden_fields: false,
-        };
+        let err =
+            UserError::InvalidSortableAttribute { field: "name".to_string(), valid_fields: list };
 
         assert_eq!(err.to_string(), format!("{} {}", prefix, suffix));
     }

milli/src/index.rs

@@ -1,5 +1,5 @@
 use std::borrow::Cow;
-use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
+use std::collections::{BTreeSet, HashMap, HashSet};
 use std::fs::File;
 use std::mem::size_of;
 use std::path::Path;
@@ -61,12 +61,8 @@ pub mod main_key {
     pub const USER_DEFINED_SEARCHABLE_FIELDS_KEY: &str = "user-defined-searchable-fields";
     pub const SOFT_EXTERNAL_DOCUMENTS_IDS_KEY: &str = "soft-external-documents-ids";
     pub const STOP_WORDS_KEY: &str = "stop-words";
-    pub const NON_SEPARATOR_TOKENS_KEY: &str = "non-separator-tokens";
-    pub const SEPARATOR_TOKENS_KEY: &str = "separator-tokens";
-    pub const DICTIONARY_KEY: &str = "dictionary";
     pub const STRING_FACETED_DOCUMENTS_IDS_PREFIX: &str = "string-faceted-documents-ids";
     pub const SYNONYMS_KEY: &str = "synonyms";
-    pub const USER_DEFINED_SYNONYMS_KEY: &str = "user-defined-synonyms";
     pub const WORDS_FST_KEY: &str = "words-fst";
     pub const WORDS_PREFIXES_FST_KEY: &str = "words-prefixes-fst";
     pub const CREATED_AT_KEY: &str = "created-at";
@@ -655,26 +651,6 @@ impl Index {
         }
     }
 
-    /* remove hidden fields */
-    pub fn remove_hidden_fields(
-        &self,
-        rtxn: &RoTxn,
-        fields: impl IntoIterator<Item = impl AsRef<str>>,
-    ) -> Result<(BTreeSet<String>, bool)> {
-        let mut valid_fields =
-            fields.into_iter().map(|f| f.as_ref().to_string()).collect::<BTreeSet<String>>();
-
-        let fields_len = valid_fields.len();
-
-        if let Some(dn) = self.displayed_fields(rtxn)? {
-            let displayable_names = dn.iter().map(|s| s.to_string()).collect();
-            valid_fields = &valid_fields & &displayable_names;
-        }
-
-        let hidden_fields = fields_len > valid_fields.len();
-        Ok((valid_fields, hidden_fields))
-    }
-
     /* searchable fields */
 
     /// Write the user defined searchable fields and generate the real searchable fields from the specified fields ids map.
@@ -1079,116 +1055,18 @@ impl Index {
         }
     }
 
-    /* non separator tokens */
-
-    pub(crate) fn put_non_separator_tokens(
-        &self,
-        wtxn: &mut RwTxn,
-        set: &BTreeSet<String>,
-    ) -> heed::Result<()> {
-        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::NON_SEPARATOR_TOKENS_KEY, set)
-    }
-
-    pub(crate) fn delete_non_separator_tokens(&self, wtxn: &mut RwTxn) -> heed::Result<bool> {
-        self.main.delete::<_, Str>(wtxn, main_key::NON_SEPARATOR_TOKENS_KEY)
-    }
-
-    pub fn non_separator_tokens(&self, rtxn: &RoTxn) -> Result<Option<BTreeSet<String>>> {
-        Ok(self.main.get::<_, Str, SerdeBincode<BTreeSet<String>>>(
-            rtxn,
-            main_key::NON_SEPARATOR_TOKENS_KEY,
-        )?)
-    }
-
-    /* separator tokens */
-
-    pub(crate) fn put_separator_tokens(
-        &self,
-        wtxn: &mut RwTxn,
-        set: &BTreeSet<String>,
-    ) -> heed::Result<()> {
-        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::SEPARATOR_TOKENS_KEY, set)
-    }
-
-    pub(crate) fn delete_separator_tokens(&self, wtxn: &mut RwTxn) -> heed::Result<bool> {
-        self.main.delete::<_, Str>(wtxn, main_key::SEPARATOR_TOKENS_KEY)
-    }
-
-    pub fn separator_tokens(&self, rtxn: &RoTxn) -> Result<Option<BTreeSet<String>>> {
-        Ok(self
-            .main
-            .get::<_, Str, SerdeBincode<BTreeSet<String>>>(rtxn, main_key::SEPARATOR_TOKENS_KEY)?)
-    }
-
-    /* separators easing method */
-
-    pub fn allowed_separators(&self, rtxn: &RoTxn) -> Result<Option<BTreeSet<String>>> {
-        let default_separators =
-            charabia::separators::DEFAULT_SEPARATORS.iter().map(|s| s.to_string());
-        let mut separators: Option<BTreeSet<_>> = None;
-        if let Some(mut separator_tokens) = self.separator_tokens(rtxn)? {
-            separator_tokens.extend(default_separators.clone());
-            separators = Some(separator_tokens);
-        }
-
-        if let Some(non_separator_tokens) = self.non_separator_tokens(rtxn)? {
-            separators = separators
-                .or_else(|| Some(default_separators.collect()))
-                .map(|separators| &separators - &non_separator_tokens);
-        }
-
-        Ok(separators)
-    }
-
-    /* dictionary */
-
-    pub(crate) fn put_dictionary(
-        &self,
-        wtxn: &mut RwTxn,
-        set: &BTreeSet<String>,
-    ) -> heed::Result<()> {
-        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::DICTIONARY_KEY, set)
-    }
-
-    pub(crate) fn delete_dictionary(&self, wtxn: &mut RwTxn) -> heed::Result<bool> {
-        self.main.delete::<_, Str>(wtxn, main_key::DICTIONARY_KEY)
-    }
-
-    pub fn dictionary(&self, rtxn: &RoTxn) -> Result<Option<BTreeSet<String>>> {
-        Ok(self
-            .main
-            .get::<_, Str, SerdeBincode<BTreeSet<String>>>(rtxn, main_key::DICTIONARY_KEY)?)
-    }
-
     /* synonyms */
 
     pub(crate) fn put_synonyms(
         &self,
         wtxn: &mut RwTxn,
         synonyms: &HashMap<Vec<String>, Vec<Vec<String>>>,
-        user_defined_synonyms: &BTreeMap<String, Vec<String>>,
     ) -> heed::Result<()> {
-        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::SYNONYMS_KEY, synonyms)?;
-        self.main.put::<_, Str, SerdeBincode<_>>(
-            wtxn,
-            main_key::USER_DEFINED_SYNONYMS_KEY,
-            user_defined_synonyms,
-        )
+        self.main.put::<_, Str, SerdeBincode<_>>(wtxn, main_key::SYNONYMS_KEY, synonyms)
     }
 
     pub(crate) fn delete_synonyms(&self, wtxn: &mut RwTxn) -> heed::Result<bool> {
-        self.main.delete::<_, Str>(wtxn, main_key::SYNONYMS_KEY)?;
-        self.main.delete::<_, Str>(wtxn, main_key::USER_DEFINED_SYNONYMS_KEY)
-    }
-
-    pub fn user_defined_synonyms(
-        &self,
-        rtxn: &RoTxn,
-    ) -> heed::Result<BTreeMap<String, Vec<String>>> {
-        Ok(self
-            .main
-            .get::<_, Str, SerdeBincode<_>>(rtxn, main_key::USER_DEFINED_SYNONYMS_KEY)?
-            .unwrap_or_default())
+        self.main.delete::<_, Str>(wtxn, main_key::SYNONYMS_KEY)
     }
 
     pub fn synonyms(&self, rtxn: &RoTxn) -> heed::Result<HashMap<Vec<String>, Vec<Vec<String>>>> {

milli/src/proximity.rs

@@ -2,7 +2,7 @@ use std::cmp;
 
 use crate::{relative_from_absolute_position, Position};
 
-pub const MAX_DISTANCE: u32 = 4;
+pub const MAX_DISTANCE: u32 = 8;
 
 pub fn index_proximity(lhs: u32, rhs: u32) -> u32 {
     if lhs <= rhs {

milli/src/search/facet/filter.rs

@@ -3,7 +3,7 @@ use std::fmt::{Debug, Display};
 use std::ops::Bound::{self, Excluded, Included};
 
 use either::Either;
-pub use filter_parser::{Condition, Error as FPError, FilterCondition, Token};
+pub use filter_parser::{Condition, Error as FPError, FilterCondition, Span, Token};
 use roaring::RoaringBitmap;
 use serde_json::Value;
 

milli/src/search/mod.rs

@@ -11,7 +11,7 @@ use once_cell::sync::Lazy;
 use roaring::bitmap::RoaringBitmap;
 
 pub use self::facet::{FacetDistribution, Filter, OrderBy, DEFAULT_VALUES_PER_FACET};
-pub use self::new::matches::{FormatOptions, MatchBounds, MatcherBuilder, MatchingWords};
+pub use self::new::matches::{FormatOptions, MatchBounds, Matcher, MatcherBuilder, MatchingWords};
 use self::new::PartialSearchResult;
 use crate::error::UserError;
 use crate::heed_codec::facet::{FacetGroupKey, FacetGroupValue};
@@ -280,13 +280,9 @@ impl<'a> SearchForFacetValues<'a> {
 
         let filterable_fields = index.filterable_fields(rtxn)?;
         if !filterable_fields.contains(&self.facet) {
-            let (valid_fields, hidden_fields) =
-                index.remove_hidden_fields(rtxn, filterable_fields)?;
-
             return Err(UserError::InvalidFacetSearchFacetName {
                 field: self.facet.clone(),
-                valid_fields,
-                hidden_fields,
+                valid_fields: filterable_fields.into_iter().collect(),
             }
             .into());
         }