Update Dockerfile

Co-authored-by: Markus Machatschek <markus.machatschek@hey.com>

.cargo/config.toml

@@ -1,2 +0,0 @@
-[alias]
-xtask = "run --release --package xtask --"

.github/ISSUE_TEMPLATE/bug_report.md

@@ -23,8 +23,7 @@ A clear and concise description of what you expected to happen.
 **Screenshots**
 If applicable, add screenshots to help explain your problem.
 
-**Meilisearch version:**
-[e.g. v0.20.0]
+**Meilisearch version:** [e.g. v0.20.0]
 
 **Additional context**
 Additional information that may be relevant to the issue.

.github/ISSUE_TEMPLATE/sprint_issue.md

@@ -1,44 +0,0 @@
----
-name: New sprint issue
-about: ⚠️ Should only be used by the engine team ⚠️
-title: ''
-labels: 'missing usage in PRD, impacts docs'
-assignees: ''
-
----
-
-Related product team resources: [PRD]() (_internal only_)
-Related product discussion:
-
-## Motivation
-
-<!---Copy/paste the information in PRD or briefly detail the product motivation. Ask product team if any hesitation.-->
-
-## Usage
-
-<!---Link to the public part of the PRD, or to the related product discussion for experimental features-->
-
-## TODO
-
-<!---If necessary, create a list with technical/product steps-->
-
-### Reminders when modifying the Setting API
-
-<!--- Special steps to remind when adding a new index setting -->
-
-- [ ] Ensure the new setting route is at least tested by the [`test_setting_routes` macro](https://github.com/meilisearch/meilisearch/blob/5204c0b60b384cbc79621b6b2176fca086069e8e/meilisearch/tests/settings/get_settings.rs#L276)
-- [ ] Ensure Analytics are fully implemented
-  - [ ] `/settings/my-new-setting` configurated in the [`make_setting_routes` macro](https://github.com/meilisearch/meilisearch/blob/5204c0b60b384cbc79621b6b2176fca086069e8e/meilisearch/src/routes/indexes/settings.rs#L141-L165)
-  - [ ] global `/settings` route configurated in the [`update_all` function](https://github.com/meilisearch/meilisearch/blob/5204c0b60b384cbc79621b6b2176fca086069e8e/meilisearch/src/routes/indexes/settings.rs#L655-L751)
-- [ ] Ensure the dump serializing is consistent with the `/settings` route serializing, e.g., enums case can be different (`camelCase` in route and `PascalCase` in the dump)
-
-#### Special cases when adding a setting for an experimental feature
-
-- [ ] ⚠️ API stability: The setting does not appear on the main settings route when the feature has never been enabled (e.g. mark it `Unset` when returned from the index in this situation. See [an example](https://github.com/meilisearch/meilisearch/blob/7a89abd2a025606a42f8b219e539117eb2eb029f/meilisearch-types/src/settings.rs#L608))
-- [ ] The setting cannot be set when the feature is disabled, either by the main settings route or the subroute (see [`validate_settings` function](https://github.com/meilisearch/meilisearch/blob/7a89abd2a025606a42f8b219e539117eb2eb029f/meilisearch/src/routes/indexes/settings.rs#L811))
-- [ ] If possible, the setting is reset when the feature is disabled (hard if it requires reindexing)
-
-## Impacted teams
-
-<!---Ping the related teams. Ask for the engine manager if any hesitation-->
-<!---@meilisearch/docs-team when there is any API change, e.g. settings addition-->

.github/scripts/check-release.sh

@@ -1,41 +1,24 @@
-#!/usr/bin/env bash
-set -eu -o pipefail
+#!/bin/bash
 
-check_tag() {
-    local expected=$1
-    local actual=$2
-    local filename=$3
-
-    if [[ $actual != $expected ]]; then
-        echo >&2 "Error: the current tag does not match the version in $filename: found $actual, expected $expected"
-        return 1
-    fi
+# check_tag $current_tag $file_tag $file_name
+function check_tag {
+  if [[ "$1" != "$2" ]]; then
+      echo "Error: the current tag does not match the version in Cargo.toml: found $2 - expected $1"
+      ret=1
+  fi
 }
 
-read_version() {
-    grep '^version = ' | cut -d \" -f 2
-}
-
-if [[ -z "${GITHUB_REF:-}" ]]; then
-    echo >&2 "Error: GITHUB_REF is not set"
-    exit 1
-fi
-
-if [[ ! "$GITHUB_REF" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+(-[a-z0-9]+)?$ ]]; then
-    echo >&2 "Error: GITHUB_REF is not a valid tag: $GITHUB_REF"
-    exit 1
-fi
-
-current_tag=${GITHUB_REF#refs/tags/v}
 ret=0
+current_tag=${GITHUB_REF#'refs/tags/v'}
 
-toml_tag="$(cat Cargo.toml | read_version)"
-check_tag "$current_tag" "$toml_tag" Cargo.toml || ret=1
+file_tag="$(grep '^version = ' Cargo.toml | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')"
+check_tag $current_tag $file_tag
 
-lock_tag=$(grep -A 1 '^name = "meilisearch-auth"' Cargo.lock | read_version)
-check_tag "$current_tag" "$lock_tag" Cargo.lock || ret=1
+lock_file='Cargo.lock'
+lock_tag=$(grep -A 1 'name = "meilisearch-auth"' $lock_file | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')
+check_tag $current_tag $lock_tag $lock_file
 
-if (( ret == 0 )); then
-    echo 'OK'
+if [[ "$ret" -eq 0 ]] ; then
+  echo 'OK'
 fi
 exit $ret

.github/uffizzi/Dockerfile

@@ -0,0 +1,48 @@
+# Compile
+FROM    rust:alpine3.16 AS compiler
+
+RUN     apk add -q --update-cache --no-cache build-base openssl-dev
+
+WORKDIR /meilisearch
+
+ARG     COMMIT_SHA
+ARG     COMMIT_DATE
+ARG     GIT_TAG
+ENV     COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE} VERGEN_GIT_SEMVER_LIGHTWEIGHT=${GIT_TAG}
+ENV     RUSTFLAGS="-C target-feature=-crt-static"
+
+COPY    . .
+RUN     set -eux; \
+        apkArch="$(apk --print-arch)"; \
+        if [ "$apkArch" = "aarch64" ]; then \
+            export JEMALLOC_SYS_WITH_LG_PAGE=16; \
+        fi && \
+        cargo build --release
+
+# Run
+FROM    uffizzi/ttyd:alpine
+
+ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+ENV     MEILI_SERVER_PROVIDER docker
+ENV     MEILI_NO_ANALYTICS true
+
+RUN     apk update --quiet \
+        && apk add -q --no-cache libgcc tini curl
+
+# add meilisearch to the `/bin` so you can run it from anywhere and it's easy
+# to find.
+COPY    --from=compiler /meilisearch/target/release/meilisearch /bin/meilisearch
+# To stay compatible with the older version of the container (pre v0.27.0) we're
+# going to symlink the meilisearch binary in the path to `/meilisearch`
+RUN     ln -s /bin/meilisearch /meilisearch
+
+# This directory should hold all the data related to meilisearch so we're going
+# to move our PWD in there.
+# We don't want to put the meilisearch binary
+WORKDIR /meili_data
+
+
+EXPOSE  7700/tcp
+
+ENTRYPOINT ["tini", "--"]
+CMD     ["ttyd", "/bin/zsh"]

.github/uffizzi/docker-compose.uffizzi.yml

@@ -0,0 +1,26 @@
+version: "3"
+
+x-uffizzi:
+  ingress:
+    service: nginx
+    port: 8081
+
+services:
+  meilisearch:
+    image: "${MEILISEARCH_IMAGE}"
+    restart: unless-stopped
+    ports:
+    - "7681:7681"
+    - "7700:7700"
+    deploy:
+      resources:
+        limits:
+          memory: 500M
+  
+  nginx:
+    image: nginx:alpine
+    restart: unless-stopped
+    ports:
+    - "8081:8081"
+    volumes:
+    - ./.github/uffizzi/nginx:/etc/nginx

.github/uffizzi/nginx/nginx.conf

@@ -0,0 +1,28 @@
+
+events {
+  worker_connections  4096;  ## Default: 1024
+}
+
+http {
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        '' close;
+    }
+
+    server {
+        listen 8081;
+
+        location / {
+            proxy_pass http://localhost:7681;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection $connection_upgrade;
+        }
+
+        location /meilisearch/ {
+            # rewrite /meilisearch/(.*) /$1  break;
+            proxy_pass http://localhost:7700/;
+        }
+    }
+}
+

.github/workflows/bench-manual.yml

@@ -1,28 +0,0 @@
-name: Bench (manual)
-
-on:
-    workflow_dispatch:
-        inputs:
-            workload:
-                description: 'The path to the workloads to execute (workloads/...)'
-                required: true
-                default: 'workloads/movies.json'
-
-env:
-    WORKLOAD_NAME: ${{ github.event.inputs.workload }}
-
-jobs:
-    benchmarks:
-        name: Run and upload benchmarks
-        runs-on: benchmarks
-        timeout-minutes: 180 # 3h
-        steps:
-            - uses: actions/checkout@v3
-            - uses: dtolnay/rust-toolchain@1.79
-              with:
-                profile: minimal
-
-            - name: Run benchmarks - workload ${WORKLOAD_NAME} - branch ${{ github.ref }} - commit ${{ github.sha }}
-              run: |
-               cargo xtask bench --api-key "${{ secrets.BENCHMARK_API_KEY }}" --dashboard-url "${{ vars.BENCHMARK_DASHBOARD_URL }}" --reason "Manual [Run #${{ github.run_id }}](https://github.com/meilisearch/meilisearch/actions/runs/${{ github.run_id }})" -- ${WORKLOAD_NAME}
-

.github/workflows/bench-pr.yml

@@ -1,82 +0,0 @@
-name: Bench (PR)
-on:
-    issue_comment:
-        types: [created]
-
-permissions:
-    issues: write
-
-env:
-    GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-
-jobs:
-    run-benchmarks-on-comment:
-      if: startsWith(github.event.comment.body, '/bench')
-      name: Run and upload benchmarks
-      runs-on: benchmarks
-      timeout-minutes: 180 # 3h
-      steps:
-        - name: Check permissions
-          id: permission
-          env:
-            PR_AUTHOR: ${{github.event.issue.user.login }}
-            COMMENT_AUTHOR: ${{github.event.comment.user.login }}
-            REPOSITORY: ${{github.repository}}
-            PR_ID: ${{github.event.issue.number}}
-          run: |
-            PR_REPOSITORY=$(gh api /repos/"$REPOSITORY"/pulls/"$PR_ID" --jq .head.repo.full_name)
-            if $(gh api /repos/"$REPOSITORY"/collaborators/"$PR_AUTHOR"/permission --jq .user.permissions.push)
-            then
-              echo "::notice title=Authentication success::PR author authenticated"
-            else
-              echo "::error title=Authentication error::PR author doesn't have push permission on this repository"
-              exit 1
-            fi
-            if $(gh api /repos/"$REPOSITORY"/collaborators/"$COMMENT_AUTHOR"/permission --jq .user.permissions.push)
-            then
-              echo "::notice title=Authentication success::Comment author authenticated"
-            else
-              echo "::error title=Authentication error::Comment author doesn't have push permission on this repository"
-              exit 1
-            fi
-            if [ "$PR_REPOSITORY" = "$REPOSITORY" ]
-            then
-              echo "::notice title=Authentication success::PR started from main repository"
-            else
-              echo "::error title=Authentication error::PR started from a fork"
-              exit 1
-            fi
-
-        - name: Check for Command
-          id: command
-          uses: xt0rted/slash-command-action@v2
-          with:
-              command: bench
-              reaction-type: "rocket"
-              repo-token: ${{ env.GH_TOKEN }}
-
-        - uses: xt0rted/pull-request-comment-branch@v2
-          id: comment-branch
-          with:
-            repo_token: ${{ env.GH_TOKEN }}
-
-        - uses: actions/checkout@v3
-          if: success()
-          with:
-            fetch-depth: 0 # fetch full history to be able to get main commit sha
-            ref: ${{ steps.comment-branch.outputs.head_ref }}
-
-        - uses: dtolnay/rust-toolchain@1.79
-          with:
-            profile: minimal
-
-        - name: Run benchmarks on PR ${{ github.event.issue.id }}
-          run: |
-            cargo xtask bench --api-key "${{ secrets.BENCHMARK_API_KEY }}" \
-               --dashboard-url "${{ vars.BENCHMARK_DASHBOARD_URL }}" \
-               --reason "[Comment](${{ github.event.comment.html_url }}) on [#${{ github.event.issue.number }}](${{ github.event.issue.html_url }})" \
-               -- ${{ steps.command.outputs.command-arguments }} > benchlinks.txt
-
-        - name: Send comment in PR
-          run: |
-            gh pr comment ${{github.event.issue.number}} --body-file benchlinks.txt

.github/workflows/bench-push-indexing.yml

@@ -1,23 +0,0 @@
-name: Indexing bench (push)
-
-on:
-    push:
-        branches:
-            - main
-
-jobs:
-    benchmarks:
-        name: Run and upload benchmarks
-        runs-on: benchmarks
-        timeout-minutes: 180 # 3h
-        steps:
-          - uses: actions/checkout@v3
-          - uses: dtolnay/rust-toolchain@1.79
-            with:
-              profile: minimal
-
-          # Run benchmarks
-          - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch main - Commit ${{ github.sha }}
-            run: |
-              cargo xtask bench --api-key "${{ secrets.BENCHMARK_API_KEY }}" --dashboard-url "${{ vars.BENCHMARK_DASHBOARD_URL }}" --reason "Push on `main` [Run #${{ github.run_id }}](https://github.com/meilisearch/meilisearch/actions/runs/${{ github.run_id }})" -- workloads/*.json
-

.github/workflows/benchmarks-pr.yml

@@ -1,127 +0,0 @@
-name: Benchmarks (PR)
-on: issue_comment
-permissions:
-  issues: write
-
-env:
-  GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-
-jobs:
-  run-benchmarks-on-comment:
-    if: startsWith(github.event.comment.body, '/benchmark')
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    timeout-minutes: 4320 # 72h
-    steps:
-      - name: Check permissions
-        id: permission
-        env:
-          PR_AUTHOR: ${{github.event.issue.user.login }}
-          COMMENT_AUTHOR: ${{github.event.comment.user.login }}
-          REPOSITORY: ${{github.repository}}
-          PR_ID: ${{github.event.issue.number}}
-        run: |
-          PR_REPOSITORY=$(gh api /repos/"$REPOSITORY"/pulls/"$PR_ID" --jq .head.repo.full_name)
-          if $(gh api /repos/"$REPOSITORY"/collaborators/"$PR_AUTHOR"/permission --jq .user.permissions.push)
-          then
-            echo "::notice title=Authentication success::PR author authenticated"
-          else
-            echo "::error title=Authentication error::PR author doesn't have push permission on this repository"
-            exit 1
-          fi
-          if $(gh api /repos/"$REPOSITORY"/collaborators/"$COMMENT_AUTHOR"/permission --jq .user.permissions.push)
-          then
-            echo "::notice title=Authentication success::Comment author authenticated"
-          else
-            echo "::error title=Authentication error::Comment author doesn't have push permission on this repository"
-            exit 1
-          fi
-          if [ "$PR_REPOSITORY" = "$REPOSITORY" ]
-          then
-            echo "::notice title=Authentication success::PR started from main repository"
-          else
-            echo "::error title=Authentication error::PR started from a fork"
-            exit 1
-          fi
-
-      - uses: dtolnay/rust-toolchain@1.79
-        with:
-          profile: minimal
-
-      - name: Check for Command
-        id: command
-        uses: xt0rted/slash-command-action@v2
-        with:
-          command: benchmark
-          reaction-type: "eyes"
-          repo-token: ${{ env.GH_TOKEN }}
-
-      - uses: xt0rted/pull-request-comment-branch@v2
-        id: comment-branch
-        with:
-          repo_token: ${{ env.GH_TOKEN }}
-
-      - uses: actions/checkout@v3
-        if: success()
-        with:
-          fetch-depth: 0 # fetch full history to be able to get main commit sha
-          ref: ${{ steps.comment-branch.outputs.head_ref }}
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(git rev-parse --abbrev-ref HEAD | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${{ steps.command.outputs.command-arguments }}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${{ steps.command.outputs.command-arguments }} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${{ steps.command.outputs.command-arguments }} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Compute the diff of the benchmarks and send a message on the GitHub PR
-      - name: Compute and send a message in the PR
-        env:
-          GITHUB_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-        run: |
-          set -x
-          export base_ref=$(git merge-base origin/main ${{ steps.comment-branch.outputs.head_ref }} | head -c8)
-          export base_filename=$(echo ${{ steps.command.outputs.command-arguments }}_main_${base_ref}.json)
-          export bench_name=$(echo ${{ steps.command.outputs.command-arguments }})
-          echo "Here are your $bench_name benchmarks diff 👊" >> body.txt
-          echo '```' >> body.txt
-          ./benchmarks/scripts/compare.sh $base_filename ${{ steps.file.outputs.basename }}.json >> body.txt
-          echo '```' >> body.txt
-          gh pr comment ${{ steps.current_branch.outputs.name }} --body-file body.txt

.github/workflows/create-issue-dependencies.yml

@@ -0,0 +1,28 @@
+name: Create issue to upgrade dependencies
+on:
+  schedule:
+    # Run the first of the month, every 3 month
+    - cron: '0 0 1 */3 *'
+  workflow_dispatch:
+
+jobs:
+  create-issue:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Create an issue
+      uses: actions-ecosystem/action-create-issue@v1
+      with:
+        github_token: ${{ secrets.MEILI_BOT_GH_PAT }}
+        title: Upgrade dependencies
+        body: |
+          This issue is about updating Meilisearch dependencies:
+          - [ ] Cargo toml dependencies of Meilisearch; but also the main engine-team repositories that Meilisearch depends on (charabia, heed...)
+          - [ ] If new Rust versions have been released, update the Rust version in the Clippy job of this [GitHub Action file](./.github/workflows/rust.yml)
+
+          ⚠️ To avoid last minute bugs, this issue should only be done at the beginning of the sprint!
+
+          The GitHub action dependencies are managed by [Dependabot](./.github/dependabot.yml)
+        labels: |
+          dependencies
+          maintenance

.github/workflows/dependency-issue.yml

@@ -1,24 +0,0 @@
-name: Create issue to upgrade dependencies
-
-on:
-  schedule:
-    # Run the first of the month, every 6 month
-    - cron: '0 0 1 */6 *'
-  workflow_dispatch:
-
-jobs:
-  create-issue:
-    runs-on: ubuntu-latest
-    env:
-      ISSUE_TEMPLATE: issue-template.md
-      GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-    steps:
-    - uses: actions/checkout@v3
-    - name: Download the issue template
-      run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/dependency-issue.md > $ISSUE_TEMPLATE
-    - name: Create issue
-      run: |
-        gh issue create \
-          --title 'Upgrade dependencies' \
-          --label 'dependencies,maintenance' \
-          --body-file $ISSUE_TEMPLATE

.github/workflows/flaky.yml

@@ -1,5 +1,4 @@
 name: Look for flaky tests
-
 on:
   workflow_dispatch:
   schedule:
@@ -9,15 +8,18 @@ jobs:
   flaky:
     runs-on: ubuntu-latest
     container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
     steps:
     - uses: actions/checkout@v3
     - name: Install needed dependencies
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: dtolnay/rust-toolchain@1.79
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
     - name: Install cargo-flaky
       run: cargo install cargo-flaky
     - name: Run cargo flaky in the dumps

.github/workflows/fuzzer-indexing.yml

@@ -1,22 +0,0 @@
-name: Run the indexing fuzzer
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  fuzz:
-    name: Setup the action
-    runs-on: ubuntu-latest
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
-        with:
-          profile: minimal
-
-      # Run benchmarks
-      - name: Run the fuzzer
-        run: |
-          cargo run --release --bin fuzz-indexing

.github/workflows/manual_benchmarks.yml

@@ -1,4 +1,4 @@
-name: Benchmarks (manual)
+name: Benchmarks
 
 on:
   workflow_dispatch:
@@ -18,9 +18,11 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
+          toolchain: stable
+          override: true
 
       # Set variables
       - name: Set current branch name
@@ -72,4 +74,4 @@ jobs:
           echo "${{ steps.file.outputs.basename }}.json has just been pushed."
           echo 'How to compare this benchmark with another one?'
           echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scripts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"
+          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/milestone-workflow.yml

@@ -110,44 +110,6 @@ jobs:
             --milestone $MILESTONE_VERSION \
             --assignee curquiza
 
-  create-update-version-issue:
-    needs: get-release-version
-    # Create the update-version issue even if the release is a patch release
-    if: github.event.action == 'created'
-    runs-on: ubuntu-latest
-    env:
-      ISSUE_TEMPLATE: issue-template.md
-    steps:
-      - uses: actions/checkout@v3
-      - name: Download the issue template
-        run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/update-version-issue.md > $ISSUE_TEMPLATE
-      - name: Create the issue
-        run: |
-          gh issue create \
-            --title "Update version in Cargo.toml for $MILESTONE_VERSION" \
-            --label 'maintenance' \
-            --body-file $ISSUE_TEMPLATE \
-            --milestone $MILESTONE_VERSION
-
-  create-update-openapi-issue:
-    needs: get-release-version
-    # Create the openAPI issue if the release is not only a patch release
-    if: github.event.action == 'created' && needs.get-release-version.outputs.is-patch == 'false'
-    runs-on: ubuntu-latest
-    env:
-      ISSUE_TEMPLATE: issue-template.md
-    steps:
-      - uses: actions/checkout@v3
-      - name: Download the issue template
-        run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/update-openapi-issue.md > $ISSUE_TEMPLATE
-      - name: Create the issue
-        run: |
-          gh issue create \
-            --title "Update Open API file for $MILESTONE_VERSION" \
-            --label 'maintenance' \
-            --body-file $ISSUE_TEMPLATE \
-            --milestone $MILESTONE_VERSION
-
 # ----------------
 # MILESTONE CLOSED
 # ----------------

.github/workflows/publish-binaries.yml

@@ -1,5 +1,3 @@
-name: Publish binaries to GitHub release
-
 on:
   workflow_dispatch:
   schedule:
@@ -7,6 +5,8 @@ on:
   release:
     types: [published]
 
+name: Publish binaries to release
+
 jobs:
   check-version:
     name: Check the version validity
@@ -37,21 +37,24 @@ jobs:
     runs-on: ubuntu-latest
     needs: check-version
     container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
+      # Use ubuntu-18.04 to compile with glibc 2.27
+      image: ubuntu:18.04
     steps:
     - uses: actions/checkout@v3
     - name: Install needed dependencies
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: dtolnay/rust-toolchain@1.79
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
     - name: Build
       run: cargo build --release --locked
     # No need to upload binaries for dry run (cron)
     - name: Upload binaries to release
       if: github.event_name == 'release'
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.4.0
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/release/meilisearch
@@ -75,13 +78,16 @@ jobs:
             asset_name: meilisearch-windows-amd64.exe
     steps:
     - uses: actions/checkout@v3
-    - uses: dtolnay/rust-toolchain@1.79
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
     - name: Build
       run: cargo build --release --locked
     # No need to upload binaries for dry run (cron)
     - name: Upload binaries to release
       if: github.event_name == 'release'
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.4.0
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/release/${{ matrix.artifact_name }}
@@ -90,21 +96,25 @@ jobs:
 
   publish-macos-apple-silicon:
     name: Publish binary for macOS silicon
-    runs-on: macos-12
+    runs-on: ${{ matrix.os }}
     needs: check-version
     strategy:
+      fail-fast: false
       matrix:
         include:
-          - target: aarch64-apple-darwin
+          - os: macos-12
+            target: aarch64-apple-darwin
             asset_name: meilisearch-macos-apple-silicon
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
       - name: Installing Rust toolchain
-        uses: dtolnay/rust-toolchain@1.79
+        uses: actions-rs/toolchain@v1
         with:
+          toolchain: stable
           profile: minimal
           target: ${{ matrix.target }}
+          override: true
       - name: Cargo build
         uses: actions-rs/cargo@v1
         with:
@@ -113,7 +123,7 @@ jobs:
       - name: Upload the binary to release
         # No need to upload binaries for dry run (cron)
         if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.4.0
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/${{ matrix.target }}/release/meilisearch
@@ -122,37 +132,37 @@ jobs:
 
   publish-aarch64:
     name: Publish binary for aarch64
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     needs: check-version
-    env:
-      DEBIAN_FRONTEND: noninteractive
-    container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
     strategy:
+      fail-fast: false
       matrix:
         include:
-          - target: aarch64-unknown-linux-gnu
+          - build: aarch64
+            os: ubuntu-18.04
+            target: aarch64-unknown-linux-gnu
+            linker: gcc-aarch64-linux-gnu
+            use-cross: true
             asset_name: meilisearch-linux-aarch64
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update -y && apt upgrade -y
-          apt-get install -y curl build-essential gcc-aarch64-linux-gnu
-      - name: Set up Docker for cross compilation
-        run: |
-          apt-get install -y curl apt-transport-https ca-certificates software-properties-common
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
-          add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-          apt-get update -y && apt-get install -y docker-ce
       - name: Installing Rust toolchain
-        uses: dtolnay/rust-toolchain@1.79
+        uses: actions-rs/toolchain@v1
         with:
+          toolchain: stable
           profile: minimal
           target: ${{ matrix.target }}
+          override: true
+      - name: APT update
+        run: |
+          sudo apt update
+      - name: Install target specific tools
+        if: matrix.use-cross
+        run: |
+          sudo apt-get install -y ${{ matrix.linker }}
       - name: Configure target aarch64 GNU
+        if: matrix.target == 'aarch64-unknown-linux-gnu'
         ## Environment variable is not passed using env:
         ## LD gold won't work with MUSL
         # env:
@@ -162,23 +172,18 @@ jobs:
           echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
           echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
           echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
-      - name: Install a default toolchain that will be used to build cargo cross
-        run: |
-          rustup default stable
       - name: Cargo build
         uses: actions-rs/cargo@v1
         with:
           command: build
-          use-cross: true
+          use-cross: ${{ matrix.use-cross }}
           args: --release --target ${{ matrix.target }}
-        env:
-          CROSS_DOCKER_IN_DOCKER: true
       - name: List target output files
         run: ls -lR ./target
       - name: Upload the binary to release
         # No need to upload binaries for dry run (cron)
         if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.4.0
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/${{ matrix.target }}/release/meilisearch

.github/workflows/publish-deb-brew-pkg.yml

@@ -1,4 +1,4 @@
-name: Publish to APT & Homebrew
+name: Publish to APT repository & Homebrew
 
 on:
   release:
@@ -18,21 +18,24 @@ jobs:
     runs-on: ubuntu-latest
     needs: check-version
     container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
+      # Use ubuntu-18.04 to compile with glibc 2.27
+      image: ubuntu:18.04
     steps:
     - name: Install needed dependencies
       run: |
         apt-get update && apt-get install -y curl
         apt-get install build-essential -y
-    - uses: dtolnay/rust-toolchain@1.79
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
     - name: Install cargo-deb
       run: cargo install cargo-deb
     - uses: actions/checkout@v3
     - name: Build deb package
       run: cargo deb -p meilisearch -o target/debian/meilisearch.deb
     - name: Upload debian pkg to release
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.4.0
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/debian/meilisearch.deb
@@ -47,9 +50,8 @@ jobs:
     needs: check-version
     steps:
       - name: Create PR to Homebrew
-        uses: mislav/bump-homebrew-formula-action@v3
+        uses: mislav/bump-homebrew-formula-action@v2
         with:
           formula-name: meilisearch
-          formula-path: Formula/m/meilisearch.rb
         env:
           COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }}

.github/workflows/publish-docker-images.yml

@@ -1,5 +1,4 @@
-name: Publish images to Docker Hub
-
+---
 on:
   push:
     # Will run for every tag pushed except `latest`
@@ -13,6 +12,8 @@ on:
     - cron: '0 23 * * *' # Every day at 11:00pm
   workflow_dispatch:
 
+name: Publish tagged images to Docker Hub
+
 jobs:
   docker:
     runs-on: docker
@@ -57,20 +58,20 @@ jobs:
           echo "date=$commit_date" >> $GITHUB_OUTPUT
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to Docker Hub
-        uses: docker/login-action@v3
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v4
         with:
           images: getmeili/meilisearch
           # Prevent `latest` to be updated for each new tag pushed.
@@ -80,11 +81,10 @@ jobs:
             type=ref,event=tag
             type=raw,value=nightly,enable=${{ github.event_name != 'push' }}
             type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
-            type=semver,pattern=v{{major}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
             type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
 
       - name: Build and push
-        uses: docker/build-push-action@v6
+        uses: docker/build-push-action@v4
         with:
           push: true
           platforms: linux/amd64,linux/arm64
@@ -98,7 +98,7 @@ jobs:
       - name: Send CI information to Cloud team
         # Do not send if nightly build (i.e. 'schedule' or 'workflow_dispatch' event)
         if: github.event_name == 'push'
-        uses: peter-evans/repository-dispatch@v3
+        uses: peter-evans/repository-dispatch@v2
         with:
           token: ${{ secrets.MEILI_BOT_GH_PAT }}
           repository: meilisearch/meilisearch-cloud

.github/workflows/push_benchmarks_indexing.yml

@@ -1,4 +1,4 @@
-name: Benchmarks of indexing (push)
+name: Benchmarks indexing (push)
 
 on:
   push:
@@ -16,9 +16,11 @@ jobs:
     timeout-minutes: 4320 # 72h
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
+          toolchain: stable
+          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/push_benchmarks_search_geo.yml

@@ -1,4 +1,4 @@
-name: Benchmarks of search for geo (push)
+name: Benchmarks search geo (push)
 
 on:
   push:
@@ -15,9 +15,11 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
+          toolchain: stable
+          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/push_benchmarks_search_songs.yml

@@ -1,4 +1,4 @@
-name: Benchmarks of search for songs (push)
+name: Benchmarks search songs (push)
 
 on:
   push:
@@ -15,9 +15,11 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
+          toolchain: stable
+          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/push_benchmarks_search_wiki.yml

@@ -1,4 +1,4 @@
-name: Benchmarks of search for Wikipedia articles (push)
+name: Benchmarks search wikipedia articles (push)
 
 on:
   push:
@@ -15,9 +15,11 @@ jobs:
     runs-on: benchmarks
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
+          toolchain: stable
+          override: true
 
       # Set variables
       - name: Set current branch name

.github/workflows/rust.yml

@@ -0,0 +1,145 @@
+name: Rust
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Everyday at 5:00am
+    - cron: '0 5 * * *'
+  pull_request:
+  push:
+    # trying and staging branches are for Bors config
+    branches:
+      - trying
+      - staging
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  RUSTFLAGS: "-D warnings"
+
+jobs:
+  test-linux:
+    name: Tests on ubuntu-18.04
+    runs-on: ubuntu-latest
+    container:
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install needed dependencies
+      run: |
+        apt-get update && apt-get install -y curl
+        apt-get install build-essential -y
+    - name: Run test with Rust stable
+      if: github.event_name != 'schedule'
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
+    - name: Run test with Rust nightly
+      if: github.event_name == 'schedule'
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: nightly
+        override: true
+    # Disable cache due to disk space issues with Windows workers in CI
+    # - name: Cache dependencies
+    #   uses: Swatinem/rust-cache@v2.2.0
+    - name: Run cargo check without any default features
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --locked --release --no-default-features --all
+    - name: Run cargo test
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --locked --release --all
+
+  test-others:
+    name: Tests on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-12, windows-2022]
+    steps:
+    - uses: actions/checkout@v3
+#     - name: Cache dependencies
+#       uses: Swatinem/rust-cache@v2.2.0
+    - name: Run cargo check without any default features
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --locked --release --no-default-features --all
+    - name: Run cargo test
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --locked --release --all
+
+  # We run tests in debug also, to make sure that the debug_assertions are hit
+  test-debug:
+    name: Run tests in debug
+    runs-on: ubuntu-latest
+    container:
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install needed dependencies
+        run: |
+          apt-get update && apt-get install -y curl
+          apt-get install build-essential -y
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+      # - name: Cache dependencies
+      #   uses: Swatinem/rust-cache@v2.2.0
+      - name: Run tests in debug
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --locked --all
+
+  clippy:
+    name: Run Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.67.0
+          override: true
+          components: clippy
+      # - name: Cache dependencies
+      #   uses: Swatinem/rust-cache@v2.2.0
+      - name: Run cargo clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          # allow unlined_format_args https://github.com/rust-lang/rust-clippy/issues/10087
+          args: --all-targets -- --deny warnings --allow clippy::uninlined_format_args
+
+  fmt:
+    name: Run Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+      # - name: Cache dependencies
+      #   uses: Swatinem/rust-cache@v2.2.0
+      - name: Run cargo fmt
+        # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
+        # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate
+        # we are going to create an empty file where rustfmt expects it.
+        run: |
+          echo -ne "\n" > benchmarks/benches/datasets_paths.rs
+          cargo fmt --all -- --check

.github/workflows/sdks-tests.yml

@@ -1,386 +0,0 @@
-# If any test fails, the engine team should ensure the "breaking" changes are expected and contact the integration team
-name: SDKs tests
-
-on:
-  workflow_dispatch:
-    inputs:
-      docker_image:
-        description: 'The Meilisearch Docker image used'
-        required: false
-        default: nightly
-  schedule:
-    - cron: "0 6 * * MON" # Every Monday at 6:00AM
-
-env:
-  MEILI_MASTER_KEY: 'masterKey'
-  MEILI_NO_ANALYTICS: 'true'
-  DISABLE_COVERAGE: 'true'
-
-jobs:
-  define-docker-image:
-    runs-on: ubuntu-latest
-    outputs:
-      docker-image: ${{ steps.define-image.outputs.docker-image }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Define the Docker image we need to use
-        id: define-image
-        run: |
-          event=${{ github.event_name }}
-          echo "docker-image=nightly" >> $GITHUB_OUTPUT
-          if [[ $event == 'workflow_dispatch' ]]; then
-            echo "docker-image=${{ github.event.inputs.docker_image }}" >> $GITHUB_OUTPUT
-          fi
-      - name: Docker image is ${{ steps.define-image.outputs.docker-image }}
-        run: echo "Docker image is ${{ steps.define-image.outputs.docker-image }}"
-
-##########
-## SDKs ##
-##########
-
-  meilisearch-dotnet-tests:
-    needs: define-docker-image
-    name: .NET SDK tests
-    runs-on: ubuntu-latest
-    env:
-      MEILISEARCH_VERSION: ${{ needs.define-docker-image.outputs.docker-image }}
-
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-dotnet
-      - name: Setup .NET Core
-        uses: actions/setup-dotnet@v4
-        with:
-          dotnet-version: "6.0.x"
-      - name: Install dependencies
-        run: dotnet restore
-      - name: Build
-        run: dotnet build --configuration Release --no-restore
-      - name: Meilisearch (latest version) setup with Docker
-        run: docker compose up -d
-      - name: Run tests
-        run: dotnet test --no-restore --verbosity normal
-
-  meilisearch-dart-tests:
-    needs: define-docker-image
-    name: Dart SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-dart
-      - uses: dart-lang/setup-dart@v1
-        with:
-          sdk: 'latest'
-      - name: Install dependencies
-        run: dart pub get
-      - name: Run integration tests
-        run: dart test --concurrency=4
-
-  meilisearch-go-tests:
-    needs: define-docker-image
-    name: Go SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v5
-        with:
-          go-version: stable
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-go
-      - name: Get dependencies
-        run: |
-          go get -v -t -d ./...
-          if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-          fi
-      - name: Run integration tests
-        run: go test -v ./...
-
-  meilisearch-java-tests:
-    needs: define-docker-image
-    name: Java SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-java
-      - name: Set up Java
-        uses: actions/setup-java@v4
-        with:
-          java-version: 8
-          distribution: 'zulu'
-          cache: gradle
-      - name: Grant execute permission for gradlew
-        run: chmod +x gradlew
-      - name: Build and run unit and integration tests
-        run: ./gradlew build integrationTest
-
-  meilisearch-js-tests:
-    needs: define-docker-image
-    name: JS SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-js
-      - name: Setup node
-        uses: actions/setup-node@v4
-        with:
-          cache: 'yarn'
-      - name: Install dependencies
-        run: yarn --dev
-      - name: Run tests
-        run: yarn test
-      - name: Build project
-        run: yarn build
-      - name: Run ESM env
-        run: yarn test:env:esm
-      - name: Run Node.js env
-        run: yarn test:env:nodejs
-      - name: Run node typescript env
-        run: yarn test:env:node-ts
-      - name: Run Browser env
-        run: yarn test:env:browser
-
-  meilisearch-php-tests:
-    needs: define-docker-image
-    name: PHP SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-php
-      - name: Install PHP
-        uses: shivammathur/setup-php@v2
-      - name: Validate composer.json and composer.lock
-        run: composer validate
-      - name: Install dependencies
-        run: |
-          composer remove --dev friendsofphp/php-cs-fixer --no-update --no-interaction
-          composer update --prefer-dist --no-progress
-      - name: Run test suite - default HTTP client (Guzzle 7)
-        run: |
-          sh scripts/tests.sh
-          composer remove --dev guzzlehttp/guzzle http-interop/http-factory-guzzle
-
-  meilisearch-python-tests:
-    needs: define-docker-image
-    name: Python SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-python
-      - name: Set up Python
-        uses: actions/setup-python@v5
-      - name: Install pipenv
-        uses: dschep/install-pipenv-action@v1
-      - name: Install dependencies
-        run: pipenv install --dev --python=${{ matrix.python-version }}
-      - name: Test with pytest
-        run: pipenv run pytest
-
-  meilisearch-ruby-tests:
-    needs: define-docker-image
-    name: Ruby SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-ruby
-      - name: Set up Ruby 3
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3
-      - name: Install ruby dependencies
-        run: bundle install --with test
-      - name: Run test suite
-        run: bundle exec rspec
-
-  meilisearch-rust-tests:
-    needs: define-docker-image
-    name: Rust SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-rust
-      - name: Build
-        run: cargo build --verbose
-      - name: Run tests
-        run: cargo test --verbose
-
-  meilisearch-swift-tests:
-    needs: define-docker-image
-    name: Swift SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-swift
-      - name: Run tests
-        run: swift test
-
-########################
-## FRONT-END PLUGINS ##
-########################
-
-  meilisearch-js-plugins-tests:
-    needs: define-docker-image
-    name: meilisearch-js-plugins tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-js-plugins
-      - name: Setup node
-        uses: actions/setup-node@v4
-        with:
-          cache: yarn
-      - name: Install dependencies
-        run: yarn install
-      - name: Run tests
-        run: yarn test
-      - name: Build all the playgrounds and the packages
-        run: yarn build
-
-########################
-## BACK-END PLUGINS ###
-########################
-
-  meilisearch-rails-tests:
-    needs: define-docker-image
-    name: meilisearch-rails tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-rails
-      - name: Set up Ruby 3
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3
-          bundler-cache: true
-      - name: Run tests
-        run: bundle exec rspec
-
-  meilisearch-symfony-tests:
-    needs: define-docker-image
-    name: meilisearch-symfony tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          repository: meilisearch/meilisearch-symfony
-      - name: Install PHP
-        uses: shivammathur/setup-php@v2
-        with:
-          tools: composer:v2, flex
-      - name: Validate composer.json and composer.lock
-        run: composer validate
-      - name: Install dependencies
-        run: composer install --prefer-dist --no-progress --quiet
-      - name: Remove doctrine/annotations
-        run: composer remove --dev doctrine/annotations
-      - name: Run test suite
-        run: composer test:unit

.github/workflows/test-suite.yml

@@ -1,173 +0,0 @@
-name: Test suite
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Everyday at 5:00am
-    - cron: '0 5 * * *'
-  pull_request:
-  push:
-    # trying and staging branches are for Bors config
-    branches:
-      - trying
-      - staging
-
-env:
-  CARGO_TERM_COLOR: always
-  RUST_BACKTRACE: 1
-  RUSTFLAGS: "-D warnings"
-
-jobs:
-  test-linux:
-    name: Tests on ubuntu-20.04
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update && apt-get install -y curl
-          apt-get install build-essential -y
-      - name: Setup test with Rust stable
-        uses: dtolnay/rust-toolchain@1.79
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.1
-      - name: Run cargo check without any default features
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --locked --release --no-default-features --all
-      - name: Run cargo test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked --release --all
-
-  test-others:
-    name: Tests on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-12, windows-2022]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.1
-      - uses: dtolnay/rust-toolchain@1.79
-      - name: Run cargo check without any default features
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --locked --release --no-default-features --all
-      - name: Run cargo test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked --release --all
-
-  test-all-features:
-    name: Tests almost all features
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update
-          apt-get install --assume-yes build-essential curl
-      - uses: dtolnay/rust-toolchain@1.79
-      - name: Run cargo build with almost all features
-        run: |
-          cargo build --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda)"
-      - name: Run cargo test with almost all features
-        run: |
-          cargo test --workspace --locked --release --features "$(cargo xtask list-features --exclude-feature cuda)"
-
-  test-disabled-tokenization:
-    name: Test disabled tokenization
-    runs-on: ubuntu-latest
-    container:
-      image: ubuntu:20.04
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update
-          apt-get install --assume-yes build-essential curl
-      - uses: dtolnay/rust-toolchain@1.79
-      - name: Run cargo tree without default features and check lindera is not present
-        run: |
-          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -qz lindera; then
-            echo "lindera has been found in the sources and it shouldn't"
-            exit 1
-          fi
-      - name: Run cargo tree with default features and check lindera is pressent
-        run: |
-          cargo tree -f '{p} {f}' -e normal | grep lindera -qz
-
-  # We run tests in debug also, to make sure that the debug_assertions are hit
-  test-debug:
-    name: Run tests in debug
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-20.04 to compile with glibc 2.28
-      image: ubuntu:20.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update && apt-get install -y curl
-          apt-get install build-essential -y
-      - uses: dtolnay/rust-toolchain@1.79
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.1
-      - name: Run tests in debug
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked --all
-
-  clippy:
-    name: Run Clippy
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
-        with:
-          profile: minimal
-          components: clippy
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.1
-      - name: Run cargo clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all-targets -- --deny warnings
-
-  fmt:
-    name: Run Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
-        with:
-          profile: minimal
-          toolchain: nightly-2024-07-09
-          override: true
-          components: rustfmt
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.7.1
-      - name: Run cargo fmt
-        # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
-        # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate
-        # we are going to create an empty file where rustfmt expects it.
-        run: |
-          echo -ne "\n" > benchmarks/benches/datasets_paths.rs
-          cargo fmt --all -- --check

.github/workflows/uffizzi-build.yml

@@ -0,0 +1,100 @@
+name: Uffizzi - Build PR Image
+on:
+  pull_request:
+    types: [opened,synchronize,reopened,closed]
+
+jobs:
+  build-meilisearch:
+    name: Build and push `meilisearch`
+    runs-on: ubuntu-latest
+    outputs:
+      tags: ${{ steps.meta.outputs.tags }}
+    if: ${{ github.event.action != 'closed' }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Generate UUID image name
+        id: uuid
+        run: echo "UUID_TAG=$(uuidgen)" >> $GITHUB_ENV
+
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: registry.uffizzi.com/${{ env.UUID_TAG }}
+          tags: | 
+            type=raw,value=60d
+
+      - name: Build Image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./
+          file: .github/uffizzi/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  render-compose-file:
+    name: Render Docker Compose File
+    # Pass output of this workflow to another triggered by `workflow_run` event.
+    runs-on: ubuntu-latest
+    needs: 
+      - build-meilisearch
+    outputs:
+      compose-file-cache-key:  ${{ env.COMPOSE_FILE_HASH }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Render Compose File
+        run: |
+          MEILISEARCH_IMAGE=$(echo ${{ needs.build-meilisearch.outputs.tags }})
+          export MEILISEARCH_IMAGE
+          # Render simple template from environment variables.
+          envsubst < .github/uffizzi/docker-compose.uffizzi.yml > docker-compose.rendered.yml
+          cat docker-compose.rendered.yml
+      - name: Upload Rendered Compose File as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec 
+          path: docker-compose.rendered.yml
+          retention-days: 2
+      - name: Serialize PR Event to File
+        run:  |
+          cat << EOF > event.json
+          ${{ toJSON(github.event) }} 
+          
+          EOF
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2
+
+  delete-preview:
+    name: Call for Preview Deletion
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'closed' }}
+    steps:
+      # If this PR is closing, we will not render a compose file nor pass it to the next workflow.
+      - name: Serialize PR Event to File
+        run:  |
+          cat << EOF > event.json
+          ${{ toJSON(github.event) }} 
+          
+          EOF
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2

.github/workflows/uffizzi-preview-deploy.yml

@@ -0,0 +1,103 @@
+name: Uffizzi - Deploy Preview
+
+on:
+  workflow_run:
+    workflows:
+      - "Uffizzi - Build PR Image"
+    types:
+      - completed
+
+jobs:
+  cache-compose-file:
+    name: Cache Compose File
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    outputs:
+      compose-file-cache-key: ${{ env.COMPOSE_FILE_HASH }}
+      pr-number: ${{ env.PR_NUMBER }}
+      expected-url: ${{ env.EXPECTED_URL }}
+    steps:
+      - name: 'Download artifacts'
+        # Fetch output (zip archive) from the workflow run that triggered this workflow.
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "preview-spec"
+            })[0];
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/preview-spec.zip`, Buffer.from(download.data));
+      
+      - name: 'Unzip artifact'
+        run: unzip preview-spec.zip
+      
+      - name: Read Event into ENV
+        run: |
+          echo 'EVENT_JSON<<EOF' >> $GITHUB_ENV
+          cat event.json >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      
+      - name: Hash Rendered Compose File
+        id: hash
+        # If the previous workflow was triggered by a PR close event, we will not have a compose file artifact.
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.rendered.yml | awk '{ print $1 }')" >> $GITHUB_ENV
+      
+      - name: Cache Rendered Compose File
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        uses: actions/cache@v3
+        with:
+          path: docker-compose.rendered.yml
+          key: ${{ env.COMPOSE_FILE_HASH }}
+
+      - name: Read PR Number From Event Object
+        id: pr
+        run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV
+      
+      - name: DEBUG - Print Job Outputs
+        if: ${{ runner.debug }}
+        run: |
+          echo "PR number: ${{ env.PR_NUMBER }}"
+          echo "Compose file hash: ${{ env.COMPOSE_FILE_HASH }}"
+          cat event.json
+
+      - name: Add expected URL env var
+        if: ${{ runner.debug }}
+        run: |
+          REPO=$(echo ${{ github.repository }} | sed 's/\./+/g')
+          echo "EXPECTED_URL=${{ inputs.server }}/github.com/$REPO/pull/${{ env.PR_NUMBER }}" >> $GITHUB_ENV
+
+  deploy-uffizzi-preview:
+    name: Use Remote Workflow to Preview on Uffizzi
+    needs:
+      - cache-compose-file
+    uses: UffizziCloud/preview-action/.github/workflows/reusable.yaml@desc
+    with:
+      # If this workflow was triggered by a PR close event, cache-key will be an empty string
+      # and this reusable workflow will delete the preview deployment.
+      compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }}
+      compose-file-cache-path: docker-compose.rendered.yml
+      server: https://app.uffizzi.com
+      pr-number: ${{ needs.cache-compose-file.outputs.pr-number }}
+      description: |
+        The meilisearch preview environment contains a web terminal from where you can run the
+        `meilisearch` command. You should be able to access this instance of meilisearch running in
+        the preview from the link Meilisearch Endpoint link given below.
+         
+        Web Terminal Endpoint : ${{ needs.cache-compose-file.outputs.expected-url }}
+        Meilisearch Endpoint : ${{ needs.cache-compose-file.outputs.expected-url }}/meilisearch
+    permissions:
+      contents: read
+      pull-requests: write
+      id-token: write

.github/workflows/update-cargo-toml-version.yml

@@ -1,4 +1,4 @@
-name: Update Meilisearch version in Cargo.toml
+name: Update Meilisearch version in all Cargo.toml files
 
 on:
   workflow_dispatch:
@@ -14,16 +14,18 @@ env:
 
 jobs:
   update-version-cargo-toml:
-    name: Update version in Cargo.toml
+    name: Update version in Cargo.toml files
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: dtolnay/rust-toolchain@1.79
+      - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
+          toolchain: stable
+          override: true
       - name: Install sd
         run: cargo install sd
-      - name: Update Cargo.toml file
+      - name: Update Cargo.toml files
         run: |
           raw_new_version=$(echo $NEW_VERSION | cut -d 'v' -f 2)
           new_string="version = \"$raw_new_version\""
@@ -33,13 +35,13 @@ jobs:
       - name: Commit and push the changes to the ${{ env.NEW_BRANCH }} branch
         uses: EndBug/add-and-commit@v9
         with:
-          message: "Update version for the next release (${{ env.NEW_VERSION }}) in Cargo.toml"
+          message: "Update version for the next release (${{ env.NEW_VERSION }}) in Cargo.toml files"
           new_branch: ${{ env.NEW_BRANCH }}
       - name: Create the PR pointing to ${{ github.ref_name }}
         run: |
           gh pr create \
-            --title "Update version for the next release ($NEW_VERSION) in Cargo.toml" \
-            --body '⚠️ This PR is automatically generated. Check the new version is the expected one and Cargo.lock has been updated before merging.' \
+            --title "Update version for the next release ($NEW_VERSION) in Cargo.toml files" \
+            --body '⚠️ This PR is automatically generated. Check the new version is the expected one before merging.' \
             --label 'skip changelog' \
             --milestone $NEW_VERSION \
             --base $GITHUB_REF_NAME

.gitignore

@@ -9,8 +9,6 @@
 /data.ms
 /snapshots
 /dumps
-/bench
-/_xtask_benchmark.ms
 
 # Snapshots
 ## ... large

BENCHMARKS.md

@@ -1,371 +0,0 @@
-# Benchmarks
-
-Currently this repository hosts two kinds of benchmarks:
-
-1. The older "milli benchmarks", that use [criterion](https://github.com/bheisler/criterion.rs) and live in the "benchmarks" directory.
-2. The newer "bench" that are workload-based and so split between the [`workloads`](./workloads/) directory and the [`xtask::bench`](./xtask/src/bench/) module.
-
-This document describes the newer "bench" benchmarks. For more details on the "milli benchmarks", see [benchmarks/README.md](./benchmarks/README.md).
-
-## Design philosophy for the benchmarks
-
-The newer "bench" benchmarks are **integration** benchmarks, in the sense that they spawn an actual Meilisearch server and measure its performance end-to-end, including HTTP request overhead.
-
-Since this is prone to fluctuating, the benchmarks regain a bit of precision by measuring the runtime of the individual spans using the [logging machinery](./CONTRIBUTING.md#logging) of Meilisearch.
-
-A span roughly translates to a function call. The benchmark runner collects all the spans by name using the [logs route](https://github.com/orgs/meilisearch/discussions/721) and sums their runtime. The processed results are then sent to the [benchmark dashboard](https://bench.meilisearch.dev), which is in charge of storing and presenting the data.
-
-## Running the benchmarks
-
-Benchmarks can run locally or in CI.
-
-### Locally
-
-#### With a local benchmark dashboard
-
-The benchmarks dashboard lives in its [own repository](https://github.com/meilisearch/benchboard). We provide binaries for Ubuntu/Debian, but you can build from source for other platforms (MacOS should work as it was developed under that platform).
-
-Run the `benchboard` binary to create a fresh database of results. By default it will serve the results and the API to gather results on `http://localhost:9001`.
-
-From the Meilisearch repository, you can then run benchmarks with:
-
-```sh
-cargo xtask bench -- workloads/my_workload_1.json ..
-```
-
-This command will build and run Meilisearch locally on port 7700, so make sure that this port is available.
-To run benchmarks on a different commit, just use the usual git command to get back to the desired commit.
-
-#### Without a local benchmark dashboard
-
-To work with the raw results, you can also skip using a local benchmark dashboard.
-
-Run:
-
-```sh
-cargo xtask bench --no-dashboard -- workloads/my_workload_1.json workloads/my_workload_2.json ..
-```
-
-For processing the results, look at [Looking at benchmark results/Without dashboard](#without-dashboard).
-
-### In CI
-
-We have dedicated runners to run workloads on CI. Currently, there are three ways of running the CI:
-
-1. Automatically, on every push to `main`.
-2. Manually, by clicking the [`Run workflow`](https://github.com/meilisearch/meilisearch/actions/workflows/bench-manual.yml) button and specifying the target reference (tag, commit or branch) as well as one or multiple workloads to run. The workloads must exist in the Meilisearch repository (conventionally, in the [`workloads`](./workloads/) directory) on the target reference. Globbing (e.g., `workloads/*.json`) works.
-3. Manually on a PR, by posting a comment containing a `/bench` command, followed by one or multiple workloads to run. Globbing works. The workloads must exist in the Meilisearch repository in the branch of the PR.
-  ```
-  /bench workloads/movies*.json /hackernews_1M.json
-  ```
-
-## Looking at benchmark results
-
-### On the dashboard
-
-Results are available on the global dashboard used by CI at <https://bench.meilisearch.dev> or on your [local dashboard](#with-a-local-benchmark-dashboard).
-
-The dashboard homepage presents three sections:
-
-1. The latest invocations (a call to `cargo xtask bench`, either local or by CI) with their reason (generally set to some helpful link in CI) and their status.
-2. The latest workloads ran on `main`.
-3. The latest workloads ran on other references.
-
-By default, the workload shows the total runtime delta with the latest applicable commit on `main`. The latest applicable commit is the latest commit for workload invocations that do not originate on `main`, and the latest previous commit for workload invocations that originate on `main`.
-
-You can explicitly request a detailed comparison by span with the `main` branch, the branch or origin, or any previous commit, by clicking the links at the bottom of the workload invocation.
-
-In the detailed comparison view, the spans are sorted by improvements, regressions, stable (no statistically significant change) and unstable (the span runtime is comparable to its standard deviation).
-
-You can click on the name of any span to get a box plot comparing the target commit with multiple commits of the selected branch.
-
-### Without dashboard
-
-After the workloads are done running, the reports will live in the Meilisearch repository, in the `bench/reports` directory (by default).
-
-You can then convert these reports into other formats.
-
-- To [Firefox profiler](https://profiler.firefox.com) format. Run:
-  ```sh
-  cd bench/reports
-  cargo run --release --bin trace-to-firefox -- my_workload_1-0-trace.json
-  ```
-  You can then upload the resulting `firefox-my_workload_1-0-trace.json` file to the online profiler.
-
-
-## Designing benchmark workloads
-
-Benchmark workloads conventionally live in the `workloads` directory of the Meilisearch repository.
-
-They are JSON files with the following structure (comments are not actually supported, to make your own, remove them or copy some existing workload file):
-
-```jsonc
-{
-  // Name of the workload. Must be unique to the workload, as it will be used to group results on the dashboard.
-  "name": "hackernews.ndjson_1M,no-threads",
-  // Number of consecutive runs of the commands that should be performed.
-  // Each run uses a fresh instance of Meilisearch and a fresh database.
-  // Each run produces its own report file.
-  "run_count": 3,
-  // List of arguments to add to the Meilisearch command line.
-  "extra_cli_args": ["--max-indexing-threads=1"],
-  // An expression that can be parsed as a comma-separated list of targets and levels
-  // as described in [tracing_subscriber's documentation](https://docs.rs/tracing-subscriber/latest/tracing_subscriber/filter/targets/struct.Targets.html#examples).
-  // The expression is used to filter the spans that are measured for profiling purposes.
-  // Optional, defaults to "indexing::=trace" (for indexing workloads), common other values is
-  // "search::=trace"
-  "target": "indexing::=trace",
-  // List of named assets that can be used in the commands.
-  "assets": {
-    // name of the asset.
-    // Must be unique at the workload level.
-    // For better results, the same asset (same sha256) should have the same name accross workloads.
-    // Having multiple assets with the same name and distinct hashes is supported accross workloads,
-    // but will lead to superfluous downloads.
-    //
-    // Assets are stored in the `bench/assets/` directory by default.
-    "hackernews-100_000.ndjson": {
-      // If the assets exists in the local filesystem (Meilisearch repository or for your local workloads)
-      // Its file path can be specified here.
-      // `null` if the asset should be downloaded from a remote location.
-      "local_location": null,
-      // URL of the remote location where the asset can be downloaded.
-      // Use the `--assets-key` of the runner to pass an API key in the `Authorization: Bearer` header of the download requests.
-      // `null` if the asset should be imported from a local location.
-      // if both local and remote locations are specified, then the local one is tried first, then the remote one
-      // if the file is locally missing or its hash differs.
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-100_000.ndjson",
-      // SHA256 of the asset.
-      // Optional, the `sha256` of the asset will be displayed during a run of the workload if it is missing.
-      // If present, the hash of the asset in the `bench/assets/` directory will be compared against this hash before
-      // running the workload. If the hashes differ, the asset will be downloaded anew.
-      "sha256": "60ecd23485d560edbd90d9ca31f0e6dba1455422f2a44e402600fbb5f7f1b213",
-      // Optional, one of "Auto", "Json", "NdJson" or "Raw".
-      // If missing, assumed to be "Auto".
-      // If "Auto", the format will be determined from the extension in the asset name.
-      "format": "NdJson"
-    },
-    "hackernews-200_000.ndjson": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-200_000.ndjson",
-      "sha256": "785b0271fdb47cba574fab617d5d332276b835c05dd86e4a95251cf7892a1685"
-    },
-    "hackernews-300_000.ndjson": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-300_000.ndjson",
-      "sha256": "de73c7154652eddfaf69cdc3b2f824d5c452f095f40a20a1c97bb1b5c4d80ab2"
-    },
-    "hackernews-400_000.ndjson": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-400_000.ndjson",
-      "sha256": "c1b00a24689110f366447e434c201c086d6f456d54ed1c4995894102794d8fe7"
-    },
-    "hackernews-500_000.ndjson": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-500_000.ndjson",
-      "sha256": "ae98f9dbef8193d750e3e2dbb6a91648941a1edca5f6e82c143e7996f4840083"
-    },
-    "hackernews-600_000.ndjson": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-600_000.ndjson",
-      "sha256": "b495fdc72c4a944801f786400f22076ab99186bee9699f67cbab2f21f5b74dbe"
-    },
-    "hackernews-700_000.ndjson": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-700_000.ndjson",
-      "sha256": "4b2c63974f3dabaa4954e3d4598b48324d03c522321ac05b0d583f36cb78a28b"
-    },
-    "hackernews-800_000.ndjson": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-800_000.ndjson",
-      "sha256": "cb7b6afe0e6caa1be111be256821bc63b0771b2a0e1fad95af7aaeeffd7ba546"
-    },
-    "hackernews-900_000.ndjson": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-900_000.ndjson",
-      "sha256": "e1154ddcd398f1c867758a93db5bcb21a07b9e55530c188a2917fdef332d3ba9"
-    },
-    "hackernews-1_000_000.ndjson": {
-      "local_location": null,
-      "remote_location": "https://milli-benchmarks.fra1.digitaloceanspaces.com/bench/datasets/hackernews/hackernews-1_000_000.ndjson",
-      "sha256": "27e25efd0b68b159b8b21350d9af76938710cb29ce0393fa71b41c4f3c630ffe"
-    }
-  },
-  // Core of the workload.
-  // A list of commands to run sequentially.
-  // Optional: A precommand is a request to the Meilisearch instance that is executed before the profiling runs.
-  "precommands": [
-    {
-      // Meilisearch route to call. `http://localhost:7700/` will be prepended.
-      "route": "indexes/movies/settings",
-      // HTTP method to call.
-      "method": "PATCH",
-      // If applicable, body of the request.
-      // Optional, if missing, the body will be empty.
-      "body": {
-        // One of "empty", "inline" or "asset".
-        // If using "empty", you can skip the entire "body" key.
-        "inline": {
-          // when "inline" is used, the body is the JSON object that is the value of the `"inline"` key.
-          "displayedAttributes": [
-            "title",
-            "by",
-            "score",
-            "time"
-          ],
-          "searchableAttributes": [
-            "title"
-          ],
-          "filterableAttributes": [
-            "by"
-          ],
-          "sortableAttributes": [
-            "score",
-            "time"
-          ]
-        }
-      },
-      // Whether to wait before running the next request.
-      // One of:
-      // - DontWait: run the next command without waiting the response to this one.
-      // - WaitForResponse: run the next command as soon as the response from the server is received.
-      // - WaitForTask: run the next command once **all** the Meilisearch tasks created up to now have finished processing.
-      "synchronous": "WaitForTask"
-    }
-  ],
-  // A command is a request to the Meilisearch instance that is executed while the profiling runs.
-  "commands": [
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        // When using "asset", use the name of an asset as value to use the content of that asset as body.
-        // the content type is derived of the format of the asset:
-        // "NdJson" => "application/x-ndjson"
-        // "Json" => "application/json"
-        // "Raw" => "application/octet-stream"
-        // See [AssetFormat::to_content_type](https://github.com/meilisearch/meilisearch/blob/7b670a4afadb132ac4a01b6403108700501a391d/xtask/src/bench/assets.rs#L30)
-        // for details and up-to-date list.
-        "asset": "hackernews-100_000.ndjson"
-      },
-      "synchronous": "WaitForTask"
-    },
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "hackernews-200_000.ndjson"
-      },
-      "synchronous": "WaitForResponse"
-    },
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "hackernews-300_000.ndjson"
-      },
-      "synchronous": "WaitForResponse"
-    },
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "hackernews-400_000.ndjson"
-      },
-      "synchronous": "WaitForResponse"
-    },
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "hackernews-500_000.ndjson"
-      },
-      "synchronous": "WaitForResponse"
-    },
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "hackernews-600_000.ndjson"
-      },
-      "synchronous": "WaitForResponse"
-    },
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "hackernews-700_000.ndjson"
-      },
-      "synchronous": "WaitForResponse"
-    },
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "hackernews-800_000.ndjson"
-      },
-      "synchronous": "WaitForResponse"
-    },
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "hackernews-900_000.ndjson"
-      },
-      "synchronous": "WaitForResponse"
-    },
-    {
-      "route": "indexes/movies/documents",
-      "method": "POST",
-      "body": {
-        "asset": "hackernews-1_000_000.ndjson"
-      },
-      "synchronous": "WaitForTask"
-    }
-  ]
-}
-```
-
-### Adding new assets
-
-Assets reside in our DigitalOcean S3 space. Assuming you have team access to the DigitalOcean S3 space:
-
-1. go to <https://cloud.digitalocean.com/spaces/milli-benchmarks?i=d1c552&path=bench%2Fdatasets%2F>
-2. upload your dataset:
-   1. if your dataset is a single file, upload that single file using the "upload" button,
-   2. otherwise, create a folder using the "create folder" button, then inside that folder upload your individual files.
-
-## Upgrading `https://bench.meilisearch.dev`
-
-The URL of the server is in our password manager (look for "benchboard").
-
-1. Make the needed modifications on the [benchboard repository](https://github.com/meilisearch/benchboard) and merge them to main.
-2. Publish a new release to produce the Ubuntu/Debian binary.
-3. Download the binary locally, send it to the server:
-  ```
-  scp -6 ~/Downloads/benchboard root@\[<ipv6-address>\]:/bench/new-benchboard
-  ```
-  Note that the ipv6 must be between escaped square brackets for SCP.
-4. SSH to the server:
-  ```
-  ssh root@<ipv6-address>
-  ```
-  Note the ipv6 must **NOT** be between escaped square brackets for SSH 🥲
-5. On the server, set the correct permissions for the new binary:
-   ```
-   chown bench:bench /bench/new-benchboard
-   chmod 700 /bench/new-benchboard
-   ```
-6. On the server, move the new binary to the location of the running binary (if unsure, start by making a backup of the running binary):
-  ```
-  mv /bench/{new-,}benchboard
-  ```
-7. Restart the benchboard service.
-  ```
-  systemctl restart benchboard
-  ```
-8. Check that the service runs correctly.
-  ```
-  systemctl status benchboard
-  ```
-9. Check the availability of the service by going to <https://bench.meilisearch.dev> on your browser.

CONTRIBUTING.md

@@ -4,7 +4,7 @@ First, thank you for contributing to Meilisearch! The goal of this document is t
 
 Remember that there are many ways to contribute other than writing code: writing [tutorials or blog posts](https://github.com/meilisearch/awesome-meilisearch), improving [the documentation](https://github.com/meilisearch/documentation), submitting [bug reports](https://github.com/meilisearch/meilisearch/issues/new?assignees=&labels=&template=bug_report.md&title=) and [feature requests](https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal)...
 
-Meilisearch can manage multiple indexes, handle the update store, and expose an HTTP API. Search and indexation are the domain of our core engine, [`milli`](https://github.com/meilisearch/meilisearch/tree/main/milli), while tokenization is handled by [our `charabia` library](https://github.com/meilisearch/charabia/).
+The code in this repository is only concerned with managing multiple indexes, handling the update store, and exposing an HTTP API. Search and indexation are the domain of our core engine, [`milli`](https://github.com/meilisearch/milli), while tokenization is handled by [our `charabia` library](https://github.com/meilisearch/charabia/).
 
 If Meilisearch does not offer optimized support for your language, please consider contributing to `charabia` by following the [CONTRIBUTING.md file](https://github.com/meilisearch/charabia/blob/main/CONTRIBUTING.md) and integrating your intended normalizer/segmenter.
 
@@ -18,9 +18,9 @@ If Meilisearch does not offer optimized support for your language, please consid
 
 ## Assumptions
 
-1. **You're familiar with [GitHub](https://github.com) and the [Pull Requests (PR)](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests) workflow.**
-2. **You've read the Meilisearch [documentation](https://www.meilisearch.com/docs).**
-3. **You know about the [Meilisearch community on Discord](https://discord.meilisearch.com).
+1. **You're familiar with [GitHub](https://github.com) and the [Pull Requests](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)(PR) workflow.**
+2. **You've read the Meilisearch [documentation](https://docs.meilisearch.com).**
+3. **You know about the [Meilisearch community](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html).
    Please use this for help.**
 
 ## How to Contribute
@@ -52,20 +52,6 @@ cargo test
 
 This command will be triggered to each PR as a requirement for merging it.
 
-#### Faster build
-
-You can set the `LINDERA_CACHE` environment variable to speed up your successive builds by up to 2 minutes.
-It'll store some built artifacts in the directory of your choice.
-
-We recommend using the standard `$HOME/.cache/lindera` directory:
-```sh
-export LINDERA_CACHE=$HOME/.cache/lindera
-```
-
-Furthermore, you can improve incremental compilation by setting the `MEILI_NO_VERGEN` environment variable.
-Setting this variable will prevent the Meilisearch binary from being rebuilt each time the directory that hosts the Meilisearch repository changes.
-Do not enable this environment variable for production builds (as it will break the `version` route, among other things).
-
 #### Snapshot-based tests
 
 We are using [insta](https://insta.rs) to perform snapshot-based testing.
@@ -77,7 +63,7 @@ Furthermore, we provide some macros on top of insta, notably a way to use snapsh
 
 To effectively debug snapshot-based hashes, we recommend you export the `MEILI_TEST_FULL_SNAPS` environment variable so that snapshot are fully created locally:
 
-```sh
+```
 export MEILI_TEST_FULL_SNAPS=true # add this to your .bashrc, .zshrc, ...
 ```
 
@@ -89,36 +75,6 @@ If you get a "Too many open files" error you might want to increase the open fil
 ulimit -Sn 3000
 ```
 
-#### Build tools
-
-Meilisearch follows the [cargo xtask](https://github.com/matklad/cargo-xtask) workflow to provide some build tools.
-
-Run `cargo xtask --help` from the root of the repository to find out what is available.
-
-### Logging
-
-Meilisearch uses [`tracing`](https://lib.rs/crates/tracing) for logging purposes. Tracing logs are structured and can be displayed as JSON to the end user, so prefer passing arguments as fields rather than interpolating them in the message.
-
-Refer to the [documentation](https://docs.rs/tracing/0.1.40/tracing/index.html#using-the-macros) for the syntax of the spans and events.
-
-Logging spans are used for 3 distinct purposes:
-
-1. Regular logging
-2. Profiling
-3. Benchmarking
-
-As a result, the spans should follow some rules:
-
-- They should not be put on functions that are called too often. That is because opening and closing a span causes some overhead. For regular logging, avoid putting spans on functions that are taking less than a few hundred nanoseconds. For profiling or benchmarking, avoid putting spans on functions that are taking less than a few microseconds.
-- For profiling and benchmarking, use the `TRACE` level.
-- For profiling and benchmarking, use the following `target` prefixes:
-  - `indexing::` for spans meant when profiling the indexing operations.
-  - `search::` for spans meant when profiling the search operations.
-
-### Benchmarking
-
-See [BENCHMARKS.md](./BENCHMARKS.md)
-
 ## Git Guidelines
 
 ### Git Branches
@@ -164,9 +120,29 @@ The full Meilisearch release process is described in [this guide](https://github
 
 Depending on the developed feature, you might need to provide a prototyped version of Meilisearch to make it easier to test by the users.
 
-This happens in two steps:
-- [Release the prototype](https://github.com/meilisearch/engine-team/blob/main/resources/prototypes.md#how-to-publish-a-prototype)
-- [Communicate about it](https://github.com/meilisearch/engine-team/blob/main/resources/prototypes.md#communication)
+The prototype name must follow this convention: `prototype-X-Y` where
+- `X` is the feature name formatted in `kebab-case`. It should not end with a single number.
+- `Y` is the version of the prototype, starting from `0`.
+
+✅ Example: `prototype-auto-resize-0`. </br>
+❌ Bad example: `auto-resize-0`: lacks the `prototype` prefix. </br>
+❌ Bad example: `prototype-auto-resize`: lacks the version suffix. </br>
+❌ Bad example: `prototype-auto-resize-0-0`: feature name ends with a single number.
+
+Steps to create a prototype:
+
+1. In your terminal, go to the last commit of your branch (the one you want to provide as a prototype).
+2. Create a tag following the convention: `git tag prototype-X-Y`
+3. Run Meilisearch and check that its launch summary features a line: `Prototype: prototype-X-Y` (you may need to switch branches and back after tagging for this to work).
+3. Push the tag: `git push origin prototype-X-Y`
+4. Check the [Docker CI](https://github.com/meilisearch/meilisearch/actions/workflows/publish-docker-images.yml) is now running.
+
+🐳 Once the CI has finished to run (~1h30), a Docker image named `prototype-X-Y` will be available on [DockerHub](https://hub.docker.com/repository/docker/getmeili/meilisearch/general). People can use it with the following command: `docker run -p 7700:7700 -v $(pwd)/meili_data:/meili_data getmeili/meilisearch:prototype-X-Y`. <br>
+More information about [how to run Meilisearch with Docker](https://docs.meilisearch.com/learn/cookbooks/docker.html#download-meilisearch-with-docker).
+
+⚙️ However, no binaries will be created. If the users do not use Docker, they can go to the `prototype-X-Y` tag in the Meilisearch repository and compile from the source code.
+
+⚠️ When sharing a prototype with users, remind them to not use it in production. Prototypes are solely for test purposes.
 
 ### Release assets
 

Cargo.toml

@@ -2,7 +2,6 @@
 resolver = "2"
 members = [
     "meilisearch",
-    "meilitool",
     "meilisearch-types",
     "meilisearch-auth",
     "meili-snap",
@@ -14,19 +13,12 @@ members = [
     "filter-parser",
     "flatten-serde-json",
     "json-depth-checker",
-    "benchmarks",
-    "fuzzers",
-    "tracing-trace",
-    "xtask",
-    "build-info",
+    "benchmarks"
 ]
 
 [workspace.package]
-version = "1.11.0"
-authors = [
-    "Quentin de Quelen <quentin@dequelen.me>",
-    "Clément Renault <clement@meilisearch.com>",
-]
+version = "1.0.0"
+authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
 description = "Meilisearch HTTP server"
 homepage = "https://meilisearch.com"
 readme = "README.md"

Dockerfile

@@ -1,36 +1,36 @@
 # Compile
-FROM    rust:1.79.0-alpine3.20 AS compiler
+FROM    rust:bullseye AS compiler
 
-RUN     apk add -q --no-cache build-base openssl-dev
-
-WORKDIR /
+WORKDIR /meilisearch
 
 ARG     COMMIT_SHA
 ARG     COMMIT_DATE
 ARG     GIT_TAG
-ENV     VERGEN_GIT_SHA=${COMMIT_SHA} VERGEN_GIT_COMMIT_TIMESTAMP=${COMMIT_DATE} VERGEN_GIT_DESCRIBE=${GIT_TAG}
+ENV     VERGEN_GIT_SHA=${COMMIT_SHA} VERGEN_GIT_COMMIT_TIMESTAMP=${COMMIT_DATE} VERGEN_GIT_SEMVER_LIGHTWEIGHT=${GIT_TAG}
 ENV     RUSTFLAGS="-C target-feature=-crt-static"
 
 COPY    . .
 RUN     set -eux; \
-        apkArch="$(apk --print-arch)"; \
-        if [ "$apkArch" = "aarch64" ]; then \
+        arch="$(dpkg --print-architecture)"; \
+        if [ "$arch" = "arm64" ]; then \
             export JEMALLOC_SYS_WITH_LG_PAGE=16; \
         fi && \
-        cargo build --release -p meilisearch -p meilitool
+        cargo build --release
 
 # Run
-FROM    alpine:3.20
+FROM    debian:11.6
 
 ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
 ENV     MEILI_SERVER_PROVIDER docker
 
-RUN     apk add -q --no-cache libgcc tini curl
+RUN     set -ex; \
+        apt-get update -q; \
+        apt-get install -q -y --no-install-recommends tini; \
+        rm -rf /var/lib/apt/lists/*
 
-# add meilisearch and meilitool to the `/bin` so you can run it from anywhere
-# and it's easy to find.
-COPY    --from=compiler /target/release/meilisearch /bin/meilisearch
-COPY    --from=compiler /target/release/meilitool /bin/meilitool
+# add meilisearch to the `/bin` so you can run it from anywhere and it's easy
+# to find.
+COPY    --from=compiler /meilisearch/target/release/meilisearch /bin/meilisearch
 # To stay compatible with the older version of the container (pre v0.27.0) we're
 # going to symlink the meilisearch binary in the path to `/meilisearch`
 RUN     ln -s /bin/meilisearch /meilisearch

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019-2024 Meili SAS
+Copyright (c) 2019-2022 Meili SAS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

PROFILING.md

@@ -1,19 +0,0 @@
-# Profiling Meilisearch
-
-Search engine technologies are complex pieces of software that require thorough profiling tools. We chose to use [Puffin](https://github.com/EmbarkStudios/puffin), which the Rust gaming industry uses extensively. You can export and import the profiling reports using the top bar's _File_ menu options [in Puffin Viewer](https://github.com/embarkstudios/puffin#ui).
-
-![An example profiling with Puffin viewer](assets/profiling-example.png)
-
-## Profiling the Indexing Process
-
-When you enable [the `exportPuffinReports` experimental feature](https://www.meilisearch.com/docs/learn/experimental/overview) of Meilisearch, Puffin reports with the `.puffin` extension will be automatically exported to disk. When this option is enabled, the engine will automatically create a "frame" whenever it executes the `IndexScheduler::tick` method.
-
-[Puffin Viewer](https://github.com/EmbarkStudios/puffin/tree/main/puffin_viewer) is used to analyze the reports. Those reports show areas where Meilisearch spent time during indexing.
-
-Another piece of advice on the Puffin viewer UI interface is to consider the _Merge children with same ID_ option. It can hide the exact actual timings at which events were sent. Please turn it off when you see strange gaps on the Flamegraph. It can help.
-
-## Profiling the Search Process
-
-We still need to take the time to profile the search side of the engine with Puffin. It would require time to profile the filtering phase, query parsing, creation, and execution. We could even profile the Actix HTTP server.
-
-The only issue we see is the framing system. Puffin requires a global frame-based profiling phase, which collides with Meilisearch's ability to accept and answer multiple requests on different threads simultaneously.

README.md

@@ -1,101 +1,94 @@
 <p align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo" target="_blank">
-    <img src="assets/meilisearch-logo-kawaii.png">
-  </a>
+  <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
+  <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
 </p>
 
 <h4 align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Website</a> |
+  <a href="https://www.meilisearch.com">Website</a> |
   <a href="https://roadmap.meilisearch.com/tabs/1-under-consideration">Roadmap</a> |
-  <a href="https://www.meilisearch.com/pricing?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Meilisearch Cloud</a> |
-  <a href="https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Blog</a> |
-  <a href="https://www.meilisearch.com/docs?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Documentation</a> |
-  <a href="https://www.meilisearch.com/docs/faq?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">FAQ</a> |
-  <a href="https://discord.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Discord</a>
+  <a href="https://blog.meilisearch.com">Blog</a> |
+  <a href="https://docs.meilisearch.com">Documentation</a> |
+  <a href="https://docs.meilisearch.com/faq/">FAQ</a> |
+  <a href="https://discord.meilisearch.com">Discord</a>
 </h4>
 
 <p align="center">
   <a href="https://deps.rs/repo/github/meilisearch/meilisearch"><img src="https://deps.rs/repo/github/meilisearch/meilisearch/status.svg" alt="Dependency status"></a>
   <a href="https://github.com/meilisearch/meilisearch/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
-  <a href="https://ms-bors.herokuapp.com/repositories/52"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
+  <a href="https://app.bors.tech/repositories/26457"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
 </p>
 
 <p align="center">⚡ A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow 🔍</p>
 
-[Meilisearch](https://www.meilisearch.com?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=intro) helps you shape a delightful search experience in a snap, offering features that work out of the box to speed up your workflow.
+Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow.
 
 <p align="center" name="demo">
-  <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-light-mode-only" target="_blank">
+  <a href="https://where2watch.meilisearch.com/#gh-light-mode-only" target="_blank">
     <img src="assets/demo-light.gif#gh-light-mode-only" alt="A bright colored application for finding movies screening near the user">
   </a>
-  <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-dark-mode-only" target="_blank">
+  <a href="https://where2watch.meilisearch.com/#gh-dark-mode-only" target="_blank">
     <img src="assets/demo-dark.gif#gh-dark-mode-only" alt="A dark colored application for finding movies screening near the user">
   </a>
 </p>
 
-## 🖥 Examples
-
-- [**Movies**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=organization) — An application to help you find streaming platforms to watch movies using [hybrid search](https://www.meilisearch.com/solutions/hybrid-search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos).
-- [**Ecommerce**](https://ecommerce.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Ecommerce website using disjunctive [facets](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos), range and rating filtering, and pagination.
-- [**Songs**](https://music.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search through 47 million of songs.
-- [**SaaS**](https://saas.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) — Search for contacts, deals, and companies in this [multi-tenant](https://www.meilisearch.com/docs/learn/security/multitenancy_tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demos) CRM application.
-
-See the list of all our example apps in our [demos repository](https://github.com/meilisearch/demos).
+🔥 [**Try it!**](https://where2watch.meilisearch.com/) 🔥
 
 ## ✨ Features
-- **Hybrid search:** Combine the best of both [semantic](https://www.meilisearch.com/docs/learn/experimental/vector_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) & full-text search to get the most relevant results
-- **Search-as-you-type:** Find & display results in less than 50 milliseconds to provide an intuitive experience
-- **[Typo tolerance](https://www.meilisearch.com/docs/learn/configuration/typo_tolerance?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** get relevant matches even when queries contain typos and misspellings
-- **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your users' search experience with custom filters and build a faceted search interface in a few lines of code
-- **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** sort results based on price, date, or pretty much anything else your users need
-- **[Synonym support](https://www.meilisearch.com/docs/learn/configuration/synonyms?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** configure synonyms to include more relevant content in your search results
-- **[Geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** filter and sort documents based on geographic data
-- **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
-- **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** control which users can access what data with API keys that allow fine-grained permissions handling
-- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** personalize search results for any number of application tenants
+
+- **Search-as-you-type:** find search results in less than 50 milliseconds
+- **[Typo tolerance](https://docs.meilisearch.com/learn/getting_started/customizing_relevancy.html#typo-tolerance):** get relevant matches even when queries contain typos and misspellings
+- **[Filtering and faceted search](https://docs.meilisearch.com/learn/advanced/filtering_and_faceted_search.html):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code
+- **[Sorting](https://docs.meilisearch.com/learn/advanced/sorting.html):** sort results based on price, date, or pretty much anything else your users need
+- **[Synonym support](https://docs.meilisearch.com/learn/getting_started/customizing_relevancy.html#synonyms):** configure synonyms to include more relevant content in your search results
+- **[Geosearch](https://docs.meilisearch.com/learn/advanced/geosearch.html):** filter and sort documents based on geographic data
+- **[Extensive language support](https://docs.meilisearch.com/learn/what_is_meilisearch/language.html):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
+- **[Security management](https://docs.meilisearch.com/learn/security/master_api_keys.html):** control which users can access what data with API keys that allow fine-grained permissions handling
+- **[Multi-Tenancy](https://docs.meilisearch.com/learn/security/tenant_tokens.html):** personalize search results for any number of application tenants
 - **Highly Customizable:** customize Meilisearch to your specific needs or use our out-of-the-box and hassle-free presets
-- **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** integrate Meilisearch in your technical stack with our plugins and SDKs
+- **[RESTful API](https://docs.meilisearch.com/reference/api/overview.html):** integrate Meilisearch in your technical stack with our plugins and SDKs
 - **Easy to install, deploy, and maintain**
 
 ## 📖 Documentation
 
-You can consult Meilisearch's documentation at [meilisearch.com/docs](https://www.meilisearch.com/docs/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=docs).
+You can consult Meilisearch's documentation at [https://docs.meilisearch.com](https://docs.meilisearch.com/).
 
 ## 🚀 Getting started
 
-For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [documentation](https://www.meilisearch.com/docs?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
+For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://docs.meilisearch.com/learn/getting_started/quick_start.html) guide.
 
-## 🌍 Supercharge your Meilisearch experience
+You may also want to check out [Meilisearch 101](https://docs.meilisearch.com/learn/getting_started/filtering_and_sorting.html) for an introduction to some of Meilisearch's most popular features.
 
-Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). Additional features include analytics & monitoring in many regions around the world. No credit card is required.
+## ☁️ Meilisearch cloud
+
+Let us manage your infrastructure so you can focus on integrating a great search experience. Try [Meilisearch Cloud](https://meilisearch.com/pricing) today.
 
 ## 🧰 SDKs & integration tools
 
 Install one of our SDKs in your project for seamless integration between Meilisearch and your favorite language or framework!
 
-Take a look at the complete [Meilisearch integration list](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-link).
+Take a look at the complete [Meilisearch integration list](https://docs.meilisearch.com/learn/what_is_meilisearch/sdks.html).
 
-[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-logos)
+[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://docs.meilisearch.com/learn/what_is_meilisearch/sdks.html)
 
 ## ⚙️ Advanced usage
 
-Experienced users will want to keep our [API Reference](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) close at hand.
+Experienced users will want to keep our [API Reference](https://docs.meilisearch.com/reference/api) close at hand.
 
-We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [API keys](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), and [tenant tokens](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
+We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://docs.meilisearch.com/learn/advanced/filtering_and_faceted_search.html), [sorting](https://docs.meilisearch.com/learn/advanced/sorting.html), [geosearch](https://docs.meilisearch.com/learn/advanced/geosearch.html), [API keys](https://docs.meilisearch.com/learn/security/master_api_keys.html), and [tenant tokens](https://docs.meilisearch.com/learn/security/tenant_tokens.html).
 
-Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
+Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://docs.meilisearch.com/learn/core_concepts/documents.html) and [indexes](https://docs.meilisearch.com/learn/core_concepts/indexes.html).
 
 ## 📊 Telemetry
 
-Meilisearch collects **anonymized** user data to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
+Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html#how-to-disable-data-collection) whenever you want.
 
-To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Remember to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
+To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
 
-If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) of our documentation.
+If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html) of our documentation.
 
 ## 📫 Get in touch!
 
-Meilisearch is a search engine created by [Meili]([https://www.welcometothejungle.com/en/companies/meilisearch](https://www.meilisearch.com/careers)), a software development company headquartered in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
+Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/)
 
 🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
 
@@ -103,17 +96,18 @@ Meilisearch is a search engine created by [Meili]([https://www.welcometothejungl
 
 - For feature requests, please visit our [product repository](https://github.com/meilisearch/product/discussions)
 - Found a bug? Open an [issue](https://github.com/meilisearch/meilisearch/issues)!
-- Want to be part of our Discord community? [Join us!](https://discord.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
+- Want to be part of our Discord community? [Join us!](https://discord.gg/meilisearch)
+- For everything else, please check [this page listing some of the other places where you can find us](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html)
 
 Thank you for your support!
 
 ## 👩‍💻 Contributing
 
-Meilisearch is, and will always be, open-source! If you want to contribute to the project, please look at [our contribution guidelines](CONTRIBUTING.md).
+Meilisearch is, and will always be, open-source! If you want to contribute to the project, please take a look at [our contribution guidelines](CONTRIBUTING.md).
 
 ## 📦 Versioning
 
-Meilisearch releases and their associated binaries are available on the project's [releases page](https://github.com/meilisearch/meilisearch/releases).
+Meilisearch releases and their associated binaries are available [in this GitHub page](https://github.com/meilisearch/meilisearch/releases).
 
 The binaries are versioned following [SemVer conventions](https://semver.org/). To know more, read our [versioning policy](https://github.com/meilisearch/engine-team/blob/main/resources/versioning-policy.md).
 

assets/prometheus-basic-scraper.yml

@@ -1,19 +0,0 @@
-global:
-  scrape_interval:     15s # By default, scrape targets every 15 seconds.
-
-  # Attach these labels to any time series or alerts when communicating with
-  # external systems (federation, remote storage, Alertmanager).
-  external_labels:
-    monitor: 'codelab-monitor'
-
-# A scrape configuration containing exactly one endpoint to scrape:
-# Here it's Prometheus itself.
-scrape_configs:
-  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
-  - job_name: 'meilisearch'
-
-    # Override the global default and scrape targets from this job every 5 seconds.
-    scrape_interval: 5s
-
-    static_configs:
-      - targets: ['localhost:7700']

benchmarks/Cargo.toml

@@ -11,27 +11,27 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.86"
-csv = "1.3.0"
-milli = { path = "../milli" }
-mimalloc = { version = "0.1.43", default-features = false }
-serde_json = { version = "1.0.120", features = ["preserve_order"] }
+anyhow = "1.0.65"
+csv = "1.1.6"
+milli = { path = "../milli", default-features = false }
+mimalloc = { version = "0.1.29", default-features = false }
+serde_json = { version = "1.0.85", features = ["preserve_order"] }
 
 [dev-dependencies]
-criterion = { version = "0.5.1", features = ["html_reports"] }
+criterion = { version = "0.4.0", features = ["html_reports"] }
 rand = "0.8.5"
 rand_chacha = "0.3.1"
-roaring = "0.10.6"
+roaring = "0.10.1"
 
 [build-dependencies]
-anyhow = "1.0.86"
-bytes = "1.6.0"
+anyhow = "1.0.65"
+bytes = "1.2.1"
 convert_case = "0.6.0"
-flate2 = "1.0.30"
-reqwest = { version = "0.12.5", features = ["blocking", "rustls-tls"], default-features = false }
+flate2 = "1.0.24"
+reqwest = { version = "0.11.12", features = ["blocking", "rustls-tls"], default-features = false }
 
 [features]
-default = ["milli/all-tokenizations"]
+default = ["milli/default"]
 
 [[bench]]
 name = "search_songs"
@@ -48,3 +48,7 @@ harness = false
 [[bench]]
 name = "indexing"
 harness = false
+
+[[bench]]
+name = "formatting"
+harness = false

benchmarks/README.md

@@ -119,9 +119,9 @@ _[Download the `smol-wiki` dataset](https://milli-benchmarks.fra1.digitaloceansp
 
 ### Movies
 
-`movies` is a really small dataset we uses as our example in the [getting started](https://www.meilisearch.com/docs/learn/getting_started/quick_start)
+`movies` is a really small dataset we uses as our example in the [getting started](https://docs.meilisearch.com/learn/getting_started/)
 
-_[Download the `movies` dataset](https://www.meilisearch.com/movies.json)._
+_[Download the `movies` dataset](https://docs.meilisearch.com/movies.json)._
 
 
 ### All Countries

benchmarks/benches/formatting.rs

@@ -0,0 +1,67 @@
+use std::rc::Rc;
+
+use criterion::{criterion_group, criterion_main};
+use milli::tokenizer::TokenizerBuilder;
+use milli::{FormatOptions, MatcherBuilder, MatchingWord, MatchingWords};
+
+#[global_allocator]
+static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
+struct Conf<'a> {
+    name: &'a str,
+    text: &'a str,
+    matching_words: MatcherBuilder<'a, Vec<u8>>,
+}
+
+fn bench_formatting(c: &mut criterion::Criterion) {
+    #[rustfmt::skip]
+    let confs = &[
+    	Conf {
+    		name: "'the door d'",
+			text: r#"He used to do the door sounds in "Star Trek" with his mouth, phssst, phssst. The MD-11 passenger and cargo doors also tend to behave like electromagnetic apertures, because the doors do not have continuous electrical contact with the door frames around the door perimeter. But Theodor said that the doors don't work."#,
+			matching_words: MatcherBuilder::new(MatchingWords::new(vec![
+	            (vec![Rc::new(MatchingWord::new("t".to_string(), 0, false).unwrap()), Rc::new(MatchingWord::new("he".to_string(), 0, false).unwrap())], vec![0]),
+	            (vec![Rc::new(MatchingWord::new("the".to_string(), 0, false).unwrap())], vec![0]),
+	            (vec![Rc::new(MatchingWord::new("door".to_string(), 1, false).unwrap())], vec![1]),
+	            (vec![Rc::new(MatchingWord::new("do".to_string(), 0, false).unwrap()), Rc::new(MatchingWord::new("or".to_string(), 0, false).unwrap())], vec![0]),
+	            (vec![Rc::new(MatchingWord::new("thedoor".to_string(), 1, false).unwrap())], vec![0, 1]),
+	            (vec![Rc::new(MatchingWord::new("d".to_string(), 0, true).unwrap())], vec![2]),
+	            (vec![Rc::new(MatchingWord::new("thedoord".to_string(), 1, true).unwrap())], vec![0, 1, 2]),
+	            (vec![Rc::new(MatchingWord::new("doord".to_string(), 1, true).unwrap())], vec![1, 2]),
+        	]
+            ), TokenizerBuilder::default().build()),
+		},
+    ];
+
+    let format_options = &[
+        FormatOptions { highlight: false, crop: None },
+        FormatOptions { highlight: true, crop: None },
+        FormatOptions { highlight: false, crop: Some(10) },
+        FormatOptions { highlight: true, crop: Some(10) },
+        FormatOptions { highlight: false, crop: Some(20) },
+        FormatOptions { highlight: true, crop: Some(20) },
+    ];
+
+    for option in format_options {
+        let highlight = if option.highlight { "highlight" } else { "no-highlight" };
+
+        let name = match option.crop {
+            Some(size) => format!("{}-crop({})", highlight, size),
+            None => format!("{}-no-crop", highlight),
+        };
+
+        let mut group = c.benchmark_group(&name);
+        for conf in confs {
+            group.bench_function(conf.name, |b| {
+                b.iter(|| {
+                    let mut matcher = conf.matching_words.build(conf.text);
+                    matcher.format(*option);
+                })
+            });
+        }
+        group.finish();
+    }
+}
+
+criterion_group!(benches, bench_formatting);
+criterion_main!(benches);

benchmarks/benches/indexing.rs

@@ -6,7 +6,9 @@ use std::path::Path;
 
 use criterion::{criterion_group, criterion_main, Criterion};
 use milli::heed::{EnvOpenOptions, RwTxn};
-use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
+use milli::update::{
+    DeleteDocuments, IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings,
+};
 use milli::Index;
 use rand::seq::SliceRandom;
 use rand_chacha::rand_core::SeedableRng;
@@ -36,7 +38,7 @@ fn setup_index() -> Index {
 }
 
 fn setup_settings<'t>(
-    wtxn: &mut RwTxn<'t>,
+    wtxn: &mut RwTxn<'t, '_>,
     index: &'t Index,
     primary_key: &str,
     searchable_fields: &[&str],
@@ -264,7 +266,17 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                delete_documents_from_ids(index, document_ids_to_delete)
+                let mut wtxn = index.write_txn().unwrap();
+
+                for ids in document_ids_to_delete {
+                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
+                    builder.delete_documents(&ids);
+                    builder.execute().unwrap();
+                }
+
+                wtxn.commit().unwrap();
+
+                index.prepare_for_closing().wait();
             },
         )
     });
@@ -601,7 +613,17 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                delete_documents_from_ids(index, document_ids_to_delete)
+                let mut wtxn = index.write_txn().unwrap();
+
+                for ids in document_ids_to_delete {
+                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
+                    builder.delete_documents(&ids);
+                    builder.execute().unwrap();
+                }
+
+                wtxn.commit().unwrap();
+
+                index.prepare_for_closing().wait();
             },
         )
     });
@@ -853,31 +875,22 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                delete_documents_from_ids(index, document_ids_to_delete)
+                let mut wtxn = index.write_txn().unwrap();
+
+                for ids in document_ids_to_delete {
+                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
+                    builder.delete_documents(&ids);
+                    builder.execute().unwrap();
+                }
+
+                wtxn.commit().unwrap();
+
+                index.prepare_for_closing().wait();
             },
         )
     });
 }
 
-fn delete_documents_from_ids(index: Index, document_ids_to_delete: Vec<RoaringBitmap>) {
-    let mut wtxn = index.write_txn().unwrap();
-
-    let indexer_config = IndexerConfig::default();
-    for ids in document_ids_to_delete {
-        let config = IndexDocumentsConfig::default();
-
-        let mut builder =
-            IndexDocuments::new(&mut wtxn, &index, &indexer_config, config, |_| (), || false)
-                .unwrap();
-        (builder, _) = builder.remove_documents_from_db_no_batch(&ids).unwrap();
-        builder.execute().unwrap();
-    }
-
-    wtxn.commit().unwrap();
-
-    index.prepare_for_closing().wait();
-}
-
 fn indexing_movies_in_three_batches(c: &mut Criterion) {
     let mut group = c.benchmark_group("indexing");
     group.sample_size(BENCHMARK_ITERATION);
@@ -1099,7 +1112,17 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                delete_documents_from_ids(index, document_ids_to_delete)
+                let mut wtxn = index.write_txn().unwrap();
+
+                for ids in document_ids_to_delete {
+                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
+                    builder.delete_documents(&ids);
+                    builder.execute().unwrap();
+                }
+
+                wtxn.commit().unwrap();
+
+                index.prepare_for_closing().wait();
             },
         )
     });
@@ -1315,7 +1338,17 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                delete_documents_from_ids(index, document_ids_to_delete)
+                let mut wtxn = index.write_txn().unwrap();
+
+                for ids in document_ids_to_delete {
+                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
+                    builder.delete_documents(&ids);
+                    builder.execute().unwrap();
+                }
+
+                wtxn.commit().unwrap();
+
+                index.prepare_for_closing().wait();
             },
         )
     });

bors.toml

@@ -1,5 +1,5 @@
 status = [
-    'Tests on ubuntu-20.04',
+    'Tests on ubuntu-18.04',
     'Tests on macos-12',
     'Tests on windows-2022',
     'Run Clippy',

build-info/Cargo.toml

@@ -1,18 +0,0 @@
-[package]
-name = "build-info"
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
-# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
-
-[dependencies]
-time = { version = "0.3.36", features = ["parsing"] }
-
-[build-dependencies]
-anyhow = "1.0.86"
-vergen-git2 = "1.0.0"

build-info/build.rs

@@ -1,29 +0,0 @@
-fn main() {
-    if let Err(err) = emit_git_variables() {
-        println!("cargo:warning=vergen: {}", err);
-    }
-}
-
-fn emit_git_variables() -> anyhow::Result<()> {
-    println!("cargo::rerun-if-env-changed=MEILI_NO_VERGEN");
-
-    let has_vergen =
-        !matches!(std::env::var_os("MEILI_NO_VERGEN"), Some(x) if x != "false" && x != "0");
-
-    anyhow::ensure!(has_vergen, "disabled via `MEILI_NO_VERGEN`");
-
-    // Note: any code that needs VERGEN_ environment variables should take care to define them manually in the Dockerfile and pass them
-    // in the corresponding GitHub workflow (publish_docker.yml).
-    // This is due to the Dockerfile building the binary outside of the git directory.
-    let mut builder = vergen_git2::Git2Builder::default();
-
-    builder.branch(true);
-    builder.commit_timestamp(true);
-    builder.commit_message(true);
-    builder.describe(true, true, None);
-    builder.sha(false);
-
-    let git2 = builder.build()?;
-
-    vergen_git2::Emitter::default().fail_on_error().add_instructions(&git2)?.emit()
-}

build-info/src/lib.rs

@@ -1,203 +0,0 @@
-use time::format_description::well_known::Iso8601;
-
-#[derive(Debug, Clone)]
-pub struct BuildInfo {
-    pub branch: Option<&'static str>,
-    pub describe: Option<DescribeResult>,
-    pub commit_sha1: Option<&'static str>,
-    pub commit_msg: Option<&'static str>,
-    pub commit_timestamp: Option<time::OffsetDateTime>,
-}
-
-impl BuildInfo {
-    pub fn from_build() -> Self {
-        let branch: Option<&'static str> = option_env!("VERGEN_GIT_BRANCH");
-        let describe = DescribeResult::from_build();
-        let commit_sha1 = option_env!("VERGEN_GIT_SHA");
-        let commit_msg = option_env!("VERGEN_GIT_COMMIT_MESSAGE");
-        let commit_timestamp = option_env!("VERGEN_GIT_COMMIT_TIMESTAMP");
-
-        let commit_timestamp = commit_timestamp.and_then(|commit_timestamp| {
-            time::OffsetDateTime::parse(commit_timestamp, &Iso8601::DEFAULT).ok()
-        });
-
-        Self { branch, describe, commit_sha1, commit_msg, commit_timestamp }
-    }
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
-pub enum DescribeResult {
-    Prototype { name: &'static str },
-    Release { version: &'static str, major: u64, minor: u64, patch: u64 },
-    Prerelease { version: &'static str, major: u64, minor: u64, patch: u64, rc: u64 },
-    NotATag { describe: &'static str },
-}
-
-impl DescribeResult {
-    pub fn new(describe: &'static str) -> Self {
-        if let Some(name) = prototype_name(describe) {
-            Self::Prototype { name }
-        } else if let Some(release) = release_version(describe) {
-            release
-        } else if let Some(prerelease) = prerelease_version(describe) {
-            prerelease
-        } else {
-            Self::NotATag { describe }
-        }
-    }
-
-    pub fn from_build() -> Option<Self> {
-        let describe: &'static str = option_env!("VERGEN_GIT_DESCRIBE")?;
-        Some(Self::new(describe))
-    }
-
-    pub fn as_tag(&self) -> Option<&'static str> {
-        match self {
-            DescribeResult::Prototype { name } => Some(name),
-            DescribeResult::Release { version, .. } => Some(version),
-            DescribeResult::Prerelease { version, .. } => Some(version),
-            DescribeResult::NotATag { describe: _ } => None,
-        }
-    }
-
-    pub fn as_prototype(&self) -> Option<&'static str> {
-        match self {
-            DescribeResult::Prototype { name } => Some(name),
-            DescribeResult::Release { .. }
-            | DescribeResult::Prerelease { .. }
-            | DescribeResult::NotATag { .. } => None,
-        }
-    }
-}
-
-/// Parses the input as a prototype name.
-///
-/// Returns `Some(prototype_name)` if the following conditions are met on this value:
-///
-/// 1. starts with `prototype-`,
-/// 2. ends with `-<some_number>`,
-/// 3. does not end with `<some_number>-<some_number>`.
-///
-/// Otherwise, returns `None`.
-fn prototype_name(describe: &'static str) -> Option<&'static str> {
-    if !describe.starts_with("prototype-") {
-        return None;
-    }
-
-    let mut rsplit_prototype = describe.rsplit('-');
-    // last component MUST be a number
-    rsplit_prototype.next()?.parse::<u64>().ok()?;
-    // before than last component SHALL NOT be a number
-    rsplit_prototype.next()?.parse::<u64>().err()?;
-
-    Some(describe)
-}
-
-fn release_version(describe: &'static str) -> Option<DescribeResult> {
-    if !describe.starts_with('v') {
-        return None;
-    }
-
-    // full release version don't contain a `-`
-    if describe.contains('-') {
-        return None;
-    }
-
-    // full release version parse as vX.Y.Z, with X, Y, Z numbers.
-    let mut dots = describe[1..].split('.');
-    let major: u64 = dots.next()?.parse().ok()?;
-    let minor: u64 = dots.next()?.parse().ok()?;
-    let patch: u64 = dots.next()?.parse().ok()?;
-
-    if dots.next().is_some() {
-        return None;
-    }
-
-    Some(DescribeResult::Release { version: describe, major, minor, patch })
-}
-
-fn prerelease_version(describe: &'static str) -> Option<DescribeResult> {
-    // prerelease version is in the shape vM.N.P-rc.C
-    let mut hyphen = describe.rsplit('-');
-    let prerelease = hyphen.next()?;
-    if !prerelease.starts_with("rc.") {
-        return None;
-    }
-
-    let rc: u64 = prerelease[3..].parse().ok()?;
-
-    let release = hyphen.next()?;
-
-    let DescribeResult::Release { version: _, major, minor, patch } = release_version(release)?
-    else {
-        return None;
-    };
-
-    Some(DescribeResult::Prerelease { version: describe, major, minor, patch, rc })
-}
-
-#[cfg(test)]
-mod test {
-    use super::DescribeResult;
-
-    fn assert_not_a_tag(describe: &'static str) {
-        assert_eq!(DescribeResult::NotATag { describe }, DescribeResult::new(describe))
-    }
-
-    fn assert_proto(describe: &'static str) {
-        assert_eq!(DescribeResult::Prototype { name: describe }, DescribeResult::new(describe))
-    }
-
-    fn assert_release(describe: &'static str, major: u64, minor: u64, patch: u64) {
-        assert_eq!(
-            DescribeResult::Release { version: describe, major, minor, patch },
-            DescribeResult::new(describe)
-        )
-    }
-
-    fn assert_prerelease(describe: &'static str, major: u64, minor: u64, patch: u64, rc: u64) {
-        assert_eq!(
-            DescribeResult::Prerelease { version: describe, major, minor, patch, rc },
-            DescribeResult::new(describe)
-        )
-    }
-
-    #[test]
-    fn not_a_tag() {
-        assert_not_a_tag("whatever-fuzzy");
-        assert_not_a_tag("whatever-fuzzy-5-ggg-dirty");
-        assert_not_a_tag("whatever-fuzzy-120-ggg-dirty");
-
-        // technically a tag, but not a proto nor a version, so not parsed as a tag
-        assert_not_a_tag("whatever");
-
-        // dirty version
-        assert_not_a_tag("v1.7.0-1-ggga-dirty");
-        assert_not_a_tag("v1.7.0-rc.1-1-ggga-dirty");
-
-        // after version
-        assert_not_a_tag("v1.7.0-1-ggga");
-        assert_not_a_tag("v1.7.0-rc.1-1-ggga");
-
-        // after proto
-        assert_not_a_tag("protoype-tag-0-1-ggga");
-        assert_not_a_tag("protoype-tag-0-1-ggga-dirty");
-    }
-
-    #[test]
-    fn prototype() {
-        assert_proto("prototype-tag-0");
-        assert_proto("prototype-tag-10");
-        assert_proto("prototype-long-name-tag-10");
-    }
-
-    #[test]
-    fn release() {
-        assert_release("v1.7.2", 1, 7, 2);
-    }
-
-    #[test]
-    fn prerelease() {
-        assert_prerelease("v1.7.2-rc.3", 1, 7, 2, 3);
-    }
-}

config.toml

@@ -1,134 +1,120 @@
 # This file shows the default configuration of Meilisearch.
-# All variables are defined here: https://www.meilisearch.com/docs/learn/configuration/instance_options#environment-variables
+# All variables are defined here: https://docs.meilisearch.com/learn/configuration/instance_options.html#environment-variables
 
-# Designates the location where database files will be created and retrieved.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#database-path
 db_path = "./data.ms"
+# Designates the location where database files will be created and retrieved.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#database-path
 
-# Configures the instance's environment. Value must be either `production` or `development`.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#environment
 env = "development"
+# Configures the instance's environment. Value must be either `production` or `development`.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#environment
 
-# The address on which the HTTP server will listen.
 http_addr = "localhost:7700"
+# The address on which the HTTP server will listen.
 
-# Sets the instance's master key, automatically protecting all routes except GET /health.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#master-key
 # master_key = "YOUR_MASTER_KEY_VALUE"
+# Sets the instance's master key, automatically protecting all routes except GET /health.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#master-key
 
+# no_analytics = true
 # Deactivates Meilisearch's built-in telemetry when provided.
 # Meilisearch automatically collects data from all instances that do not opt out using this flag.
 # All gathered data is used solely for the purpose of improving Meilisearch, and can be deleted at any time.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#disable-analytics
-# no_analytics = true
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#disable-analytics
 
-# Sets the maximum size of accepted payloads.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#payload-limit-size
 http_payload_size_limit = "100 MB"
+# Sets the maximum size of accepted payloads.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#payload-limit-size
 
+log_level = "INFO"
 # Defines how much detail should be present in Meilisearch's logs.
 # Meilisearch currently supports six log levels, listed in order of increasing verbosity:  `OFF`, `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#log-level
-log_level = "INFO"
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#log-level
 
-# Sets the maximum amount of RAM Meilisearch can use when indexing.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#max-indexing-memory
 # max_indexing_memory = "2 GiB"
+# Sets the maximum amount of RAM Meilisearch can use when indexing.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#max-indexing-memory
 
-# Sets the maximum number of threads Meilisearch can use during indexing.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#max-indexing-threads
 # max_indexing_threads = 4
+# Sets the maximum number of threads Meilisearch can use during indexing.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#max-indexing-threads
 
 #############
 ### DUMPS ###
 #############
 
-# Sets the directory where Meilisearch will create dump files.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#dump-directory
 dump_dir = "dumps/"
+# Sets the directory where Meilisearch will create dump files.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#dump-directory
 
-# Imports the dump file located at the specified path. Path must point to a .dump file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#import-dump
 # import_dump = "./path/to/my/file.dump"
+# Imports the dump file located at the specified path. Path must point to a .dump file.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#import-dump
 
-# Prevents Meilisearch from throwing an error when `import_dump` does not point to a valid dump file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-missing-dump
 ignore_missing_dump = false
+# Prevents Meilisearch from throwing an error when `import_dump` does not point to a valid dump file.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ignore-missing-dump
 
-# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_dump`.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-dump-if-db-exists
 ignore_dump_if_db_exists = false
+# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_dump`.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ignore-dump-if-db-exists
 
 
 #################
 ### SNAPSHOTS ###
 #################
 
+schedule_snapshot = false
 # Enables scheduled snapshots when true, disable when false (the default).
 # If the value is given as an integer, then enables the scheduled snapshot with the passed value as the interval
 # between each snapshot, in seconds.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#schedule-snapshot-creation
-schedule_snapshot = false
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#schedule-snapshot-creation
 
-# Sets the directory where Meilisearch will store snapshots.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#snapshot-destination
 snapshot_dir = "snapshots/"
+# Sets the directory where Meilisearch will store snapshots.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-destination
 
-# Launches Meilisearch after importing a previously-generated snapshot at the given filepath.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#import-snapshot
 # import_snapshot = "./path/to/my/snapshot"
+# Launches Meilisearch after importing a previously-generated snapshot at the given filepath.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#import-snapshot
 
-# Prevents a Meilisearch instance from throwing an error when `import_snapshot` does not point to a valid snapshot file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-missing-snapshot
 ignore_missing_snapshot = false
+# Prevents a Meilisearch instance from throwing an error when `import_snapshot` does not point to a valid snapshot file.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ignore-missing-snapshot
 
-# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_snapshot`.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-snapshot-if-db-exists
 ignore_snapshot_if_db_exists = false
+# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_snapshot`.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ignore-snapshot-if-db-exists
 
 
 ###########
 ### SSL ###
 ###########
 
-# Enables client authentication in the specified path.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-authentication-path
 # ssl_auth_path = "./path/to/root"
+# Enables client authentication in the specified path.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-authentication-path
 
-# Sets the server's SSL certificates.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-certificates-path
 # ssl_cert_path = "./path/to/certfile"
+# Sets the server's SSL certificates.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-certificates-path
 
-# Sets the server's SSL key files.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-key-path
 # ssl_key_path = "./path/to/private-key"
+# Sets the server's SSL key files.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-key-path
 
-# Sets the server's OCSP file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-ocsp-path
 # ssl_ocsp_path = "./path/to/ocsp-file"
+# Sets the server's OCSP file.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-ocsp-path
 
-# Makes SSL authentication mandatory.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-require-auth
 ssl_require_auth = false
+# Makes SSL authentication mandatory.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-require-auth
 
-# Activates SSL session resumption.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-resumption
 ssl_resumption = false
+# Activates SSL session resumption.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-resumption
 
-# Activates SSL tickets.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-tickets
 ssl_tickets = false
-
-#############################
-### Experimental features ###
-#############################
-
-# Experimental metrics feature. For more information, see: <https://github.com/meilisearch/meilisearch/discussions/3518>
-# Enables the Prometheus metrics on the `GET /metrics` endpoint.
-experimental_enable_metrics = false
-
-# Experimental RAM reduction during indexing, do not use in production, see: <https://github.com/meilisearch/product/discussions/652>
-experimental_reduce_indexing_memory_usage = false
-
-# Experimentally reduces the maximum number of tasks that will be processed at once, see: <https://github.com/orgs/meilisearch/discussions/713>
-# experimental_max_number_of_batched_tasks = 100
+# Activates SSL tickets.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-tickets

download-latest.sh

@@ -103,7 +103,7 @@ not_available_failure_usage() {
     printf "$RED%s\n$DEFAULT" 'ERROR: Meilisearch binary is not available for your OS distribution or your architecture yet.'
     echo ''
     echo 'However, you can easily compile the binary from the source files.'
-    echo 'Follow the steps at the page ("Source" tab): https://www.meilisearch.com/docs/learn/getting_started/installation'
+    echo 'Follow the steps at the page ("Source" tab): https://docs.meilisearch.com/learn/getting_started/installation.html'
 }
 
 fetch_release_failure_usage() {

dump/Cargo.toml

@@ -11,21 +11,22 @@ readme.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.86"
-flate2 = "1.0.30"
-http = "1.1.0"
+anyhow = "1.0.65"
+flate2 = "1.0.22"
+http = "0.2.8"
+log = "0.4.17"
+meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-once_cell = "1.19.0"
-regex = "1.10.5"
-roaring = { version = "0.10.6", features = ["serde"] }
-serde = { version = "1.0.204", features = ["derive"] }
-serde_json = { version = "1.0.120", features = ["preserve_order"] }
-tar = "0.4.41"
-tempfile = "3.10.1"
-thiserror = "1.0.61"
-time = { version = "0.3.36", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-tracing = "0.1.40"
-uuid = { version = "1.10.0", features = ["serde", "v4"] }
+once_cell = "1.15.0"
+regex = "1.6.0"
+roaring = { version = "0.10.0", features = ["serde"] }
+serde = { version = "1.0.136", features = ["derive"] }
+serde_json = { version = "1.0.85", features = ["preserve_order"] }
+tar = "0.4.38"
+tempfile = "3.3.0"
+thiserror = "1.0.30"
+time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+uuid = { version = "1.1.2", features = ["serde", "v4"] }
 
 [dev-dependencies]
 big_s = "1.0.2"

dump/src/lib.rs

@@ -101,14 +101,6 @@ pub enum KindDump {
         documents_ids: Vec<String>,
     },
     DocumentClear,
-    DocumentDeletionByFilter {
-        filter: serde_json::Value,
-    },
-    DocumentEdition {
-        filter: Option<serde_json::Value>,
-        context: Option<serde_json::Map<String, serde_json::Value>>,
-        function: String,
-    },
     Settings {
         settings: Box<meilisearch_types::settings::Settings<Unchecked>>,
         is_deletion: bool,
@@ -174,12 +166,6 @@ impl From<KindWithContent> for KindDump {
             KindWithContent::DocumentDeletion { documents_ids, .. } => {
                 KindDump::DocumentDeletion { documents_ids }
             }
-            KindWithContent::DocumentDeletionByFilter { filter_expr, .. } => {
-                KindDump::DocumentDeletionByFilter { filter: filter_expr }
-            }
-            KindWithContent::DocumentEdition { filter_expr, context, function, .. } => {
-                KindDump::DocumentEdition { filter: filter_expr, context, function }
-            }
             KindWithContent::DocumentClear { .. } => KindDump::DocumentClear,
             KindWithContent::SettingsUpdate {
                 new_settings,
@@ -216,14 +202,12 @@ pub(crate) mod test {
     use std::str::FromStr;
 
     use big_s::S;
-    use maplit::{btreemap, btreeset};
-    use meilisearch_types::facet_values_sort::FacetValuesSort;
-    use meilisearch_types::features::RuntimeTogglableFeatures;
+    use maplit::btreeset;
     use meilisearch_types::index_uid_pattern::IndexUidPattern;
     use meilisearch_types::keys::{Action, Key};
-    use meilisearch_types::milli;
     use meilisearch_types::milli::update::Setting;
-    use meilisearch_types::settings::{Checked, FacetingSettings, Settings};
+    use meilisearch_types::milli::{self};
+    use meilisearch_types::settings::{Checked, Settings};
     use meilisearch_types::tasks::{Details, Status};
     use serde_json::{json, Map, Value};
     use time::macros::datetime;
@@ -264,29 +248,17 @@ pub(crate) mod test {
 
     pub fn create_test_settings() -> Settings<Checked> {
         let settings = Settings {
-            displayed_attributes: Setting::Set(vec![S("race"), S("name")]).into(),
-            searchable_attributes: Setting::Set(vec![S("name"), S("race")]).into(),
+            displayed_attributes: Setting::Set(vec![S("race"), S("name")]),
+            searchable_attributes: Setting::Set(vec![S("name"), S("race")]),
             filterable_attributes: Setting::Set(btreeset! { S("race"), S("age") }),
             sortable_attributes: Setting::Set(btreeset! { S("age") }),
             ranking_rules: Setting::NotSet,
             stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
             synonyms: Setting::NotSet,
             distinct_attribute: Setting::NotSet,
-            proximity_precision: Setting::NotSet,
             typo_tolerance: Setting::NotSet,
-            faceting: Setting::Set(FacetingSettings {
-                max_values_per_facet: Setting::Set(111),
-                sort_facet_values_by: Setting::Set(
-                    btreemap! { S("age") => FacetValuesSort::Count },
-                ),
-            }),
+            faceting: Setting::NotSet,
             pagination: Setting::NotSet,
-            embedders: Setting::NotSet,
-            search_cutoff_ms: Setting::NotSet,
-            localized_attributes: Setting::NotSet,
             _kind: std::marker::PhantomData,
         };
         settings.check()
@@ -434,11 +406,6 @@ pub(crate) mod test {
         }
         keys.flush().unwrap();
 
-        // ========== experimental features
-        let features = create_test_features();
-
-        dump.create_experimental_features(features).unwrap();
-
         // create the dump
         let mut file = tempfile::tempfile().unwrap();
         dump.persist_to(&mut file).unwrap();
@@ -447,10 +414,6 @@ pub(crate) mod test {
         file
     }
 
-    fn create_test_features() -> RuntimeTogglableFeatures {
-        RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
-    }
-
     #[test]
     fn test_creating_and_read_dump() {
         let mut file = create_test_dump();
@@ -495,9 +458,5 @@ pub(crate) mod test {
         for (key, expected) in dump.keys().unwrap().zip(create_test_api_keys()) {
             assert_eq!(key.unwrap(), expected);
         }
-
-        // ==== checking the features
-        let expected = create_test_features();
-        assert_eq!(dump.features().unwrap().unwrap(), expected);
     }
 }

dump/src/reader/compat/v1_to_v2.rs

@@ -120,7 +120,7 @@ impl From<v1::settings::Settings> for v2::Settings<v2::Unchecked> {
                                 criterion.as_ref().map(ToString::to_string)
                             }
                             Err(()) => {
-                                tracing::warn!(
+                                log::warn!(
                                     "Could not import the following ranking rule: `{}`.",
                                     ranking_rule
                                 );
@@ -152,11 +152,11 @@ impl From<v1::update::UpdateStatus> for Option<v2::updates::UpdateStatus> {
         use v2::updates::UpdateStatus as UpdateStatusV2;
         Some(match source {
             UpdateStatusV1::Enqueued { content } => {
-                tracing::warn!(
+                log::warn!(
                     "Cannot import task {} (importing enqueued tasks from v1 dumps is unsupported)",
                     content.update_id
                 );
-                tracing::warn!("Task will be skipped in the queue of imported tasks.");
+                log::warn!("Task will be skipped in the queue of imported tasks.");
 
                 return None;
             }
@@ -229,7 +229,7 @@ impl From<v1::update::UpdateType> for Option<v2::updates::UpdateMeta> {
         Some(match source {
             v1::update::UpdateType::ClearAll => v2::updates::UpdateMeta::ClearDocuments,
             v1::update::UpdateType::Customs => {
-                tracing::warn!("Ignoring task with type 'Customs' that is no longer supported");
+                log::warn!("Ignoring task with type 'Customs' that is no longer supported");
                 return None;
             }
             v1::update::UpdateType::DocumentsAddition { .. } => {
@@ -296,7 +296,7 @@ impl From<v1::settings::RankingRule> for Option<v2::settings::Criterion> {
             v1::settings::RankingRule::Proximity => Some(v2::settings::Criterion::Proximity),
             v1::settings::RankingRule::Attribute => Some(v2::settings::Criterion::Attribute),
             v1::settings::RankingRule::WordsPosition => {
-                tracing::warn!("Removing the 'WordsPosition' ranking rule that is no longer supported, please check the resulting ranking rules of your indexes");
+                log::warn!("Removing the 'WordsPosition' ranking rule that is no longer supported, please check the resulting ranking rules of your indexes");
                 None
             }
             v1::settings::RankingRule::Exactness => Some(v2::settings::Criterion::Exactness),

dump/src/reader/compat/v2_to_v3.rs

@@ -1,3 +1,4 @@
+use std::convert::TryInto;
 use std::str::FromStr;
 
 use time::OffsetDateTime;
@@ -24,6 +25,7 @@ impl CompatV2ToV3 {
             CompatV2ToV3::Compat(compat) => compat.index_uuid(),
         };
         v2_uuids
+            .into_iter()
             .into_iter()
             .map(|index| v3::meta::IndexUuid { uid: index.uid, uuid: index.uuid })
             .collect()
@@ -145,8 +147,8 @@ impl From<v2::updates::UpdateStatus> for v3::updates::UpdateStatus {
                         started_processing_at: processing.started_processing_at,
                     }),
                     Err(e) => {
-                        tracing::warn!("Error with task {}: {}", processing.from.update_id, e);
-                        tracing::warn!("Task will be marked as `Failed`.");
+                        log::warn!("Error with task {}: {}", processing.from.update_id, e);
+                        log::warn!("Task will be marked as `Failed`.");
                         v3::updates::UpdateStatus::Failed(v3::updates::Failed {
                             from: v3::updates::Processing {
                                 from: v3::updates::Enqueued {
@@ -171,8 +173,8 @@ impl From<v2::updates::UpdateStatus> for v3::updates::UpdateStatus {
                         enqueued_at: enqueued.enqueued_at,
                     }),
                     Err(e) => {
-                        tracing::warn!("Error with task {}: {}", enqueued.update_id, e);
-                        tracing::warn!("Task will be marked as `Failed`.");
+                        log::warn!("Error with task {}: {}", enqueued.update_id, e);
+                        log::warn!("Task will be marked as `Failed`.");
                         v3::updates::UpdateStatus::Failed(v3::updates::Failed {
                             from: v3::updates::Processing {
                                 from: v3::updates::Enqueued {
@@ -352,7 +354,7 @@ impl From<String> for v3::Code {
             "malformed_payload" => v3::Code::MalformedPayload,
             "missing_payload" => v3::Code::MissingPayload,
             other => {
-                tracing::warn!("Unknown error code {}", other);
+                log::warn!("Unknown error code {}", other);
                 v3::Code::UnretrievableErrorCode
             }
         }
@@ -425,7 +427,7 @@ pub(crate) mod test {
         let mut dump = v2::V2Reader::open(dir).unwrap().to_v3();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/compat/v3_to_v4.rs

@@ -76,20 +76,20 @@ impl CompatV3ToV4 {
                         let index_uid = match index_uid {
                             Some(uid) => uid,
                             None => {
-                                tracing::warn!(
+                                log::warn!(
                                     "Error while importing the update {}.",
                                     task.update.id()
                                 );
-                                tracing::warn!(
+                                log::warn!(
                                     "The index associated to the uuid `{}` could not be retrieved.",
                                     task.uuid.to_string()
                                 );
                                 if task.update.is_finished() {
                                     // we're fucking with his history but not his data, that's ok-ish.
-                                    tracing::warn!("The index-uuid will be set as `unknown`.");
+                                    log::warn!("The index-uuid will be set as `unknown`.");
                                     String::from("unknown")
                                 } else {
-                                    tracing::warn!("The task will be ignored.");
+                                    log::warn!("The task will be ignored.");
                                     return None;
                                 }
                             }
@@ -358,7 +358,7 @@ pub(crate) mod test {
         let mut dump = v3::V3Reader::open(dir).unwrap().to_v4();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/compat/v4_to_v5.rs

@@ -305,7 +305,7 @@ impl From<v4::ResponseError> for v5::ResponseError {
             "invalid_api_key_expires_at" => v5::Code::InvalidApiKeyExpiresAt,
             "invalid_api_key_description" => v5::Code::InvalidApiKeyDescription,
             other => {
-                tracing::warn!("Unknown error code {}", other);
+                log::warn!("Unknown error code {}", other);
                 v5::Code::UnretrievableErrorCode
             }
         };
@@ -394,8 +394,8 @@ pub(crate) mod test {
         let mut dump = v4::V4Reader::open(dir).unwrap().to_v5();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
+        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/compat/v5_to_v6.rs

@@ -191,10 +191,6 @@ impl CompatV5ToV6 {
             })
         })))
     }
-
-    pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
-        Ok(None)
-    }
 }
 
 pub enum CompatIndexV5ToV6 {
@@ -304,7 +300,7 @@ impl From<v5::ResponseError> for v6::ResponseError {
             "immutable_field" => v6::Code::BadRequest,
             "api_key_already_exists" => v6::Code::ApiKeyAlreadyExists,
             other => {
-                tracing::warn!("Unknown error code {}", other);
+                log::warn!("Unknown error code {}", other);
                 v6::Code::UnretrievableErrorCode
             }
         };
@@ -315,8 +311,8 @@ impl From<v5::ResponseError> for v6::ResponseError {
 impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
     fn from(settings: v5::Settings<T>) -> Self {
         v6::Settings {
-            displayed_attributes: v6::Setting::from(settings.displayed_attributes).into(),
-            searchable_attributes: v6::Setting::from(settings.searchable_attributes).into(),
+            displayed_attributes: settings.displayed_attributes.into(),
+            searchable_attributes: settings.searchable_attributes.into(),
             filterable_attributes: settings.filterable_attributes.into(),
             sortable_attributes: settings.sortable_attributes.into(),
             ranking_rules: {
@@ -329,7 +325,7 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                                     new_ranking_rules.push(new_rule);
                                 }
                                 Err(_) => {
-                                    tracing::warn!("Error while importing settings. The ranking rule `{rule}` does not exist anymore.")
+                                    log::warn!("Error while importing settings. The ranking rule `{rule}` does not exist anymore.")
                                 }
                             }
                         }
@@ -340,12 +336,8 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                 }
             },
             stop_words: settings.stop_words.into(),
-            non_separator_tokens: v6::Setting::NotSet,
-            separator_tokens: v6::Setting::NotSet,
-            dictionary: v6::Setting::NotSet,
             synonyms: settings.synonyms.into(),
             distinct_attribute: settings.distinct_attribute.into(),
-            proximity_precision: v6::Setting::NotSet,
             typo_tolerance: match settings.typo_tolerance {
                 v5::Setting::Set(typo) => v6::Setting::Set(v6::TypoTolerance {
                     enabled: typo.enabled.into(),
@@ -366,7 +358,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
             faceting: match settings.faceting {
                 v5::Setting::Set(faceting) => v6::Setting::Set(v6::FacetingSettings {
                     max_values_per_facet: faceting.max_values_per_facet.into(),
-                    sort_facet_values_by: v6::Setting::NotSet,
                 }),
                 v5::Setting::Reset => v6::Setting::Reset,
                 v5::Setting::NotSet => v6::Setting::NotSet,
@@ -378,9 +369,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                 v5::Setting::Reset => v6::Setting::Reset,
                 v5::Setting::NotSet => v6::Setting::NotSet,
             },
-            embedders: v6::Setting::NotSet,
-            localized_attributes: v6::Setting::NotSet,
-            search_cutoff_ms: v6::Setting::NotSet,
             _kind: std::marker::PhantomData,
         }
     }
@@ -443,8 +431,8 @@ pub(crate) mod test {
         let mut dump = v5::V5Reader::open(dir).unwrap().to_v6();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
+        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/mod.rs

@@ -13,12 +13,12 @@ use crate::{Result, Version};
 
 mod compat;
 
-mod v1;
-mod v2;
-mod v3;
-mod v4;
-mod v5;
-mod v6;
+pub(self) mod v1;
+pub(self) mod v2;
+pub(self) mod v3;
+pub(self) mod v4;
+pub(self) mod v5;
+pub(self) mod v6;
 
 pub type Document = serde_json::Map<String, serde_json::Value>;
 pub type UpdateFile = dyn Iterator<Item = Result<Document>>;
@@ -107,13 +107,6 @@ impl DumpReader {
             DumpReader::Compat(compat) => compat.keys(),
         }
     }
-
-    pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
-        match self {
-            DumpReader::Current(current) => Ok(current.features()),
-            DumpReader::Compat(compat) => compat.features(),
-        }
-    }
 }
 
 impl From<V6Reader> for DumpReader {
@@ -195,187 +188,6 @@ pub(crate) mod test {
     use meili_snap::insta;
 
     use super::*;
-    use crate::reader::v6::RuntimeTogglableFeatures;
-
-    #[test]
-    fn import_dump_v6_with_vectors() {
-        // dump containing two indexes
-        //
-        // "vector", configured with an embedder
-        // contains:
-        // - one document with an overriden vector,
-        // - one document with a natural vector
-        // - one document with a _vectors map containing one additional embedder name and a natural vector
-        // - one document with a _vectors map containing one additional embedder name and an overriden vector
-        //
-        // "novector", no embedder
-        // contains:
-        // - a document without vector
-        // - a document with a random _vectors field
-        let dump = File::open("tests/assets/v6-with-vectors.dump").unwrap();
-        let mut dump = DumpReader::open(dump).unwrap();
-
-        // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2024-05-16 15:51:34.151044 +00:00:00");
-        insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
-
-        // tasks
-        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"278f63325ef06ca04d01df98d8207b94");
-        assert_eq!(update_files.len(), 10);
-        assert!(update_files[0].is_none()); // the dump creation
-        assert!(update_files[1].is_none());
-        assert!(update_files[2].is_none());
-        assert!(update_files[3].is_none());
-        assert!(update_files[4].is_none());
-        assert!(update_files[5].is_none());
-        assert!(update_files[6].is_none());
-        assert!(update_files[7].is_none());
-        assert!(update_files[8].is_none());
-        assert!(update_files[9].is_none());
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut vector_index = indexes.pop().unwrap();
-        let mut novector_index = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        // vector
-
-        insta::assert_json_snapshot!(vector_index.metadata(), @r###"
-        {
-          "uid": "vector",
-          "primaryKey": "id",
-          "createdAt": "2024-05-16T15:33:17.240962Z",
-          "updatedAt": "2024-05-16T15:40:55.723052Z"
-        }
-        "###);
-
-        {
-            let documents: Result<Vec<_>> = vector_index.documents().unwrap().collect();
-            let mut documents = documents.unwrap();
-            assert_eq!(documents.len(), 4);
-
-            documents.sort_by_key(|doc| doc.get("id").unwrap().to_string());
-
-            {
-                let document = documents.pop().unwrap();
-                insta::assert_json_snapshot!(document);
-            }
-
-            {
-                let document = documents.pop().unwrap();
-                insta::assert_json_snapshot!(document);
-            }
-
-            {
-                let document = documents.pop().unwrap();
-                insta::assert_json_snapshot!(document);
-            }
-
-            {
-                let document = documents.pop().unwrap();
-                insta::assert_json_snapshot!(document);
-            }
-        }
-
-        // novector
-
-        insta::assert_json_snapshot!(novector_index.metadata(), @r###"
-        {
-          "uid": "novector",
-          "primaryKey": "id",
-          "createdAt": "2024-05-16T15:33:03.568055Z",
-          "updatedAt": "2024-05-16T15:33:07.530217Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(novector_index.settings().unwrap().embedders, @"null");
-
-        {
-            let documents: Result<Vec<_>> = novector_index.documents().unwrap().collect();
-            let mut documents = documents.unwrap();
-            assert_eq!(documents.len(), 2);
-
-            documents.sort_by_key(|doc| doc.get("id").unwrap().to_string());
-
-            {
-                let document = documents.pop().unwrap();
-                insta::assert_json_snapshot!(document, @r###"
-                {
-                  "id": "e1",
-                  "other": "random1",
-                  "_vectors": "toto"
-                }
-                "###);
-            }
-
-            {
-                let document = documents.pop().unwrap();
-                insta::assert_json_snapshot!(document, @r###"
-                {
-                  "id": "e0",
-                  "other": "random0"
-                }
-                "###);
-            }
-        }
-
-        assert_eq!(
-            dump.features().unwrap().unwrap(),
-            RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
-        );
-    }
-
-    #[test]
-    fn import_dump_v6_experimental() {
-        let dump = File::open("tests/assets/v6-with-experimental.dump").unwrap();
-        let mut dump = DumpReader::open(dump).unwrap();
-
-        // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2023-07-06 7:10:27.21958 +00:00:00");
-        insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
-
-        // tasks
-        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"d45cd8571703e58ae53c7bd7ce3f5c22");
-        assert_eq!(update_files.len(), 2);
-        assert!(update_files[0].is_none()); // the dump creation
-        assert!(update_files[1].is_none()); // the processed document addition
-
-        // keys
-        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"13c2da155e9729c2344688cab29af71d");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut test = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        insta::assert_json_snapshot!(test.metadata(), @r###"
-        {
-          "uid": "test",
-          "primaryKey": "id",
-          "createdAt": "2023-07-06T07:07:41.364694Z",
-          "updatedAt": "2023-07-06T07:07:41.396114Z"
-        }
-        "###);
-
-        assert_eq!(test.documents().unwrap().count(), 1);
-
-        assert_eq!(
-            dump.features().unwrap().unwrap(),
-            RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
-        );
-    }
 
     #[test]
     fn import_dump_v5() {
@@ -383,8 +195,8 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
+        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
@@ -453,8 +265,6 @@ pub(crate) mod test {
         let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
-
-        assert_eq!(dump.features().unwrap(), None);
     }
 
     #[test]
@@ -463,8 +273,8 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
+        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
@@ -540,7 +350,7 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
         assert_eq!(dump.instance_uid().unwrap(), None);
 
         // tasks
@@ -633,7 +443,7 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
         assert_eq!(dump.instance_uid().unwrap(), None);
 
         // tasks
@@ -660,12 +470,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2022-10-09T20:27:22.688964637Z",
-          "updatedAt": "2022-10-09T20:27:23.951017769Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -675,12 +485,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2022-10-09T20:27:22.197788495Z",
-          "updatedAt": "2022-10-09T20:28:01.93111053Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -705,12 +515,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2022-10-09T20:27:24.242683494Z",
-          "updatedAt": "2022-10-09T20:27:24.312809641Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -726,7 +536,7 @@ pub(crate) mod test {
         let mut dump = DumpReader::open(dump).unwrap();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
         assert_eq!(dump.instance_uid().unwrap(), None);
 
         // tasks
@@ -751,12 +561,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2023-01-30T16:25:56.595257Z",
-          "updatedAt": "2023-01-30T16:25:58.70348Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -766,12 +576,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2023-01-30T16:25:56.192178Z",
-          "updatedAt": "2023-01-30T16:25:56.455714Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -781,12 +591,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"0227598af846e574139ee0b80e03a720");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2023-01-30T16:25:58.876405Z",
-          "updatedAt": "2023-01-30T16:25:59.079906Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 

dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-5.snap

@@ -1,783 +0,0 @@
----
-source: dump/src/reader/mod.rs
-expression: document
----
-{
-  "id": "e3",
-  "desc": "overriden vector + map",
-  "_vectors": {
-    "default": [
-      0.2,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1
-    ],
-    "toto": [
-      0.1
-    ]
-  }
-}

dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-6.snap

@@ -1,786 +0,0 @@
----
-source: dump/src/reader/mod.rs
-expression: document
----
-{
-  "id": "e2",
-  "desc": "natural vector + map",
-  "_vectors": {
-    "toto": [],
-    "default": {
-      "embeddings": [
-        [
-          -0.05189208313822746,
-          -0.9273212552070618,
-          0.1443813145160675,
-          0.0932632014155388,
-          0.2665371894836426,
-          0.36266782879829407,
-          0.6402910947799683,
-          0.32014018297195435,
-          0.030915971845388412,
-          -0.9312191605567932,
-          -0.3718109726905823,
-          -0.2700554132461548,
-          -1.1014580726623535,
-          0.9154956936836244,
-          -0.3406888246536255,
-          1.0077725648880005,
-          0.6577560901641846,
-          -0.3955195546150207,
-          -0.4148270785808563,
-          0.1855088472366333,
-          0.5062315464019775,
-          -0.3632686734199524,
-          -0.2277890294790268,
-          0.2560805082321167,
-          -0.3853609561920166,
-          -0.1604762226343155,
-          -0.13947471976280212,
-          -0.20147813856601715,
-          -0.4466346800327301,
-          -0.3761846721172333,
-          0.1443382054567337,
-          0.18205296993255615,
-          0.49359792470932007,
-          -0.22538000345230105,
-          -0.4996317625045776,
-          -0.22734887897968292,
-          -0.6034309267997742,
-          -0.7857939600944519,
-          -0.34923747181892395,
-          -0.3466345965862274,
-          0.21176661550998688,
-          -0.5101462006568909,
-          -0.3403083384037018,
-          0.000315118464641273,
-          0.236465722322464,
-          -0.10246097296476364,
-          -1.3013339042663574,
-          0.3419138789176941,
-          -0.32963496446609497,
-          -0.0901619717478752,
-          -0.5426247119903564,
-          0.22656650841236117,
-          -0.44758284091949463,
-          0.14151698350906372,
-          -0.1089438870549202,
-          0.5500766634941101,
-          -0.670711100101471,
-          -0.6227269768714905,
-          0.3894464075565338,
-          -0.27609574794769287,
-          0.7028202414512634,
-          -0.19697771966457367,
-          0.328511506319046,
-          0.5063360929489136,
-          0.4065195322036743,
-          0.2614171802997589,
-          -0.30274391174316406,
-          1.0393824577331543,
-          -0.7742937207221985,
-          -0.7874112129211426,
-          -0.6749666929244995,
-          0.5190866589546204,
-          0.004123548045754433,
-          -0.28312963247299194,
-          -0.038731709122657776,
-          -1.0142987966537476,
-          -0.09519586712121964,
-          0.8755272626876831,
-          0.4876938760280609,
-          0.7811151742935181,
-          0.85174959897995,
-          0.11826585978269576,
-          0.5373436808586121,
-          0.3649002015590668,
-          0.19064077734947205,
-          -0.00287026260048151,
-          -0.7305403351783752,
-          -0.015206154435873032,
-          -0.7899249196052551,
-          0.19407285749912265,
-          0.08596625179052353,
-          -0.28976231813430786,
-          -0.1525907665491104,
-          0.3798313438892365,
-          0.050306469202041626,
-          -0.5697937607765198,
-          0.4219021201133728,
-          0.276252806186676,
-          0.1559903472661972,
-          0.10030482709407806,
-          -0.4043720066547394,
-          -0.1969818025827408,
-          0.5739826560020447,
-          0.2116064727306366,
-          -1.4620544910430908,
-          -0.7802462577819824,
-          -0.24739810824394223,
-          -0.09791352599859238,
-          -0.4413802027702331,
-          0.21549351513385773,
-          -0.9520436525344848,
-          -0.08762510865926743,
-          0.08154498040676117,
-          -0.6154940724372864,
-          -1.01079523563385,
-          0.885427713394165,
-          0.6967288851737976,
-          0.27186504006385803,
-          -0.43194177746772766,
-          -0.11248451471328735,
-          0.7576630711555481,
-          0.4998855590820313,
-          0.0264343973249197,
-          0.9872855544090272,
-          0.5634694695472717,
-          0.053698331117630005,
-          0.19410227239131927,
-          0.3570743501186371,
-          -0.23670297861099243,
-          -0.9114483594894408,
-          0.07884842902421951,
-          0.7318344116210938,
-          0.44630110263824463,
-          0.08745364099740982,
-          -0.347101628780365,
-          -0.4314247667789459,
-          -0.5060274004936218,
-          0.003706763498485088,
-          0.44320008158683777,
-          -0.00788921769708395,
-          -0.1368623524904251,
-          -0.17391923069953918,
-          0.14473655819892883,
-          0.10927865654230118,
-          0.6974599361419678,
-          0.005052129738032818,
-          -0.016953065991401672,
-          -0.1256176233291626,
-          -0.036742497235536575,
-          0.5591985583305359,
-          -0.37619709968566895,
-          0.22429119050502777,
-          0.5403043031692505,
-          -0.8603790998458862,
-          -0.3456307053565979,
-          0.9292937517166138,
-          0.5074859261512756,
-          0.6310645937919617,
-          -0.3091641068458557,
-          0.46902573108673096,
-          0.7891915440559387,
-          0.4499550759792328,
-          0.2744995653629303,
-          0.2712305784225464,
-          -0.04349074140191078,
-          -0.3638863265514374,
-          0.7839881777763367,
-          0.7352104783058167,
-          -0.19457511603832245,
-          -0.5957832932472229,
-          -0.43704694509506226,
-          -1.084769368171692,
-          0.4904985725879669,
-          0.5385226011276245,
-          0.1891629993915558,
-          0.12338479608297348,
-          0.8315675258636475,
-          -0.07830192148685455,
-          1.0916285514831543,
-          -0.28066861629486084,
-          -1.3585069179534912,
-          0.5203898549079895,
-          0.08678033947944641,
-          -0.2566044330596924,
-          0.09484415501356123,
-          -0.0180208683013916,
-          1.0264745950698853,
-          -0.023572135716676712,
-          0.5864979028701782,
-          0.7625196576118469,
-          -0.2543414533138275,
-          -0.8877770900726318,
-          0.7611982822418213,
-          -0.06220436468720436,
-          0.937336564064026,
-          0.2704363465309143,
-          -0.37733694911003113,
-          0.5076137781143188,
-          -0.30641937255859375,
-          0.6252772808074951,
-          -0.0823579877614975,
-          -0.03736555948853493,
-          0.4131673276424408,
-          -0.6514252424240112,
-          0.12918265163898468,
-          -0.4483584463596344,
-          0.6750786304473877,
-          -0.37008383870124817,
-          -0.02324833907186985,
-          0.38027650117874146,
-          -0.26374951004981995,
-          0.4346931278705597,
-          0.42882832884788513,
-          -0.48798441886901855,
-          1.1882442235946655,
-          0.5132288336753845,
-          0.5284568667411804,
-          -0.03538886830210686,
-          0.29620853066444397,
-          -1.0683696269989014,
-          0.25936177372932434,
-          0.10404160618782043,
-          -0.25796034932136536,
-          0.027896970510482788,
-          -0.09225251525640488,
-          1.4811025857925415,
-          0.641173779964447,
-          -0.13838383555412292,
-          -0.3437179923057556,
-          0.5667019486427307,
-          -0.5400741696357727,
-          0.31090837717056274,
-          0.6470608115196228,
-          -0.3747067153453827,
-          -0.7364534735679626,
-          -0.07431528717279434,
-          0.5173454880714417,
-          -0.6578747034072876,
-          0.7107478976249695,
-          -0.7918999791145325,
-          -0.0648345872759819,
-          0.609937846660614,
-          -0.7329513430595398,
-          0.9741371870040894,
-          0.17912346124649048,
-          -0.02658769302070141,
-          0.5162150859832764,
-          -0.3978803157806397,
-          -0.7833885550498962,
-          -0.6497276425361633,
-          -0.3898126780986786,
-          -0.0952848568558693,
-          0.2663288116455078,
-          -0.1604052186012268,
-          0.373076468706131,
-          -0.8357769250869751,
-          -0.05217683315277099,
-          -0.2680160701274872,
-          0.8389158248901367,
-          0.6833611130714417,
-          -0.6712407469749451,
-          0.7406917214393616,
-          -0.44522786140441895,
-          -0.34645363688468933,
-          -0.27384576201438904,
-          -0.9878405928611756,
-          -0.8166060447692871,
-          0.06268279999494553,
-          0.38567957282066345,
-          -0.3274703919887543,
-          0.5296315550804138,
-          -0.11810623109340668,
-          0.23029841482639313,
-          0.08616159111261368,
-          -0.2195747196674347,
-          0.09430307894945145,
-          0.4057176411151886,
-          0.4892159104347229,
-          -0.1636916548013687,
-          -0.6071445345878601,
-          0.41256585717201233,
-          0.622254490852356,
-          -0.41223976016044617,
-          -0.6686707139015198,
-          -0.7474371790885925,
-          -0.8509522080421448,
-          -0.16754287481307983,
-          -0.9078601002693176,
-          -0.29653599858283997,
-          -0.5020652413368225,
-          0.4692700505256653,
-          0.01281109917908907,
-          -0.16071580350399017,
-          0.03388889133930206,
-          -0.020511148497462273,
-          0.5027827024459839,
-          -0.20729811489582065,
-          0.48107290267944336,
-          0.33669769763946533,
-          -0.5275911688804626,
-          0.48271527886390686,
-          0.2738940715789795,
-          -0.033152539283037186,
-          -0.13629786670207977,
-          -0.05965912342071533,
-          -0.26200807094573975,
-          0.04002794995903969,
-          -0.34095603227615356,
-          -3.986898899078369,
-          -0.46819332242012024,
-          -0.422744482755661,
-          -0.169097900390625,
-          0.6008929014205933,
-          0.058016058057546616,
-          -0.11401277780532836,
-          -0.3077819049358368,
-          -0.09595538675785063,
-          0.6723822355270386,
-          0.19367831945419312,
-          0.28304359316825867,
-          0.1609862744808197,
-          0.7567598819732666,
-          0.6889985799789429,
-          0.06907720118761063,
-          -0.04188092052936554,
-          -0.7434936165809631,
-          0.13321782648563385,
-          0.8456063270568848,
-          -0.10364038497209548,
-          -0.45084846019744873,
-          -0.4758241474628449,
-          0.43882066011428833,
-          -0.6432598829269409,
-          0.7217311859130859,
-          -0.24189773201942444,
-          0.12737572193145752,
-          -1.1008601188659668,
-          -0.3305315673351288,
-          0.14614742994308472,
-          -0.7819333076477051,
-          0.5287120342254639,
-          -0.055538054555654526,
-          0.1877404749393463,
-          -0.6907662153244019,
-          0.5616975426673889,
-          -0.4611121714115143,
-          -0.26109233498573303,
-          -0.12898315489292145,
-          -0.3724522292613983,
-          -0.7191406488418579,
-          -0.4425233602523804,
-          -0.644108235836029,
-          0.8424481153488159,
-          0.17532426118850708,
-          -0.5121750235557556,
-          -0.6467239260673523,
-          -0.0008507720194756985,
-          0.7866212129592896,
-          -0.02644744887948036,
-          -0.005045140627771616,
-          0.015782782807946205,
-          0.16334445774555206,
-          -0.1913367658853531,
-          -0.13697923719882965,
-          -0.6684983372688293,
-          0.18346354365348816,
-          -0.341105580329895,
-          0.5427411198616028,
-          0.3779832422733307,
-          -0.6778115034103394,
-          -0.2931850254535675,
-          -0.8805161714553833,
-          -0.4212774932384491,
-          -0.5368952751159668,
-          -1.3937891721725464,
-          -1.225494146347046,
-          0.4276703894138336,
-          1.1205668449401855,
-          -0.6005299687385559,
-          0.15732505917549133,
-          -0.3914784789085388,
-          -1.357046604156494,
-          -0.4707142114639282,
-          -0.1497287154197693,
-          -0.25035548210144043,
-          -0.34328439831733704,
-          0.39083412289619446,
-          0.1623048633337021,
-          -0.9275814294815063,
-          -0.6430015563964844,
-          0.2973862886428833,
-          0.5580436587333679,
-          -0.6232585310935974,
-          -0.6611042022705078,
-          0.4015969038009643,
-          -1.0232892036437988,
-          -0.2585645020008087,
-          -0.5431421399116516,
-          0.5021264553070068,
-          -0.48601630330085754,
-          -0.010242084041237833,
-          0.5862035155296326,
-          0.7316920161247253,
-          0.4036808013916016,
-          0.4269520044326782,
-          -0.705938458442688,
-          0.7747307419776917,
-          0.10164368897676468,
-          0.7887958884239197,
-          -0.9612497091293336,
-          0.12755516171455383,
-          0.06812842190265656,
-          -0.022603651508688927,
-          0.14722754061222076,
-          -0.5588505268096924,
-          -0.20689940452575684,
-          0.3557641804218292,
-          -0.6812759637832642,
-          0.2860803008079529,
-          -0.38954633474349976,
-          0.1759403496980667,
-          -0.5678874850273132,
-          -0.1692986786365509,
-          -0.14578519761562347,
-          0.5711379051208496,
-          1.0208125114440918,
-          0.7759483456611633,
-          -0.372348427772522,
-          -0.5460885763168335,
-          0.7190321683883667,
-          -0.6914990544319153,
-          0.13365162909030914,
-          -0.4854792356491089,
-          0.4054908752441406,
-          0.4502798914909363,
-          -0.3041122555732727,
-          -0.06726965308189392,
-          -0.05570871382951737,
-          -0.0455719493329525,
-          0.4785125255584717,
-          0.8867972493171692,
-          0.4107886850833893,
-          0.6121342182159424,
-          -0.20477132499217987,
-          -0.5598517656326294,
-          -0.6443566679954529,
-          -0.5905212759971619,
-          -0.5571200251579285,
-          0.17573799192905426,
-          -0.28621870279312134,
-          0.1685224026441574,
-          0.09719007462263109,
-          -0.04223639518022537,
-          -0.28623101115226746,
-          -0.1449810117483139,
-          -0.3789580464363098,
-          -0.5227636098861694,
-          -0.049728814512491226,
-          0.7849089503288269,
-          0.16792525351047516,
-          0.9849340915679932,
-          -0.6559549570083618,
-          0.35723909735679626,
-          -0.6822739243507385,
-          1.2873116731643677,
-          0.19993330538272855,
-          0.03512010723352432,
-          -0.6972134113311768,
-          0.18453484773635864,
-          -0.2437680810689926,
-          0.2156416028738022,
-          0.5230382680892944,
-          0.22020135819911957,
-          0.8314080238342285,
-          0.15627102553844452,
-          -0.7330264449119568,
-          0.3888184726238251,
-          -0.22034703195095065,
-          0.5457669496536255,
-          -0.48084837198257446,
-          -0.45576658844947815,
-          -0.09287727624177931,
-          -0.06968110054731369,
-          0.35125672817230225,
-          -0.4278119504451752,
-          0.2038476765155792,
-          0.11392722278833388,
-          0.9433983564376832,
-          -0.4097744226455689,
-          0.035297419875860214,
-          -0.4274404048919678,
-          -0.25100165605545044,
-          1.0943366289138794,
-          -0.07634022831916809,
-          -0.2925529479980469,
-          -0.7512530088424683,
-          0.2649727463722229,
-          -0.4078235328197479,
-          -0.3372223973274231,
-          0.05190162733197212,
-          0.005654910113662481,
-          -0.0001571219472680241,
-          -0.35445958375930786,
-          -0.7837416529655457,
-          0.1500556766986847,
-          0.4383024573326111,
-          0.6099548935890198,
-          0.05951934307813645,
-          -0.21325334906578064,
-          0.0199207104742527,
-          -0.22704418003559113,
-          -0.6481077671051025,
-          0.37442275881767273,
-          -1.015955924987793,
-          0.38637226819992065,
-          -0.06489371508359909,
-          -0.494120329618454,
-          0.3469836115837097,
-          0.15402406454086304,
-          -0.7660972476005554,
-          -0.7053225040435791,
-          -0.25964751839637756,
-          0.014004424214363098,
-          -0.2860170006752014,
-          -0.17565494775772095,
-          -0.45117494463920593,
-          -0.0031954257283359766,
-          0.09676837921142578,
-          -0.514464259147644,
-          0.41698193550109863,
-          -0.21642713248729703,
-          -0.5398141145706177,
-          -0.3647628426551819,
-          0.37005379796028137,
-          0.239425927400589,
-          -0.08833975344896317,
-          0.934946596622467,
-          -0.48340797424316406,
-          0.6241437792778015,
-          -0.7253676652908325,
-          -0.04303571209311485,
-          1.1125205755233765,
-          -0.15692919492721558,
-          -0.2914651036262512,
-          -0.5117168426513672,
-          0.21365483105182648,
-          0.4924402534961701,
-          0.5269662141799927,
-          0.0352792888879776,
-          -0.149167999625206,
-          -0.6019760370254517,
-          0.08245442807674408,
-          0.4900692105293274,
-          0.518824577331543,
-          -0.00005570516441366635,
-          -0.553304135799408,
-          0.22217543423175812,
-          0.5047767758369446,
-          0.135724738240242,
-          1.1511540412902832,
-          -0.3541218340396881,
-          -0.9712511897087096,
-          0.8353699445724487,
-          -0.39227569103240967,
-          -0.9117669463157654,
-          -0.26349931955337524,
-          0.05597023293375969,
-          0.20695461332798004,
-          0.3178807199001312,
-          1.0663238763809204,
-          0.5062212347984314,
-          0.7288597822189331,
-          0.09899299591779707,
-          0.553720235824585,
-          0.675009548664093,
-          -0.20067055523395536,
-          0.3138423264026642,
-          -0.6886593103408813,
-          -0.2910398542881012,
-          -1.3186300992965698,
-          -0.4684459865093231,
-          -0.095743365585804,
-          -0.1257995069026947,
-          -0.4858281314373016,
-          -0.4935407340526581,
-          -0.3266896903514862,
-          -0.3928797245025635,
-          -0.40803104639053345,
-          -0.9975396394729614,
-          0.4229583740234375,
-          0.37309643626213074,
-          0.4431034922599793,
-          0.30364808440208435,
-          -0.3765178918838501,
-          0.5616499185562134,
-          0.16904796659946442,
-          -0.7343707084655762,
-          0.2560209631919861,
-          0.6166825294494629,
-          0.3200829327106476,
-          -0.4483652710914612,
-          0.16224201023578644,
-          -0.31495288014411926,
-          -0.42713335156440735,
-          0.7270734906196594,
-          0.7049484848976135,
-          -0.0571461021900177,
-          0.04477125033736229,
-          -0.6647796034812927,
-          1.183672308921814,
-          0.36199676990509033,
-          0.046881116926670074,
-          0.4515796303749085,
-          0.9278061985969543,
-          0.31471705436706543,
-          -0.7073333859443665,
-          -0.3443860113620758,
-          0.5440067052841187,
-          -0.15020819008350372,
-          -0.541202962398529,
-          0.5203295946121216,
-          1.2192286252975464,
-          -0.9983593225479126,
-          -0.18758884072303772,
-          0.2758221924304962,
-          -0.6511523723602295,
-          -0.1584404855966568,
-          -0.236241415143013,
-          0.2692437767982483,
-          -0.4941152036190033,
-          0.4987454116344452,
-          -0.3331359028816223,
-          0.3163745701313019,
-          0.745529294013977,
-          -0.2905873656272888,
-          0.13602906465530396,
-          0.4679684340953827,
-          1.0555986166000366,
-          1.075700044631958,
-          0.5368486046791077,
-          -0.5118206739425659,
-          0.8668332099914551,
-          -0.5726966857910156,
-          -0.7811751961708069,
-          0.1938626915216446,
-          -0.1929349899291992,
-          0.1757766306400299,
-          0.6384295225143433,
-          0.26462844014167786,
-          0.9542630314826964,
-          0.19313029944896695,
-          1.264248013496399,
-          -0.6304428577423096,
-          0.0487106591463089,
-          -0.16211535036563873,
-          -0.7894763350486755,
-          0.3582514822483063,
-          -0.04153040423989296,
-          0.635784387588501,
-          0.6554391980171204,
-          -0.47010496258735657,
-          -0.8302040696144104,
-          -0.1350124627351761,
-          0.2568812072277069,
-          0.13614831864833832,
-          -0.2563649117946625,
-          -1.0434694290161133,
-          0.3232482671737671,
-          0.47882452607154846,
-          0.4298652410507202,
-          1.0563770532608032,
-          -0.28917592763900757,
-          -0.8533256649971008,
-          0.10648339986801147,
-          0.6376127004623413,
-          -0.20832888782024384,
-          0.2370245456695557,
-          0.0018312990432605147,
-          -0.2034837007522583,
-          0.01051164511591196,
-          -1.105310082435608,
-          0.29724350571632385,
-          0.15604574978351593,
-          0.1973688006401062,
-          0.44394731521606445,
-          0.3974513411521912,
-          -0.13625948131084442,
-          0.9571986198425292,
-          0.2257384955883026,
-          0.2323588728904724,
-          -0.5583669543266296,
-          -0.7854922413825989,
-          0.1647188365459442,
-          -1.6098142862319946,
-          0.318587988615036,
-          -0.13399995863437653,
-          -0.2172701060771942,
-          -0.767514705657959,
-          -0.5813586711883545,
-          -0.3195130527019501,
-          -0.04894036799669266,
-          0.2929930090904236,
-          -0.8213384747505188,
-          0.07181350141763687,
-          0.7469993829727173,
-          0.6407455801963806,
-          0.16365697979927063,
-          0.7870153188705444,
-          0.6524736881256104,
-          0.6399973630905151,
-          -0.04992736503481865,
-          -0.03959266096353531,
-          -0.2512352466583252,
-          0.8448855876922607,
-          -0.1422702670097351,
-          0.1216789186000824,
-          -1.2647287845611572,
-          0.5931149125099182,
-          0.7186052203178406,
-          -0.06118432432413101,
-          -1.1942816972732544,
-          -0.17677085101604462,
-          0.31543800234794617,
-          -0.32252824306488037,
-          0.8255583047866821,
-          -0.14529970288276672,
-          -0.2695446312427521,
-          -0.33378756046295166,
-          -0.1653425395488739,
-          0.1454019844532013,
-          -0.3920115828514099,
-          0.912214994430542,
-          -0.7279734015464783,
-          0.7374742031097412,
-          0.933980405330658,
-          0.13429680466651917,
-          -0.514870285987854,
-          0.3989711999893189,
-          -0.11613689363002776,
-          0.4022413492202759,
-          -0.9990655779838562,
-          -0.33749932050704956,
-          -0.4334589838981629,
-          -1.376373291015625,
-          -0.2993924915790558,
-          -0.09454808384180068,
-          -0.01314175222069025,
-          -0.001090060803107917,
-          0.2137461006641388,
-          0.2938512861728668,
-          0.17508235573768616,
-          0.8260607123374939,
-          -0.7218498587608337,
-          0.2414487451314926,
-          -0.47296759486198425,
-          -0.3002610504627228,
-          -1.238540768623352,
-          0.08663805574178696,
-          0.6805586218833923,
-          0.5909030437469482,
-          -0.42807504534721375,
-          -0.22887496650218964,
-          0.47537800669670105,
-          -1.0474627017974854,
-          0.6338009238243103,
-          0.06548397243022919,
-          0.4971011281013489,
-          1.3484878540039063
-        ]
-      ],
-      "regenerate": true
-    }
-  }
-}

dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-7.snap

@@ -1,785 +0,0 @@
----
-source: dump/src/reader/mod.rs
-expression: document
----
-{
-  "id": "e1",
-  "desc": "natural vector",
-  "_vectors": {
-    "default": {
-      "embeddings": [
-        [
-          -0.2979458272457123,
-          -0.5288640856742859,
-          -0.019957859069108963,
-          -0.18495318293571472,
-          0.7429973483085632,
-          0.5238497257232666,
-          0.432366281747818,
-          0.32744166254997253,
-          0.0020762972999364138,
-          -0.9507834911346436,
-          -0.35097137093544006,
-          0.08469701558351517,
-          -1.4176613092422483,
-          0.4647577106952667,
-          -0.69340580701828,
-          1.0372896194458008,
-          0.3716741800308227,
-          0.06031008064746857,
-          -0.6152024269104004,
-          0.007914665155112743,
-          0.7954924702644348,
-          -0.20773003995418549,
-          0.09376765787601472,
-          0.04508133605122566,
-          -0.2084471583366394,
-          -0.1518009901046753,
-          0.018195509910583496,
-          -0.07044368237257004,
-          -0.18119366466999057,
-          -0.4480230510234833,
-          0.3822529911994934,
-          0.1911812424659729,
-          0.4674372375011444,
-          0.06963984668254852,
-          -0.09341949224472046,
-          0.005675444379448891,
-          -0.6774799227714539,
-          -0.7066726684570313,
-          -0.39256376028060913,
-          0.04005039855837822,
-          0.2084812968969345,
-          -0.7872875928878784,
-          -0.8205880522727966,
-          0.2919981777667999,
-          -0.06004738807678223,
-          -0.4907574355602264,
-          -1.5937862396240234,
-          0.24249385297298431,
-          -0.14709846675395966,
-          -0.11860740929841997,
-          -0.8299489617347717,
-          0.472964346408844,
-          -0.497518390417099,
-          -0.22205302119255063,
-          -0.4196169078350067,
-          0.32697558403015137,
-          -0.360930860042572,
-          -0.9789686799049376,
-          0.1887447088956833,
-          -0.403737336397171,
-          0.18524253368377688,
-          0.3768732249736786,
-          0.3666233420372009,
-          0.3511938452720642,
-          0.6985810995101929,
-          0.41721710562705994,
-          0.09754953533411026,
-          0.6204307079315186,
-          -1.0762996673583984,
-          -0.06263761967420578,
-          -0.7376511693000793,
-          0.6849768161773682,
-          -0.1745152473449707,
-          -0.40449759364128113,
-          0.20757411420345304,
-          -0.8424443006515503,
-          0.330015629529953,
-          0.3489064872264862,
-          1.0954371690750122,
-          0.8487558960914612,
-          1.1076823472976685,
-          0.61430823802948,
-          0.4155903458595276,
-          0.4111340939998626,
-          0.05753209814429283,
-          -0.06429877132177353,
-          -0.765606164932251,
-          -0.41703930497169495,
-          -0.508820652961731,
-          0.19859947264194489,
-          -0.16607828438282013,
-          -0.28112146258354187,
-          0.11032675206661224,
-          0.38809511065483093,
-          -0.36498191952705383,
-          -0.48671194911003113,
-          0.6755134463310242,
-          0.03958442434668541,
-          0.4478721618652344,
-          -0.10335399955511092,
-          -0.9546685814857484,
-          -0.6087718605995178,
-          0.17498846352100372,
-          0.08320838958024979,
-          -1.4478336572647097,
-          -0.605027437210083,
-          -0.5867993235588074,
-          -0.14711688458919525,
-          -0.5447602272033691,
-          -0.026259321719408035,
-          -0.6997418403625488,
-          -0.07349082082509995,
-          0.10638900846242905,
-          -0.7133527398109436,
-          -0.9396815299987792,
-          1.087092399597168,
-          1.1885089874267578,
-          0.4011896848678589,
-          -0.4089202582836151,
-          -0.10938972979784012,
-          0.6726722121238708,
-          0.24576938152313232,
-          -0.24247920513153076,
-          1.1499971151351929,
-          0.47813335061073303,
-          -0.05331678315997124,
-          0.32338133454322815,
-          0.4870913326740265,
-          -0.23144258558750153,
-          -1.2023426294326782,
-          0.2349330335855484,
-          1.080536961555481,
-          0.29334118962287903,
-          0.391574501991272,
-          -0.15818795561790466,
-          -0.2948290705680847,
-          -0.024689948186278343,
-          0.06602869182825089,
-          0.5937030911445618,
-          -0.047901444137096405,
-          -0.512734591960907,
-          -0.35780075192451477,
-          0.28751692175865173,
-          0.4298716187477112,
-          0.9242428541183472,
-          -0.17208744585514069,
-          0.11515070497989656,
-          -0.0335976779460907,
-          -0.3422986567020416,
-          0.5344581604003906,
-          0.19895796477794647,
-          0.33001241087913513,
-          0.6390730142593384,
-          -0.6074934005737305,
-          -0.2553696632385254,
-          0.9644920229911804,
-          0.2699219584465027,
-          0.6403993368148804,
-          -0.6380003690719604,
-          -0.027310986071825027,
-          0.638815701007843,
-          0.27719101309776306,
-          -0.13553589582443237,
-          0.750195324420929,
-          0.1224869191646576,
-          -0.20613941550254825,
-          0.8444448709487915,
-          0.16200250387191772,
-          -0.24750925600528717,
-          -0.739950954914093,
-          -0.28443849086761475,
-          -1.176282525062561,
-          0.516107976436615,
-          0.3774825632572174,
-          0.10906043648719788,
-          0.07962015271186829,
-          0.7384604215621948,
-          -0.051241904497146606,
-          1.1730090379714966,
-          -0.4828610122203827,
-          -1.404372215270996,
-          0.8811132311820984,
-          -0.3839482367038727,
-          0.022516896948218346,
-          -0.0491158664226532,
-          -0.43027013540267944,
-          1.2049334049224854,
-          -0.27309560775756836,
-          0.6883630752563477,
-          0.8264574408531189,
-          -0.5020735263824463,
-          -0.4874092042446137,
-          0.6007202863693237,
-          -0.4965405762195587,
-          1.1302915811538696,
-          0.032572727650403976,
-          -0.3731859028339386,
-          0.658271849155426,
-          -0.9023059010505676,
-          0.7400162220001221,
-          0.014550759457051754,
-          -0.19699542224407196,
-          0.2319706380367279,
-          -0.789058268070221,
-          -0.14905710518360138,
-          -0.5826214551925659,
-          0.207652747631073,
-          -0.4507439732551574,
-          -0.3163885474205017,
-          0.3604124188423157,
-          -0.45119962096214294,
-          0.3428427278995514,
-          0.3005594313144684,
-          -0.36026081442832947,
-          1.1014249324798584,
-          0.40884315967559814,
-          0.34991952776908875,
-          -0.1806638240814209,
-          0.27440476417541504,
-          -0.7118373513221741,
-          0.4645499587059021,
-          0.214790478348732,
-          -0.2343102991580963,
-          0.10500429570674896,
-          -0.28034430742263794,
-          1.2267805337905884,
-          1.0561333894729614,
-          -0.497364342212677,
-          -0.6143305897712708,
-          0.24963727593421936,
-          -0.33136463165283203,
-          -0.01473914459347725,
-          0.495918869972229,
-          -0.6985538005828857,
-          -1.0033197402954102,
-          0.35937801003456116,
-          0.6325868368148804,
-          -0.6808838844299316,
-          1.0354058742523191,
-          -0.7214401960372925,
-          -0.33318862318992615,
-          0.874398410320282,
-          -0.6594992280006409,
-          0.6830640435218811,
-          -0.18534131348133087,
-          0.024834271520376205,
-          0.19901277124881744,
-          -0.5992477536201477,
-          -1.2126628160476685,
-          -0.9245557188987732,
-          -0.3898217976093292,
-          -0.1286519467830658,
-          0.4217943847179413,
-          -0.1143646091222763,
-          0.5630772709846497,
-          -0.5240639448165894,
-          0.21152715384960177,
-          -0.3792001008987427,
-          0.8266305327415466,
-          1.170984387397766,
-          -0.8072142004966736,
-          0.11382893472909927,
-          -0.17953898012638092,
-          -0.1789460331201553,
-          -0.15078622102737427,
-          -1.2082908153533936,
-          -0.7812382578849792,
-          -0.10903695970773696,
-          0.7303897142410278,
-          -0.39054441452026367,
-          0.19511254131793976,
-          -0.09121843427419662,
-          0.22400228679180145,
-          0.30143046379089355,
-          0.1141919493675232,
-          0.48112115263938904,
-          0.7307931780815125,
-          0.09701362252235413,
-          -0.2795647978782654,
-          -0.3997688889503479,
-          0.5540812611579895,
-          0.564578115940094,
-          -0.40065160393714905,
-          -0.3629159033298493,
-          -0.3789091110229492,
-          -0.7298538088798523,
-          -0.6996853351593018,
-          -0.4477842152118683,
-          -0.289089560508728,
-          -0.6430277824401855,
-          0.2344944179058075,
-          0.3742927014827728,
-          -0.5079357028007507,
-          0.28841453790664673,
-          0.06515737622976303,
-          0.707315981388092,
-          0.09498685598373412,
-          0.8365515470504761,
-          0.10002726316452026,
-          -0.7695478200912476,
-          0.6264724135398865,
-          0.7562043070793152,
-          -0.23112858831882477,
-          -0.2871039807796478,
-          -0.25010058283805847,
-          0.2783474028110504,
-          -0.03224996477365494,
-          -0.9119359850883484,
-          -3.6940200328826904,
-          -0.5099936127662659,
-          -0.1604711413383484,
-          0.17453284561634064,
-          0.41759559512138367,
-          0.1419190913438797,
-          -0.11362407356500626,
-          -0.33312007784843445,
-          0.11511333286762238,
-          0.4667884409427643,
-          -0.0031647447030991316,
-          0.15879854559898376,
-          0.3042248487472534,
-          0.5404849052429199,
-          0.8515422344207764,
-          0.06286454200744629,
-          0.43790125846862793,
-          -0.8682025074958801,
-          -0.06363756954669952,
-          0.5547921657562256,
-          -0.01483887154608965,
-          -0.07361344993114471,
-          -0.929947018623352,
-          0.3502565622329712,
-          -0.5080993175506592,
-          1.0380364656448364,
-          -0.2017953395843506,
-          0.21319580078125,
-          -1.0763001441955566,
-          -0.556368887424469,
-          0.1949922740459442,
-          -0.6445739269256592,
-          0.6791343688964844,
-          0.21188358962535855,
-          0.3736183941364288,
-          -0.21800459921360016,
-          0.7597446441650391,
-          -0.3732394874095917,
-          -0.4710160195827484,
-          0.025146087631583217,
-          0.05341297015547752,
-          -0.9522109627723694,
-          -0.6000866889953613,
-          -0.08469046652317047,
-          0.5966026186943054,
-          0.3444081246852875,
-          -0.461188405752182,
-          -0.5279349088668823,
-          0.10296865552663804,
-          0.5175143480300903,
-          -0.20671147108078003,
-          0.13392412662506104,
-          0.4812754988670349,
-          0.2993808686733246,
-          -0.3005635440349579,
-          0.5141698122024536,
-          -0.6239235401153564,
-          0.2877119481563568,
-          -0.4452739953994751,
-          0.5621107816696167,
-          0.5047508478164673,
-          -0.4226335883140564,
-          -0.18578553199768064,
-          -1.1967322826385498,
-          0.28178197145462036,
-          -0.8692031502723694,
-          -1.1812998056411743,
-          -1.4526212215423584,
-          0.4645712077617645,
-          0.9327932000160216,
-          -0.6560136675834656,
-          0.461549699306488,
-          -0.5621527433395386,
-          -1.328449010848999,
-          -0.08676894754171371,
-          0.00021918353741057217,
-          -0.18864136934280396,
-          0.1259666532278061,
-          0.18240638077259064,
-          -0.14919660985469818,
-          -0.8965857625007629,
-          -0.7539900541305542,
-          0.013973715715110302,
-          0.504276692867279,
-          -0.704748272895813,
-          -0.6428424119949341,
-          0.6303996443748474,
-          -0.5404738187789917,
-          -0.31176653504371643,
-          -0.21262824535369873,
-          0.18736739456653595,
-          -0.7998970746994019,
-          0.039946746081113815,
-          0.7390344738960266,
-          0.4283199906349182,
-          0.3795057237148285,
-          0.07204607129096985,
-          -0.9230587482452391,
-          0.9440426230430604,
-          0.26272690296173096,
-          0.5598306655883789,
-          -1.0520871877670288,
-          -0.2677186131477356,
-          -0.1888762265443802,
-          0.30426350235939026,
-          0.4746131896972656,
-          -0.5746733546257019,
-          -0.4197768568992615,
-          0.8565112948417664,
-          -0.6767723560333252,
-          0.23448683321475983,
-          -0.2010004222393036,
-          0.4112907350063324,
-          -0.6497949957847595,
-          -0.418667733669281,
-          -0.4950824975967407,
-          0.44438859820365906,
-          1.026281714439392,
-          0.482397586107254,
-          -0.26220494508743286,
-          -0.3640787005424499,
-          0.5907743573188782,
-          -0.8771642446517944,
-          0.09708411991596222,
-          -0.3671700060367584,
-          0.4331349730491638,
-          0.619417667388916,
-          -0.2684665620326996,
-          -0.5123821496963501,
-          -0.1502324342727661,
-          -0.012190685607492924,
-          0.3580845892429352,
-          0.8617186546325684,
-          0.3493645489215851,
-          1.0270192623138428,
-          0.18297909200191495,
-          -0.5881339311599731,
-          -0.1733516901731491,
-          -0.5040576457977295,
-          -0.340370237827301,
-          -0.26767754554748535,
-          -0.28570041060447693,
-          -0.032928116619586945,
-          0.6029254794120789,
-          0.17397655546665192,
-          0.09346921741962431,
-          0.27815181016921997,
-          -0.46699589490890503,
-          -0.8148876428604126,
-          -0.3964351713657379,
-          0.3812595009803772,
-          0.13547226786613464,
-          0.7126688361167908,
-          -0.3473474085330963,
-          -0.06573959439992905,
-          -0.6483767032623291,
-          1.4808889627456665,
-          0.30924928188323975,
-          -0.5085946917533875,
-          -0.8613000512123108,
-          0.3048902451992035,
-          -0.4241599142551422,
-          0.15909206867218018,
-          0.5764641761779785,
-          -0.07879110425710678,
-          1.015336513519287,
-          0.07599356025457382,
-          -0.7025855779647827,
-          0.30047643184661865,
-          -0.35094937682151794,
-          0.2522146999835968,
-          -0.2338722199201584,
-          -0.8326804637908936,
-          -0.13695412874221802,
-          -0.03452421352267265,
-          0.47974953055381775,
-          -0.18385636806488037,
-          0.32438594102859497,
-          0.1797013282775879,
-          0.787494957447052,
-          -0.12579888105392456,
-          -0.07507286965847015,
-          -0.4389670491218567,
-          0.2720070779323578,
-          0.8138866424560547,
-          0.01974171027541161,
-          -0.3057698905467987,
-          -0.6709924936294556,
-          0.0885881632566452,
-          -0.2862754464149475,
-          0.03475658595561981,
-          -0.1285519152879715,
-          0.3838353455066681,
-          -0.2944154739379883,
-          -0.4204859137535095,
-          -0.4416137933731079,
-          0.13426260650157928,
-          0.36733248829841614,
-          0.573428750038147,
-          -0.14928072690963745,
-          -0.026076916605234143,
-          0.33286052942276,
-          -0.5340145826339722,
-          -0.17279052734375,
-          -0.01154550164937973,
-          -0.6620771884918213,
-          0.18390542268753052,
-          -0.08265615254640579,
-          -0.2489682286977768,
-          0.2429984211921692,
-          -0.044153645634651184,
-          -0.986578404903412,
-          -0.33574509620666504,
-          -0.5387663841247559,
-          0.19767941534519196,
-          0.12540718913078308,
-          -0.3403128981590271,
-          -0.4154576361179352,
-          0.17275673151016235,
-          0.09407442808151244,
-          -0.5414086580276489,
-          0.4393929839134216,
-          0.1725579798221588,
-          -0.4998118281364441,
-          -0.6926208138465881,
-          0.16552448272705078,
-          0.6659538149833679,
-          -0.10949844866991044,
-          0.986426830291748,
-          0.01748848147690296,
-          0.4003709554672241,
-          -0.5430638194084167,
-          0.35347291827201843,
-          0.6887399554252625,
-          0.08274628221988678,
-          0.13407137989997864,
-          -0.591465950012207,
-          0.3446292281150818,
-          0.6069018244743347,
-          0.1935492902994156,
-          -0.0989871397614479,
-          0.07008486241102219,
-          -0.8503749370574951,
-          -0.09507356584072112,
-          0.6259510517120361,
-          0.13934025168418884,
-          0.06392545253038406,
-          -0.4112265408039093,
-          -0.08475656062364578,
-          0.4974113404750824,
-          -0.30606114864349365,
-          1.111435890197754,
-          -0.018766529858112335,
-          -0.8422622680664063,
-          0.4325508773326874,
-          -0.2832120656967163,
-          -0.4859798848628998,
-          -0.41498348116874695,
-          0.015977520495653152,
-          0.5292825698852539,
-          0.4538311660289765,
-          1.1328668594360352,
-          0.22632671892642975,
-          0.7918671369552612,
-          0.33401933312416077,
-          0.7306135296821594,
-          0.3548600673675537,
-          0.12506209313869476,
-          0.8573207855224609,
-          -0.5818327069282532,
-          -0.6953738927841187,
-          -1.6171947717666626,
-          -0.1699674427509308,
-          0.6318262815475464,
-          -0.05671752244234085,
-          -0.28145185112953186,
-          -0.3976689279079437,
-          -0.2041076272726059,
-          -0.5495951175689697,
-          -0.5152917504310608,
-          -0.9309796094894408,
-          0.101932130753994,
-          0.1367802917957306,
-          0.1490798443555832,
-          0.5304336547851563,
-          -0.5082434415817261,
-          0.06688683480024338,
-          0.14657628536224365,
-          -0.782435953617096,
-          0.2962816655635834,
-          0.6965363621711731,
-          0.8496337532997131,
-          -0.3042965829372406,
-          0.04343798756599426,
-          0.0330701619386673,
-          -0.5662598013877869,
-          1.1086925268173218,
-          0.756072998046875,
-          -0.204134538769722,
-          0.2404300570487976,
-          -0.47848284244537354,
-          1.3659011125564575,
-          0.5645433068275452,
-          -0.15836156904697418,
-          0.43395575881004333,
-          0.5944653749465942,
-          1.0043466091156006,
-          -0.49446743726730347,
-          -0.5954391360282898,
-          0.5341240763664246,
-          0.020598189905285835,
-          -0.4036853015422821,
-          0.4473709762096405,
-          1.1998231410980225,
-          -0.9317775368690492,
-          -0.23321466147899628,
-          0.2052552700042725,
-          -0.7423108816146851,
-          -0.19917210936546328,
-          -0.1722569614648819,
-          -0.034072667360305786,
-          -0.00671181408688426,
-          0.46396249532699585,
-          -0.1372445821762085,
-          0.053376372903585434,
-          0.7392690777778625,
-          -0.38447609543800354,
-          0.07497968524694443,
-          0.5197252631187439,
-          1.3746477365493774,
-          0.9060075879096984,
-          0.20000585913658145,
-          -0.4053704142570496,
-          0.7497360110282898,
-          -0.34087055921554565,
-          -1.101803183555603,
-          0.273650586605072,
-          -0.5125769376754761,
-          0.22472351789474487,
-          0.480757474899292,
-          -0.19845178723335263,
-          0.8857700824737549,
-          0.30752456188201904,
-          1.1109285354614258,
-          -0.6768012642860413,
-          0.524367094039917,
-          -0.22495046257972717,
-          -0.4224412739276886,
-          0.40753406286239624,
-          -0.23133376240730288,
-          0.3297771215438843,
-          0.4905449151992798,
-          -0.6813114285469055,
-          -0.7543983459472656,
-          -0.5599071383476257,
-          0.14351597428321838,
-          -0.029278717935085297,
-          -0.3970443606376648,
-          -0.303079217672348,
-          0.24161772429943085,
-          0.008353390730917454,
-          -0.0062365154735744,
-          1.0824860334396362,
-          -0.3704061508178711,
-          -1.0337258577346802,
-          0.04638749733567238,
-          1.163011074066162,
-          -0.31737643480300903,
-          0.013986887410283089,
-          0.19223114848136905,
-          -0.2260770797729492,
-          -0.210910826921463,
-          -1.0191949605941772,
-          0.22356095910072327,
-          0.09353553503751756,
-          0.18096882104873657,
-          0.14867214858531952,
-          0.43408671021461487,
-          -0.33312076330184937,
-          0.8173948526382446,
-          0.6428242921829224,
-          0.20215003192424777,
-          -0.6634518504142761,
-          -0.4132290482521057,
-          0.29815030097961426,
-          -1.579406976699829,
-          -0.0981958732008934,
-          -0.03941014781594277,
-          0.1709178239107132,
-          -0.5481140613555908,
-          -0.5338194966316223,
-          -0.3528362512588501,
-          -0.11561278253793716,
-          -0.21793591976165771,
-          -1.1570470333099363,
-          0.2157980799674988,
-          0.42083489894866943,
-          0.9639263153076172,
-          0.09747201204299928,
-          0.15671424567699432,
-          0.4034591615200043,
-          0.6728067994117737,
-          -0.5216875672340393,
-          0.09657668322324751,
-          -0.2416689097881317,
-          0.747975766658783,
-          0.1021689772605896,
-          0.11652665585279463,
-          -1.0484966039657593,
-          0.8489304780960083,
-          0.7169828414916992,
-          -0.09012343734502792,
-          -1.3173753023147583,
-          0.057890523225069046,
-          -0.006231260951608419,
-          -0.1018214002251625,
-          0.936040461063385,
-          -0.0502331368625164,
-          -0.4284322261810303,
-          -0.38209280371665955,
-          -0.22668412327766416,
-          0.0782942995429039,
-          -0.4881664514541626,
-          0.9268959760665894,
-          0.001867273123934865,
-          0.42261114716529846,
-          0.8283362984657288,
-          0.4256294071674347,
-          -0.7965338826179504,
-          0.4840078353881836,
-          -0.19861412048339844,
-          0.33977967500686646,
-          -0.4604192078113556,
-          -0.3107339143753052,
-          -0.2839638590812683,
-          -1.5734281539916992,
-          0.005220232997089624,
-          0.09239906817674635,
-          -0.7828494906425476,
-          -0.1397123783826828,
-          0.2576255202293396,
-          0.21372435986995697,
-          -0.23169949650764465,
-          0.4016408920288086,
-          -0.462497353553772,
-          -0.2186472862958908,
-          -0.5617868900299072,
-          -0.3649831712245941,
-          -1.1585862636566162,
-          -0.08222806453704834,
-          0.931126832962036,
-          0.4327389597892761,
-          -0.46451422572135925,
-          -0.5430706143379211,
-          -0.27434298396110535,
-          -0.9479129314422609,
-          0.1845661848783493,
-          0.3972720205783844,
-          0.4883299469947815,
-          1.04031240940094
-        ]
-      ],
-      "regenerate": true
-    }
-  }
-}

dump/src/reader/snapshots/dump__reader__test__import_dump_v6_with_vectors-8.snap

@@ -1,780 +0,0 @@
----
-source: dump/src/reader/mod.rs
-expression: document
----
-{
-  "id": "e0",
-  "desc": "overriden vector",
-  "_vectors": {
-    "default": [
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1,
-      0.1
-    ]
-  }
-}

dump/src/reader/v1/settings.rs

@@ -56,7 +56,8 @@ pub enum RankingRule {
     Desc(String),
 }
 
-static ASC_DESC_REGEX: Lazy<Regex> = Lazy::new(|| Regex::new(r"(asc|desc)\(([\w_-]+)\)").unwrap());
+static ASC_DESC_REGEX: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r#"(asc|desc)\(([\w_-]+)\)"#).unwrap());
 
 impl FromStr for RankingRule {
     type Err = ();

dump/src/reader/v2/mod.rs

@@ -46,7 +46,6 @@ pub type Checked = settings::Checked;
 pub type Unchecked = settings::Unchecked;
 
 pub type Task = updates::UpdateEntry;
-pub type Kind = updates::UpdateMeta;
 
 // everything related to the errors
 pub type ResponseError = errors::ResponseError;
@@ -108,11 +107,8 @@ impl V2Reader {
     pub fn indexes(&self) -> Result<impl Iterator<Item = Result<V2IndexReader>> + '_> {
         Ok(self.index_uuid.iter().map(|index| -> Result<_> {
             V2IndexReader::new(
+                index.uid.clone(),
                 &self.dump.path().join("indexes").join(format!("index-{}", index.uuid)),
-                index,
-                BufReader::new(
-                    File::open(self.dump.path().join("updates").join("data.jsonl")).unwrap(),
-                ),
             )
         }))
     }
@@ -147,41 +143,16 @@ pub struct V2IndexReader {
 }
 
 impl V2IndexReader {
-    pub fn new(path: &Path, index_uuid: &IndexUuid, tasks: BufReader<File>) -> Result<Self> {
+    pub fn new(name: String, path: &Path) -> Result<Self> {
         let meta = File::open(path.join("meta.json"))?;
         let meta: DumpMeta = serde_json::from_reader(meta)?;
 
-        let mut created_at = None;
-        let mut updated_at = None;
-
-        for line in tasks.lines() {
-            let task: Task = serde_json::from_str(&line?)?;
-            if !(task.uuid == index_uuid.uuid && task.is_finished()) {
-                continue;
-            }
-
-            let new_created_at = match task.update.meta() {
-                Kind::DocumentsAddition { .. } | Kind::Settings(_) => task.update.finished_at(),
-                _ => None,
-            };
-            let new_updated_at = task.update.finished_at();
-
-            if created_at.is_none() || created_at > new_created_at {
-                created_at = new_created_at;
-            }
-
-            if updated_at.is_none() || updated_at < new_updated_at {
-                updated_at = new_updated_at;
-            }
-        }
-
-        let current_time = OffsetDateTime::now_utc();
-
         let metadata = IndexMetadata {
-            uid: index_uuid.uid.clone(),
+            uid: name,
             primary_key: meta.primary_key,
-            created_at: created_at.unwrap_or(current_time),
-            updated_at: updated_at.unwrap_or(current_time),
+            // FIXME: Iterate over the whole task queue to find the creation and last update date.
+            created_at: OffsetDateTime::now_utc(),
+            updated_at: OffsetDateTime::now_utc(),
         };
 
         let ret = V2IndexReader {
@@ -252,7 +223,7 @@ pub(crate) mod test {
         let mut dump = V2Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
@@ -277,12 +248,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2022-10-09T20:27:22.688964637Z",
-          "updatedAt": "2022-10-09T20:27:23.951017769Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -292,12 +263,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2022-10-09T20:27:22.197788495Z",
-          "updatedAt": "2022-10-09T20:28:01.93111053Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -322,12 +293,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2022-10-09T20:27:24.242683494Z",
-          "updatedAt": "2022-10-09T20:27:24.312809641Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -349,7 +320,7 @@ pub(crate) mod test {
         let mut dump = V2Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
@@ -369,12 +340,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2023-01-30T16:25:56.595257Z",
-          "updatedAt": "2023-01-30T16:25:58.70348Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -384,12 +355,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2023-01-30T16:25:56.192178Z",
-          "updatedAt": "2023-01-30T16:25:56.455714Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -399,12 +370,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"0227598af846e574139ee0b80e03a720");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2023-01-30T16:25:58.876405Z",
-          "updatedAt": "2023-01-30T16:25:59.079906Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 

dump/src/reader/v2/updates.rs

@@ -61,7 +61,7 @@ pub enum IndexDocumentsMethod {
 #[cfg_attr(test, derive(serde::Serialize))]
 #[non_exhaustive]
 pub enum UpdateFormat {
-    /// The given update is a real **comma separated** CSV with headers on the first line.
+    /// The given update is a real **comma seperated** CSV with headers on the first line.
     Csv,
     /// The given update is a JSON array with documents inside.
     Json,
@@ -227,14 +227,4 @@ impl UpdateStatus {
             _ => None,
         }
     }
-
-    pub fn finished_at(&self) -> Option<OffsetDateTime> {
-        match self {
-            UpdateStatus::Processing(_) => None,
-            UpdateStatus::Enqueued(_) => None,
-            UpdateStatus::Processed(u) => Some(u.processed_at),
-            UpdateStatus::Aborted(_) => None,
-            UpdateStatus::Failed(u) => Some(u.failed_at),
-        }
-    }
 }

dump/src/reader/v3/errors.rs

@@ -1,6 +1,5 @@
 use serde::{Deserialize, Serialize};
 
-#[allow(clippy::enum_variant_names)]
 #[derive(Serialize, Deserialize, Debug, Clone, Copy)]
 pub enum Code {
     // index related error

dump/src/reader/v3/mod.rs

@@ -267,7 +267,7 @@ pub(crate) mod test {
         let mut dump = V3Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/v3/settings.rs

@@ -152,7 +152,6 @@ impl Settings<Unchecked> {
 }
 
 #[derive(Debug, Clone, Deserialize)]
-#[allow(dead_code)] // otherwise rustc complains that the fields go unused
 #[cfg_attr(test, derive(serde::Serialize))]
 #[serde(deny_unknown_fields)]
 #[serde(rename_all = "camelCase")]

dump/src/reader/v4/errors.rs

@@ -95,7 +95,6 @@ impl fmt::Display for ErrorType {
     }
 }
 
-#[allow(clippy::enum_variant_names)]
 #[derive(Serialize, Deserialize, Debug, Clone, Copy)]
 pub enum Code {
     // index related error

dump/src/reader/v4/mod.rs

@@ -262,8 +262,8 @@ pub(crate) mod test {
         let mut dump = V4Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
+        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/v4/settings.rs

@@ -182,7 +182,6 @@ impl Settings<Unchecked> {
     }
 }
 
-#[allow(dead_code)] // otherwise rustc complains that the fields go unused
 #[derive(Debug, Clone, Deserialize)]
 #[cfg_attr(test, derive(serde::Serialize))]
 #[serde(deny_unknown_fields)]

dump/src/reader/v5/errors.rs

@@ -31,7 +31,6 @@ impl ResponseError {
     }
 }
 
-#[allow(clippy::enum_variant_names)]
 #[derive(Deserialize, Debug, Clone, Copy)]
 #[cfg_attr(test, derive(serde::Serialize))]
 pub enum Code {

dump/src/reader/v5/mod.rs

@@ -299,8 +299,8 @@ pub(crate) mod test {
         let mut dump = V5Reader::open(dir).unwrap();
 
         // top level infos
-        insta::assert_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
+        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
 
         // tasks
         let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();

dump/src/reader/v5/tasks.rs

@@ -200,7 +200,6 @@ impl std::ops::Deref for IndexUid {
     }
 }
 
-#[allow(dead_code)] // otherwise rustc complains that the fields go unused
 #[derive(Debug)]
 #[cfg_attr(test, derive(serde::Serialize))]
 #[cfg_attr(test, serde(rename_all = "camelCase"))]

dump/src/reader/v6/mod.rs

@@ -5,7 +5,6 @@ use std::path::Path;
 pub use meilisearch_types::milli;
 use tempfile::TempDir;
 use time::OffsetDateTime;
-use tracing::debug;
 use uuid::Uuid;
 
 use super::Document;
@@ -19,7 +18,6 @@ pub type Unchecked = meilisearch_types::settings::Unchecked;
 
 pub type Task = crate::TaskDump;
 pub type Key = meilisearch_types::keys::Key;
-pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures;
 
 // ===== Other types to clarify the code of the compat module
 // everything related to the tasks
@@ -49,7 +47,6 @@ pub struct V6Reader {
     metadata: Metadata,
     tasks: BufReader<File>,
     keys: BufReader<File>,
-    features: Option<RuntimeTogglableFeatures>,
 }
 
 impl V6Reader {
@@ -61,29 +58,11 @@ impl V6Reader {
             Err(e) => return Err(e.into()),
         };
 
-        let feature_file = match fs::read(dump.path().join("experimental-features.json")) {
-            Ok(feature_file) => Some(feature_file),
-            Err(error) => match error.kind() {
-                // Allows the file to be missing, this will only result in all experimental features disabled.
-                ErrorKind::NotFound => {
-                    debug!("`experimental-features.json` not found in dump");
-                    None
-                }
-                _ => return Err(error.into()),
-            },
-        };
-        let features = if let Some(feature_file) = feature_file {
-            Some(serde_json::from_reader(&*feature_file)?)
-        } else {
-            None
-        };
-
         Ok(V6Reader {
             metadata: serde_json::from_reader(&*meta_file)?,
             instance_uid,
             tasks: BufReader::new(File::open(dump.path().join("tasks").join("queue.jsonl"))?),
             keys: BufReader::new(File::open(dump.path().join("keys.jsonl"))?),
-            features,
             dump,
         })
     }
@@ -150,10 +129,6 @@ impl V6Reader {
             (&mut self.keys).lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
         )
     }
-
-    pub fn features(&self) -> Option<RuntimeTogglableFeatures> {
-        self.features
-    }
 }
 
 pub struct UpdateFile {

dump/src/writer.rs

@@ -4,7 +4,6 @@ use std::path::PathBuf;
 
 use flate2::write::GzEncoder;
 use flate2::Compression;
-use meilisearch_types::features::RuntimeTogglableFeatures;
 use meilisearch_types::keys::Key;
 use meilisearch_types::settings::{Checked, Settings};
 use serde_json::{Map, Value};
@@ -54,13 +53,6 @@ impl DumpWriter {
         TaskWriter::new(self.dir.path().join("tasks"))
     }
 
-    pub fn create_experimental_features(&self, features: RuntimeTogglableFeatures) -> Result<()> {
-        Ok(std::fs::write(
-            self.dir.path().join("experimental-features.json"),
-            serde_json::to_string(&features)?,
-        )?)
-    }
-
     pub fn persist_to(self, mut writer: impl Write) -> Result<()> {
         let gz_encoder = GzEncoder::new(&mut writer, Compression::default());
         let mut tar_encoder = tar::Builder::new(gz_encoder);
@@ -219,7 +211,7 @@ pub(crate) mod test {
     fn _create_directory_hierarchy(dir: &Path, depth: usize) -> String {
         let mut ret = String::new();
 
-        // the entries are not guaranteed to be returned in the same order thus we need to sort them.
+        // the entries are not guarenteed to be returned in the same order thus we need to sort them.
         let mut entries =
             fs::read_dir(dir).unwrap().collect::<std::result::Result<Vec<_>, _>>().unwrap();
 
@@ -281,7 +273,7 @@ pub(crate) mod test {
         let dump_path = dump.path();
 
         // ==== checking global file hierarchy (we want to be sure there isn't too many files or too few)
-        insta::assert_snapshot!(create_directory_hierarchy(dump_path), @r###"
+        insta::assert_display_snapshot!(create_directory_hierarchy(dump_path), @r###"
         .
         ├---- indexes/
         │    └---- doggos/
@@ -292,7 +284,6 @@ pub(crate) mod test {
         │    ├---- update_files/
         │    │    └---- 1.jsonl
         │    └---- queue.jsonl
-        ├---- experimental-features.json
         ├---- instance_uid.uuid
         ├---- keys.jsonl
         └---- metadata.json

file-store/Cargo.toml

@@ -11,7 +11,9 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-tempfile = "3.10.1"
-thiserror = "1.0.61"
-tracing = "0.1.40"
-uuid = { version = "1.10.0", features = ["serde", "v4"] }
+tempfile = "3.3.0"
+thiserror = "1.0.30"
+uuid = { version = "1.1.2", features = ["serde", "v4"] }
+
+[dev-dependencies]
+faux = "0.1.8"

file-store/src/lib.rs

@@ -1,5 +1,5 @@
 use std::fs::File as StdFile;
-use std::io::Write;
+use std::ops::{Deref, DerefMut};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 
@@ -22,6 +22,20 @@ pub enum Error {
 
 pub type Result<T> = std::result::Result<T, Error>;
 
+impl Deref for File {
+    type Target = NamedTempFile;
+
+    fn deref(&self) -> &Self::Target {
+        &self.file
+    }
+}
+
+impl DerefMut for File {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        &mut self.file
+    }
+}
+
 #[derive(Clone, Debug)]
 pub struct FileStore {
     path: PathBuf,
@@ -42,7 +56,7 @@ impl FileStore {
         let file = NamedTempFile::new_in(&self.path)?;
         let uuid = Uuid::new_v4();
         let path = self.path.join(uuid.to_string());
-        let update_file = File { file: Some(file), path };
+        let update_file = File { file, path };
 
         Ok((uuid, update_file))
     }
@@ -53,7 +67,7 @@ impl FileStore {
         let file = NamedTempFile::new_in(&self.path)?;
         let uuid = Uuid::from_u128(uuid);
         let path = self.path.join(uuid.to_string());
-        let update_file = File { file: Some(file), path };
+        let update_file = File { file, path };
 
         Ok((uuid, update_file))
     }
@@ -61,13 +75,7 @@ impl FileStore {
     /// Returns the file corresponding to the requested uuid.
     pub fn get_update(&self, uuid: Uuid) -> Result<StdFile> {
         let path = self.get_update_path(uuid);
-        let file = match StdFile::open(path) {
-            Ok(file) => file,
-            Err(e) => {
-                tracing::error!("Can't access update file {uuid}: {e}");
-                return Err(e.into());
-            }
-        };
+        let file = StdFile::open(path)?;
         Ok(file)
     }
 
@@ -102,97 +110,28 @@ impl FileStore {
 
     pub fn delete(&self, uuid: Uuid) -> Result<()> {
         let path = self.path.join(uuid.to_string());
-        if let Err(e) = std::fs::remove_file(path) {
-            tracing::error!("Can't delete file {uuid}: {e}");
-            Err(e.into())
-        } else {
-            Ok(())
-        }
+        std::fs::remove_file(path)?;
+        Ok(())
     }
 
     /// List the Uuids of the files in the FileStore
     pub fn all_uuids(&self) -> Result<impl Iterator<Item = Result<Uuid>>> {
-        Ok(self.path.read_dir()?.filter_map(|entry| {
-            let file_name = match entry {
-                Ok(entry) => entry.file_name(),
-                Err(e) => return Some(Err(e.into())),
-            };
-            let file_name = match file_name.to_str() {
-                Some(file_name) => file_name,
-                None => return Some(Err(Error::CouldNotParseFileNameAsUtf8)),
-            };
-            if file_name.starts_with('.') {
-                None
-            } else {
-                Some(Uuid::from_str(file_name).map_err(|e| e.into()))
-            }
+        Ok(self.path.read_dir()?.map(|entry| {
+            Ok(Uuid::from_str(
+                entry?.file_name().to_str().ok_or(Error::CouldNotParseFileNameAsUtf8)?,
+            )?)
         }))
     }
 }
 
 pub struct File {
     path: PathBuf,
-    file: Option<NamedTempFile>,
+    file: NamedTempFile,
 }
 
 impl File {
-    pub fn dry_file() -> Result<Self> {
-        Ok(Self { path: PathBuf::new(), file: None })
-    }
-
     pub fn persist(self) -> Result<()> {
-        if let Some(file) = self.file {
-            file.persist(&self.path)?;
-        }
+        self.file.persist(&self.path)?;
         Ok(())
     }
 }
-
-impl Write for File {
-    fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
-        if let Some(file) = self.file.as_mut() {
-            file.write(buf)
-        } else {
-            Ok(buf.len())
-        }
-    }
-
-    fn flush(&mut self) -> std::io::Result<()> {
-        if let Some(file) = self.file.as_mut() {
-            file.flush()
-        } else {
-            Ok(())
-        }
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use std::io::Write;
-
-    use tempfile::TempDir;
-
-    use super::*;
-
-    #[test]
-    fn all_uuids() {
-        let dir = TempDir::new().unwrap();
-        let fs = FileStore::new(dir.path()).unwrap();
-        let (uuid, mut file) = fs.new_update().unwrap();
-        file.write_all(b"Hello world").unwrap();
-        file.persist().unwrap();
-        let all_uuids = fs.all_uuids().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(all_uuids, vec![uuid]);
-
-        let (uuid2, file) = fs.new_update().unwrap();
-        let all_uuids = fs.all_uuids().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(all_uuids, vec![uuid]);
-
-        file.persist().unwrap();
-        let mut all_uuids = fs.all_uuids().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        all_uuids.sort();
-        let mut expected = vec![uuid, uuid2];
-        expected.sort();
-        assert_eq!(all_uuids, expected);
-    }
-}

filter-parser/Cargo.toml

@@ -12,9 +12,8 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-nom = "7.1.3"
-nom_locate = "4.2.0"
-unescaper = "0.1.5"
+nom = "7.1.1"
+nom_locate = "4.0.0"
 
 [dev-dependencies]
-insta = "1.39.0"
+insta = "1.21.0"

filter-parser/src/condition.rs

@@ -20,14 +20,10 @@ pub enum Condition<'a> {
     GreaterThanOrEqual(Token<'a>),
     Equal(Token<'a>),
     NotEqual(Token<'a>),
-    Null,
-    Empty,
     Exists,
     LowerThan(Token<'a>),
     LowerThanOrEqual(Token<'a>),
     Between { from: Token<'a>, to: Token<'a> },
-    Contains { keyword: Token<'a>, word: Token<'a> },
-    StartsWith { keyword: Token<'a>, word: Token<'a> },
 }
 
 /// condition      = value ("==" | ">" ...) value
@@ -48,38 +44,6 @@ pub fn parse_condition(input: Span) -> IResult<FilterCondition> {
     Ok((input, condition))
 }
 
-/// null          = value "IS" WS+ "NULL"
-pub fn parse_is_null(input: Span) -> IResult<FilterCondition> {
-    let (input, key) = parse_value(input)?;
-
-    let (input, _) = tuple((tag("IS"), multispace1, tag("NULL")))(input)?;
-    Ok((input, FilterCondition::Condition { fid: key, op: Null }))
-}
-
-/// null          = value "IS" WS+ "NOT" WS+ "NULL"
-pub fn parse_is_not_null(input: Span) -> IResult<FilterCondition> {
-    let (input, key) = parse_value(input)?;
-
-    let (input, _) = tuple((tag("IS"), multispace1, tag("NOT"), multispace1, tag("NULL")))(input)?;
-    Ok((input, FilterCondition::Not(Box::new(FilterCondition::Condition { fid: key, op: Null }))))
-}
-
-/// empty          = value "IS" WS+ "EMPTY"
-pub fn parse_is_empty(input: Span) -> IResult<FilterCondition> {
-    let (input, key) = parse_value(input)?;
-
-    let (input, _) = tuple((tag("IS"), multispace1, tag("EMPTY")))(input)?;
-    Ok((input, FilterCondition::Condition { fid: key, op: Empty }))
-}
-
-/// empty          = value "IS" WS+ "NOT" WS+ "EMPTY"
-pub fn parse_is_not_empty(input: Span) -> IResult<FilterCondition> {
-    let (input, key) = parse_value(input)?;
-
-    let (input, _) = tuple((tag("IS"), multispace1, tag("NOT"), multispace1, tag("EMPTY")))(input)?;
-    Ok((input, FilterCondition::Not(Box::new(FilterCondition::Condition { fid: key, op: Empty }))))
-}
-
 /// exist          = value "EXISTS"
 pub fn parse_exists(input: Span) -> IResult<FilterCondition> {
     let (input, key) = terminated(parse_value, tag("EXISTS"))(input)?;
@@ -94,62 +58,6 @@ pub fn parse_not_exists(input: Span) -> IResult<FilterCondition> {
     Ok((input, FilterCondition::Not(Box::new(FilterCondition::Condition { fid: key, op: Exists }))))
 }
 
-/// contains        = value "CONTAINS" value
-pub fn parse_contains(input: Span) -> IResult<FilterCondition> {
-    let (input, (fid, contains, value)) =
-        tuple((parse_value, tag("CONTAINS"), cut(parse_value)))(input)?;
-    Ok((
-        input,
-        FilterCondition::Condition {
-            fid,
-            op: Contains { keyword: Token { span: contains, value: None }, word: value },
-        },
-    ))
-}
-
-/// contains        = value "NOT" WS+ "CONTAINS" value
-pub fn parse_not_contains(input: Span) -> IResult<FilterCondition> {
-    let keyword = tuple((tag("NOT"), multispace1, tag("CONTAINS")));
-    let (input, (fid, (_not, _spaces, contains), value)) =
-        tuple((parse_value, keyword, cut(parse_value)))(input)?;
-
-    Ok((
-        input,
-        FilterCondition::Not(Box::new(FilterCondition::Condition {
-            fid,
-            op: Contains { keyword: Token { span: contains, value: None }, word: value },
-        })),
-    ))
-}
-
-/// starts with        = value "CONTAINS" value
-pub fn parse_starts_with(input: Span) -> IResult<FilterCondition> {
-    let (input, (fid, starts_with, value)) =
-        tuple((parse_value, tag("STARTS WITH"), cut(parse_value)))(input)?;
-    Ok((
-        input,
-        FilterCondition::Condition {
-            fid,
-            op: StartsWith { keyword: Token { span: starts_with, value: None }, word: value },
-        },
-    ))
-}
-
-/// starts with        = value "NOT" WS+ "CONTAINS" value
-pub fn parse_not_starts_with(input: Span) -> IResult<FilterCondition> {
-    let keyword = tuple((tag("NOT"), multispace1, tag("STARTS WITH")));
-    let (input, (fid, (_not, _spaces, starts_with), value)) =
-        tuple((parse_value, keyword, cut(parse_value)))(input)?;
-
-    Ok((
-        input,
-        FilterCondition::Not(Box::new(FilterCondition::Condition {
-            fid,
-            op: StartsWith { keyword: Token { span: starts_with, value: None }, word: value },
-        })),
-    ))
-}
-
 /// to             = value value "TO" WS+ value
 pub fn parse_to(input: Span) -> IResult<FilterCondition> {
     let (input, (key, from, _, _, to)) =

filter-parser/src/error.rs

@@ -62,7 +62,6 @@ pub enum ErrorKind<'a> {
     MisusedGeoRadius,
     MisusedGeoBoundingBox,
     InvalidPrimary,
-    InvalidEscapedNumber,
     ExpectedEof,
     ExpectedValue(ExpectedValueKind),
     MalformedValue,
@@ -144,12 +143,11 @@ impl<'a> Display for Error<'a> {
             ErrorKind::MissingClosingDelimiter(c) => {
                 writeln!(f, "Expression `{}` is missing the following closing delimiter: `{}`.", escaped_input, c)?
             }
-            ErrorKind::InvalidPrimary => {
-                let text = if input.trim().is_empty() { "but instead got nothing.".to_string() } else { format!("at `{}`.", escaped_input) };
-                writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` {}", text)?
+            ErrorKind::InvalidPrimary if input.trim().is_empty() => {
+                writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.")?
             }
-            ErrorKind::InvalidEscapedNumber => {
-                writeln!(f, "Found an invalid escaped sequence number: `{}`.", escaped_input)?
+            ErrorKind::InvalidPrimary => {
+                writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` at `{}`.", escaped_input)?
             }
             ErrorKind::ExpectedEof => {
                 writeln!(f, "Found unexpected characters at the end of the filter: `{}`. You probably forgot an `OR` or an `AND` rule.", escaped_input)?
@@ -161,7 +159,7 @@ impl<'a> Display for Error<'a> {
                 writeln!(f, "The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.")?
             }
             ErrorKind::ReservedGeo(name) => {
-                writeln!(f, "`{}` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.", name.escape_debug())?
+                writeln!(f, "`{}` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance), or _geoBoundingBox([latitude, longitude], [latitude, longitude]) built-in rules to filter on `_geo` coordinates.", name.escape_debug())?
             }
             ErrorKind::MisusedGeoRadius => {
                 writeln!(f, "The `_geoRadius` filter is an operation and can't be used as a value.")?

filter-parser/src/lib.rs

@@ -47,10 +47,7 @@ mod value;
 use std::fmt::Debug;
 
 pub use condition::{parse_condition, parse_to, Condition};
-use condition::{
-    parse_contains, parse_exists, parse_is_empty, parse_is_not_empty, parse_is_not_null,
-    parse_is_null, parse_not_contains, parse_not_exists, parse_not_starts_with, parse_starts_with,
-};
+use condition::{parse_exists, parse_not_exists};
 use error::{cut_with_err, ExpectedValueKind, NomErrorExt};
 pub use error::{Error, ErrorKind};
 use nom::branch::alt;
@@ -144,41 +141,10 @@ pub enum FilterCondition<'a> {
     Or(Vec<Self>),
     And(Vec<Self>),
     GeoLowerThan { point: [Token<'a>; 2], radius: Token<'a> },
-    GeoBoundingBox { top_right_point: [Token<'a>; 2], bottom_left_point: [Token<'a>; 2] },
-}
-
-pub enum TraversedElement<'a> {
-    FilterCondition(&'a FilterCondition<'a>),
-    Condition(&'a Condition<'a>),
+    GeoBoundingBox { top_left_point: [Token<'a>; 2], bottom_right_point: [Token<'a>; 2] },
 }
 
 impl<'a> FilterCondition<'a> {
-    pub fn use_contains_operator(&self) -> Option<&Token> {
-        match self {
-            FilterCondition::Condition { fid: _, op } => match op {
-                Condition::GreaterThan(_)
-                | Condition::GreaterThanOrEqual(_)
-                | Condition::Equal(_)
-                | Condition::NotEqual(_)
-                | Condition::Null
-                | Condition::Empty
-                | Condition::Exists
-                | Condition::LowerThan(_)
-                | Condition::LowerThanOrEqual(_)
-                | Condition::Between { .. } => None,
-                Condition::Contains { keyword, word: _ }
-                | Condition::StartsWith { keyword, word: _ } => Some(keyword),
-            },
-            FilterCondition::Not(this) => this.use_contains_operator(),
-            FilterCondition::Or(seq) | FilterCondition::And(seq) => {
-                seq.iter().find_map(|filter| filter.use_contains_operator())
-            }
-            FilterCondition::GeoLowerThan { .. }
-            | FilterCondition::GeoBoundingBox { .. }
-            | FilterCondition::In { .. } => None,
-        }
-    }
-
     /// Returns the first token found at the specified depth, `None` if no token at this depth.
     pub fn token_at_depth(&self, depth: usize) -> Option<&Token> {
         match self {
@@ -396,8 +362,8 @@ fn parse_geo_bounding_box(input: Span) -> IResult<FilterCondition> {
     }
 
     let res = FilterCondition::GeoBoundingBox {
-        top_right_point: [args[0][0].into(), args[0][1].into()],
-        bottom_left_point: [args[1][0].into(), args[1][1].into()],
+        top_left_point: [args[0][0].into(), args[0][1].into()],
+        bottom_right_point: [args[1][0].into(), args[1][1].into()],
     };
     Ok((input, res))
 }
@@ -416,34 +382,6 @@ fn parse_geo_point(input: Span) -> IResult<FilterCondition> {
     Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))))
 }
 
-/// geoPoint      = WS* "_geoDistance(float WS* "," WS* float WS* "," WS* float)
-fn parse_geo_distance(input: Span) -> IResult<FilterCondition> {
-    // we want to forbid space BEFORE the _geoDistance but not after
-    tuple((
-        multispace0,
-        tag("_geoDistance"),
-        // if we were able to parse `_geoDistance` we are going to return a Failure whatever happens next.
-        cut(delimited(char('('), separated_list1(tag(","), ws(recognize_float)), char(')'))),
-    ))(input)
-    .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))))?;
-    // if we succeeded we still return a `Failure` because `geoDistance` filters are not allowed
-    Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))))
-}
-
-/// geo      = WS* "_geo(float WS* "," WS* float WS* "," WS* float)
-fn parse_geo(input: Span) -> IResult<FilterCondition> {
-    // we want to forbid space BEFORE the _geo but not after
-    tuple((
-        multispace0,
-        word_exact("_geo"),
-        // if we were able to parse `_geo` we are going to return a Failure whatever happens next.
-        cut(delimited(char('('), separated_list1(tag(","), ws(recognize_float)), char(')'))),
-    ))(input)
-    .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geo"))))?;
-    // if we succeeded we still return a `Failure` because `_geo` filter is not allowed
-    Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geo"))))
-}
-
 fn parse_error_reserved_keyword(input: Span) -> IResult<FilterCondition> {
     match parse_condition(input) {
         Ok(result) => Ok(result),
@@ -476,20 +414,10 @@ fn parse_primary(input: Span, depth: usize) -> IResult<FilterCondition> {
         parse_in,
         parse_not_in,
         parse_condition,
-        parse_is_null,
-        parse_is_not_null,
-        parse_is_empty,
-        parse_is_not_empty,
         parse_exists,
         parse_not_exists,
         parse_to,
-        parse_contains,
-        parse_not_contains,
-        parse_starts_with,
-        parse_not_starts_with,
         // the next lines are only for error handling and are written at the end to have the less possible performance impact
-        parse_geo,
-        parse_geo_distance,
         parse_geo_point,
         parse_error_reserved_keyword,
     ))(input)
@@ -507,6 +435,318 @@ pub fn parse_filter(input: Span) -> IResult<FilterCondition> {
     terminated(|input| parse_expression(input, 0), eof)(input)
 }
 
+#[cfg(test)]
+pub mod tests {
+    use super::*;
+
+    /// Create a raw [Token]. You must specify the string that appear BEFORE your element followed by your element
+    pub fn rtok<'a>(before: &'a str, value: &'a str) -> Token<'a> {
+        // if the string is empty we still need to return 1 for the line number
+        let lines = before.is_empty().then_some(1).unwrap_or_else(|| before.lines().count());
+        let offset = before.chars().count();
+        // the extra field is not checked in the tests so we can set it to nothing
+        unsafe { Span::new_from_raw_offset(offset, lines as u32, value, "") }.into()
+    }
+
+    #[test]
+    fn parse() {
+        use FilterCondition as Fc;
+
+        fn p(s: &str) -> impl std::fmt::Display + '_ {
+            Fc::parse(s).unwrap().unwrap()
+        }
+
+        // Test equal
+        insta::assert_display_snapshot!(p("channel = Ponce"), @"{channel} = {Ponce}");
+        insta::assert_display_snapshot!(p("subscribers = 12"), @"{subscribers} = {12}");
+        insta::assert_display_snapshot!(p("channel = 'Mister Mv'"), @"{channel} = {Mister Mv}");
+        insta::assert_display_snapshot!(p("channel = \"Mister Mv\""), @"{channel} = {Mister Mv}");
+        insta::assert_display_snapshot!(p("'dog race' = Borzoi"), @"{dog race} = {Borzoi}");
+        insta::assert_display_snapshot!(p("\"dog race\" = Chusky"), @"{dog race} = {Chusky}");
+        insta::assert_display_snapshot!(p("\"dog race\" = \"Bernese Mountain\""), @"{dog race} = {Bernese Mountain}");
+        insta::assert_display_snapshot!(p("'dog race' = 'Bernese Mountain'"), @"{dog race} = {Bernese Mountain}");
+        insta::assert_display_snapshot!(p("\"dog race\" = 'Bernese Mountain'"), @"{dog race} = {Bernese Mountain}");
+
+        // Test IN
+        insta::assert_display_snapshot!(p("colour IN[]"), @"{colour} IN[]");
+        insta::assert_display_snapshot!(p("colour IN[green]"), @"{colour} IN[{green}, ]");
+        insta::assert_display_snapshot!(p("colour IN[green,]"), @"{colour} IN[{green}, ]");
+        insta::assert_display_snapshot!(p("colour NOT IN[green,blue]"), @"NOT ({colour} IN[{green}, {blue}, ])");
+        insta::assert_display_snapshot!(p(" colour IN [  green , blue , ]"), @"{colour} IN[{green}, {blue}, ]");
+
+        // Test IN + OR/AND/()
+        insta::assert_display_snapshot!(p(" colour IN [green, blue]  AND color = green "), @"AND[{colour} IN[{green}, {blue}, ], {color} = {green}, ]");
+        insta::assert_display_snapshot!(p("NOT (colour IN [green, blue])  AND color = green "), @"AND[NOT ({colour} IN[{green}, {blue}, ]), {color} = {green}, ]");
+        insta::assert_display_snapshot!(p("x = 1 OR NOT (colour IN [green, blue]  OR color = green) "), @"OR[{x} = {1}, NOT (OR[{colour} IN[{green}, {blue}, ], {color} = {green}, ]), ]");
+
+        // Test whitespace start/end
+        insta::assert_display_snapshot!(p(" colour = green "), @"{colour} = {green}");
+        insta::assert_display_snapshot!(p(" (colour = green OR colour = red) "), @"OR[{colour} = {green}, {colour} = {red}, ]");
+        insta::assert_display_snapshot!(p(" colour IN [green, blue]  AND color = green "), @"AND[{colour} IN[{green}, {blue}, ], {color} = {green}, ]");
+        insta::assert_display_snapshot!(p(" colour NOT  IN [green, blue] "), @"NOT ({colour} IN[{green}, {blue}, ])");
+        insta::assert_display_snapshot!(p(" colour IN [green, blue] "), @"{colour} IN[{green}, {blue}, ]");
+
+        // Test conditions
+        insta::assert_display_snapshot!(p("channel != ponce"), @"{channel} != {ponce}");
+        insta::assert_display_snapshot!(p("NOT channel = ponce"), @"NOT ({channel} = {ponce})");
+        insta::assert_display_snapshot!(p("subscribers < 1000"), @"{subscribers} < {1000}");
+        insta::assert_display_snapshot!(p("subscribers > 1000"), @"{subscribers} > {1000}");
+        insta::assert_display_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
+        insta::assert_display_snapshot!(p("subscribers >= 1000"), @"{subscribers} >= {1000}");
+        insta::assert_display_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
+        insta::assert_display_snapshot!(p("subscribers 100 TO 1000"), @"{subscribers} {100} TO {1000}");
+
+        // Test NOT + EXISTS
+        insta::assert_display_snapshot!(p("subscribers EXISTS"), @"{subscribers} EXISTS");
+        insta::assert_display_snapshot!(p("NOT subscribers < 1000"), @"NOT ({subscribers} < {1000})");
+        insta::assert_display_snapshot!(p("NOT subscribers EXISTS"), @"NOT ({subscribers} EXISTS)");
+        insta::assert_display_snapshot!(p("subscribers NOT EXISTS"), @"NOT ({subscribers} EXISTS)");
+        insta::assert_display_snapshot!(p("NOT subscribers NOT EXISTS"), @"{subscribers} EXISTS");
+        insta::assert_display_snapshot!(p("subscribers NOT   EXISTS"), @"NOT ({subscribers} EXISTS)");
+        insta::assert_display_snapshot!(p("NOT subscribers 100 TO 1000"), @"NOT ({subscribers} {100} TO {1000})");
+
+        // Test nested NOT
+        insta::assert_display_snapshot!(p("NOT NOT NOT NOT x = 5"), @"{x} = {5}");
+        insta::assert_display_snapshot!(p("NOT NOT (NOT NOT x = 5)"), @"{x} = {5}");
+
+        // Test geo radius
+        insta::assert_display_snapshot!(p("_geoRadius(12, 13, 14)"), @"_geoRadius({12}, {13}, {14})");
+        insta::assert_display_snapshot!(p("NOT _geoRadius(12, 13, 14)"), @"NOT (_geoRadius({12}, {13}, {14}))");
+        insta::assert_display_snapshot!(p("_geoRadius(12,13,14)"), @"_geoRadius({12}, {13}, {14})");
+
+        // Test geo bounding box
+        insta::assert_display_snapshot!(p("_geoBoundingBox([12, 13], [14, 15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
+        insta::assert_display_snapshot!(p("NOT _geoBoundingBox([12, 13], [14, 15])"), @"NOT (_geoBoundingBox([{12}, {13}], [{14}, {15}]))");
+        insta::assert_display_snapshot!(p("_geoBoundingBox([12,13],[14,15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
+
+        // Test OR + AND
+        insta::assert_display_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain'"), @"AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
+        insta::assert_display_snapshot!(p("channel = ponce OR 'dog race' != 'bernese mountain'"), @"OR[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
+        insta::assert_display_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000"), @"OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, ]");
+        insta::assert_display_snapshot!(
+        p("channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000 OR colour = red OR colour = blue AND size = 7"),
+        @"OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, {colour} = {red}, AND[{colour} = {blue}, {size} = {7}, ], ]"
+        );
+
+        // Test parentheses
+        insta::assert_display_snapshot!(p("channel = ponce AND ( 'dog race' != 'bernese mountain' OR subscribers > 1000 )"), @"AND[{channel} = {ponce}, OR[{dog race} != {bernese mountain}, {subscribers} > {1000}, ], ]");
+        insta::assert_display_snapshot!(p("(channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000) AND _geoRadius(12, 13, 14)"), @"AND[OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, ], _geoRadius({12}, {13}, {14}), ]");
+
+        // Test recursion
+        // This is the most that is allowed
+        insta::assert_display_snapshot!(
+            p("(((((((((((((((((((((((((((((((((((((((((((((((((x = 1)))))))))))))))))))))))))))))))))))))))))))))))))"),
+            @"{x} = {1}"
+        );
+        insta::assert_display_snapshot!(
+            p("NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT x = 1"),
+            @"NOT ({x} = {1})"
+        );
+
+        // Confusing keywords
+        insta::assert_display_snapshot!(p(r#"NOT "OR" EXISTS AND "EXISTS" NOT EXISTS"#), @"AND[NOT ({OR} EXISTS), NOT ({EXISTS} EXISTS), ]");
+    }
+
+    #[test]
+    fn error() {
+        use FilterCondition as Fc;
+
+        fn p(s: &str) -> impl std::fmt::Display + '_ {
+            Fc::parse(s).unwrap_err().to_string()
+        }
+
+        insta::assert_display_snapshot!(p("channel = Ponce = 12"), @r###"
+        Found unexpected characters at the end of the filter: `= 12`. You probably forgot an `OR` or an `AND` rule.
+        17:21 channel = Ponce = 12
+        "###);
+
+        insta::assert_display_snapshot!(p("channel =    "), @r###"
+        Was expecting a value but instead got nothing.
+        14:14 channel =
+        "###);
+
+        insta::assert_display_snapshot!(p("channel = 🐻"), @r###"
+        Was expecting a value but instead got `🐻`.
+        11:12 channel = 🐻
+        "###);
+
+        insta::assert_display_snapshot!(p("channel = 🐻 AND followers < 100"), @r###"
+        Was expecting a value but instead got `🐻`.
+        11:12 channel = 🐻 AND followers < 100
+        "###);
+
+        insta::assert_display_snapshot!(p("'OR'"), @r###"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` at `\'OR\'`.
+        1:5 'OR'
+        "###);
+
+        insta::assert_display_snapshot!(p("OR"), @r###"
+        Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes.
+        1:3 OR
+        "###);
+
+        insta::assert_display_snapshot!(p("channel Ponce"), @r###"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` at `channel Ponce`.
+        1:14 channel Ponce
+        "###);
+
+        insta::assert_display_snapshot!(p("channel = Ponce OR"), @r###"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.
+        19:19 channel = Ponce OR
+        "###);
+
+        insta::assert_display_snapshot!(p("_geoRadius"), @r###"
+        The `_geoRadius` filter expects three arguments: `_geoRadius(latitude, longitude, radius)`.
+        1:11 _geoRadius
+        "###);
+
+        insta::assert_display_snapshot!(p("_geoRadius = 12"), @r###"
+        The `_geoRadius` filter expects three arguments: `_geoRadius(latitude, longitude, radius)`.
+        1:16 _geoRadius = 12
+        "###);
+
+        insta::assert_display_snapshot!(p("_geoBoundingBox"), @r###"
+        The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
+        1:16 _geoBoundingBox
+        "###);
+
+        insta::assert_display_snapshot!(p("_geoBoundingBox = 12"), @r###"
+        The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
+        1:21 _geoBoundingBox = 12
+        "###);
+
+        insta::assert_display_snapshot!(p("_geoBoundingBox(1.0, 1.0)"), @r###"
+        The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
+        1:26 _geoBoundingBox(1.0, 1.0)
+        "###);
+
+        insta::assert_display_snapshot!(p("_geoPoint(12, 13, 14)"), @r###"
+        `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance), or _geoBoundingBox([latitude, longitude], [latitude, longitude]) built-in rules to filter on `_geo` coordinates.
+        1:22 _geoPoint(12, 13, 14)
+        "###);
+
+        insta::assert_display_snapshot!(p("position <= _geoPoint(12, 13, 14)"), @r###"
+        `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance), or _geoBoundingBox([latitude, longitude], [latitude, longitude]) built-in rules to filter on `_geo` coordinates.
+        13:34 position <= _geoPoint(12, 13, 14)
+        "###);
+
+        insta::assert_display_snapshot!(p("position <= _geoRadius(12, 13, 14)"), @r###"
+        The `_geoRadius` filter is an operation and can't be used as a value.
+        13:35 position <= _geoRadius(12, 13, 14)
+        "###);
+
+        insta::assert_display_snapshot!(p("channel = 'ponce"), @r###"
+        Expression `\'ponce` is missing the following closing delimiter: `'`.
+        11:17 channel = 'ponce
+        "###);
+
+        insta::assert_display_snapshot!(p("channel = \"ponce"), @r###"
+        Expression `\"ponce` is missing the following closing delimiter: `"`.
+        11:17 channel = "ponce
+        "###);
+
+        insta::assert_display_snapshot!(p("channel = mv OR (followers >= 1000"), @r###"
+        Expression `(followers >= 1000` is missing the following closing delimiter: `)`.
+        17:35 channel = mv OR (followers >= 1000
+        "###);
+
+        insta::assert_display_snapshot!(p("channel = mv OR followers >= 1000)"), @r###"
+        Found unexpected characters at the end of the filter: `)`. You probably forgot an `OR` or an `AND` rule.
+        34:35 channel = mv OR followers >= 1000)
+        "###);
+
+        insta::assert_display_snapshot!(p("colour NOT EXIST"), @r###"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` at `colour NOT EXIST`.
+        1:17 colour NOT EXIST
+        "###);
+
+        insta::assert_display_snapshot!(p("subscribers 100 TO1000"), @r###"
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` at `subscribers 100 TO1000`.
+        1:23 subscribers 100 TO1000
+        "###);
+
+        insta::assert_display_snapshot!(p("channel = ponce ORdog != 'bernese mountain'"), @r###"
+        Found unexpected characters at the end of the filter: `ORdog != \'bernese mountain\'`. You probably forgot an `OR` or an `AND` rule.
+        17:44 channel = ponce ORdog != 'bernese mountain'
+        "###);
+
+        insta::assert_display_snapshot!(p("colour IN blue, green]"), @r###"
+        Expected `[` after `IN` keyword.
+        11:23 colour IN blue, green]
+        "###);
+
+        insta::assert_display_snapshot!(p("colour IN [blue, green, 'blue' > 2]"), @r###"
+        Expected only comma-separated field names inside `IN[..]` but instead found `> 2]`.
+        32:36 colour IN [blue, green, 'blue' > 2]
+        "###);
+
+        insta::assert_display_snapshot!(p("colour IN [blue, green, AND]"), @r###"
+        Expected only comma-separated field names inside `IN[..]` but instead found `AND]`.
+        25:29 colour IN [blue, green, AND]
+        "###);
+
+        insta::assert_display_snapshot!(p("colour IN [blue, green"), @r###"
+        Expected matching `]` after the list of field names given to `IN[`
+        23:23 colour IN [blue, green
+        "###);
+
+        insta::assert_display_snapshot!(p("colour IN ['blue, green"), @r###"
+        Expression `\'blue, green` is missing the following closing delimiter: `'`.
+        12:24 colour IN ['blue, green
+        "###);
+
+        insta::assert_display_snapshot!(p("x = EXISTS"), @r###"
+        Was expecting a value but instead got `EXISTS`, which is a reserved keyword. To use `EXISTS` as a field name or a value, surround it by quotes.
+        5:11 x = EXISTS
+        "###);
+
+        insta::assert_display_snapshot!(p("AND = 8"), @r###"
+        Was expecting a value but instead got `AND`, which is a reserved keyword. To use `AND` as a field name or a value, surround it by quotes.
+        1:4 AND = 8
+        "###);
+
+        insta::assert_display_snapshot!(p("((((((((((((((((((((((((((((((((((((((((((((((((((x = 1))))))))))))))))))))))))))))))))))))))))))))))))))"), @r###"
+        The filter exceeded the maximum depth limit. Try rewriting the filter so that it contains fewer nested conditions.
+        51:106 ((((((((((((((((((((((((((((((((((((((((((((((((((x = 1))))))))))))))))))))))))))))))))))))))))))))))))))
+        "###);
+
+        insta::assert_display_snapshot!(
+            p("NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT x = 1"),
+            @r###"
+        The filter exceeded the maximum depth limit. Try rewriting the filter so that it contains fewer nested conditions.
+        797:802 NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT x = 1
+        "###
+        );
+
+        insta::assert_display_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###"
+        Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes.
+        5:7 NOT OR EXISTS AND EXISTS NOT EXISTS
+        "###);
+    }
+
+    #[test]
+    fn depth() {
+        let filter = FilterCondition::parse("account_ids=1 OR account_ids=2 OR account_ids=3 OR account_ids=4 OR account_ids=5 OR account_ids=6").unwrap().unwrap();
+        assert!(filter.token_at_depth(1).is_some());
+        assert!(filter.token_at_depth(2).is_none());
+
+        let filter = FilterCondition::parse("(account_ids=1 OR (account_ids=2 AND account_ids=3) OR (account_ids=4 AND account_ids=5) OR account_ids=6)").unwrap().unwrap();
+        assert!(filter.token_at_depth(2).is_some());
+        assert!(filter.token_at_depth(3).is_none());
+
+        let filter = FilterCondition::parse("account_ids=1 OR account_ids=2 AND account_ids=3 OR account_ids=4 AND account_ids=5 OR account_ids=6").unwrap().unwrap();
+        assert!(filter.token_at_depth(2).is_some());
+        assert!(filter.token_at_depth(3).is_none());
+    }
+
+    #[test]
+    fn token_from_str() {
+        let s = "test string that should not be parsed";
+        let token: Token = s.into();
+        assert_eq!(token.value(), s);
+    }
+}
+
 impl<'a> std::fmt::Display for FilterCondition<'a> {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
@@ -540,10 +780,7 @@ impl<'a> std::fmt::Display for FilterCondition<'a> {
             FilterCondition::GeoLowerThan { point, radius } => {
                 write!(f, "_geoRadius({}, {}, {})", point[0], point[1], radius)
             }
-            FilterCondition::GeoBoundingBox {
-                top_right_point: top_left_point,
-                bottom_left_point: bottom_right_point,
-            } => {
+            FilterCondition::GeoBoundingBox { top_left_point, bottom_right_point } => {
                 write!(
                     f,
                     "_geoBoundingBox([{}, {}], [{}, {}])",
@@ -563,14 +800,10 @@ impl<'a> std::fmt::Display for Condition<'a> {
             Condition::GreaterThanOrEqual(token) => write!(f, ">= {token}"),
             Condition::Equal(token) => write!(f, "= {token}"),
             Condition::NotEqual(token) => write!(f, "!= {token}"),
-            Condition::Null => write!(f, "IS NULL"),
-            Condition::Empty => write!(f, "IS EMPTY"),
             Condition::Exists => write!(f, "EXISTS"),
             Condition::LowerThan(token) => write!(f, "< {token}"),
             Condition::LowerThanOrEqual(token) => write!(f, "<= {token}"),
             Condition::Between { from, to } => write!(f, "{from} TO {to}"),
-            Condition::Contains { word, keyword: _ } => write!(f, "CONTAINS {word}"),
-            Condition::StartsWith { word, keyword: _ } => write!(f, "STARTS WITH {word}"),
         }
     }
 }
@@ -579,409 +812,3 @@ impl<'a> std::fmt::Display for Token<'a> {
         write!(f, "{{{}}}", self.value())
     }
 }
-
-#[cfg(test)]
-pub mod tests {
-    use FilterCondition as Fc;
-
-    use super::*;
-
-    /// Create a raw [Token]. You must specify the string that appear BEFORE your element followed by your element
-    pub fn rtok<'a>(before: &'a str, value: &'a str) -> Token<'a> {
-        // if the string is empty we still need to return 1 for the line number
-        let lines = before.is_empty().then_some(1).unwrap_or_else(|| before.lines().count());
-        let offset = before.chars().count();
-        // the extra field is not checked in the tests so we can set it to nothing
-        unsafe { Span::new_from_raw_offset(offset, lines as u32, value, "") }.into()
-    }
-
-    #[track_caller]
-    fn p(s: &str) -> impl std::fmt::Display + '_ {
-        Fc::parse(s).unwrap().unwrap()
-    }
-
-    #[test]
-    fn parse_escaped() {
-        insta::assert_snapshot!(p(r"title = 'foo\\'"), @r#"{title} = {foo\}"#);
-        insta::assert_snapshot!(p(r"title = 'foo\\\\'"), @r#"{title} = {foo\\}"#);
-        insta::assert_snapshot!(p(r"title = 'foo\\\\\\'"), @r#"{title} = {foo\\\}"#);
-        insta::assert_snapshot!(p(r"title = 'foo\\\\\\\\'"), @r#"{title} = {foo\\\\}"#);
-        // but it also works with other sequences
-        insta::assert_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}");
-    }
-
-    #[test]
-    fn parse() {
-        // Test equal
-        insta::assert_snapshot!(p("channel = Ponce"), @"{channel} = {Ponce}");
-        insta::assert_snapshot!(p("subscribers = 12"), @"{subscribers} = {12}");
-        insta::assert_snapshot!(p("channel = 'Mister Mv'"), @"{channel} = {Mister Mv}");
-        insta::assert_snapshot!(p("channel = \"Mister Mv\""), @"{channel} = {Mister Mv}");
-        insta::assert_snapshot!(p("'dog race' = Borzoi"), @"{dog race} = {Borzoi}");
-        insta::assert_snapshot!(p("\"dog race\" = Chusky"), @"{dog race} = {Chusky}");
-        insta::assert_snapshot!(p("\"dog race\" = \"Bernese Mountain\""), @"{dog race} = {Bernese Mountain}");
-        insta::assert_snapshot!(p("'dog race' = 'Bernese Mountain'"), @"{dog race} = {Bernese Mountain}");
-        insta::assert_snapshot!(p("\"dog race\" = 'Bernese Mountain'"), @"{dog race} = {Bernese Mountain}");
-
-        // Test IN
-        insta::assert_snapshot!(p("colour IN[]"), @"{colour} IN[]");
-        insta::assert_snapshot!(p("colour IN[green]"), @"{colour} IN[{green}, ]");
-        insta::assert_snapshot!(p("colour IN[green,]"), @"{colour} IN[{green}, ]");
-        insta::assert_snapshot!(p("colour NOT IN[green,blue]"), @"NOT ({colour} IN[{green}, {blue}, ])");
-        insta::assert_snapshot!(p(" colour IN [  green , blue , ]"), @"{colour} IN[{green}, {blue}, ]");
-
-        // Test IN + OR/AND/()
-        insta::assert_snapshot!(p(" colour IN [green, blue]  AND color = green "), @"AND[{colour} IN[{green}, {blue}, ], {color} = {green}, ]");
-        insta::assert_snapshot!(p("NOT (colour IN [green, blue])  AND color = green "), @"AND[NOT ({colour} IN[{green}, {blue}, ]), {color} = {green}, ]");
-        insta::assert_snapshot!(p("x = 1 OR NOT (colour IN [green, blue]  OR color = green) "), @"OR[{x} = {1}, NOT (OR[{colour} IN[{green}, {blue}, ], {color} = {green}, ]), ]");
-
-        // Test whitespace start/end
-        insta::assert_snapshot!(p(" colour = green "), @"{colour} = {green}");
-        insta::assert_snapshot!(p(" (colour = green OR colour = red) "), @"OR[{colour} = {green}, {colour} = {red}, ]");
-        insta::assert_snapshot!(p(" colour IN [green, blue]  AND color = green "), @"AND[{colour} IN[{green}, {blue}, ], {color} = {green}, ]");
-        insta::assert_snapshot!(p(" colour NOT  IN [green, blue] "), @"NOT ({colour} IN[{green}, {blue}, ])");
-        insta::assert_snapshot!(p(" colour IN [green, blue] "), @"{colour} IN[{green}, {blue}, ]");
-
-        // Test conditions
-        insta::assert_snapshot!(p("channel != ponce"), @"{channel} != {ponce}");
-        insta::assert_snapshot!(p("NOT channel = ponce"), @"NOT ({channel} = {ponce})");
-        insta::assert_snapshot!(p("subscribers < 1000"), @"{subscribers} < {1000}");
-        insta::assert_snapshot!(p("subscribers > 1000"), @"{subscribers} > {1000}");
-        insta::assert_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
-        insta::assert_snapshot!(p("subscribers >= 1000"), @"{subscribers} >= {1000}");
-        insta::assert_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
-        insta::assert_snapshot!(p("subscribers 100 TO 1000"), @"{subscribers} {100} TO {1000}");
-
-        // Test NOT
-        insta::assert_snapshot!(p("NOT subscribers < 1000"), @"NOT ({subscribers} < {1000})");
-        insta::assert_snapshot!(p("NOT subscribers 100 TO 1000"), @"NOT ({subscribers} {100} TO {1000})");
-
-        // Test NULL + NOT NULL
-        insta::assert_snapshot!(p("subscribers IS NULL"), @"{subscribers} IS NULL");
-        insta::assert_snapshot!(p("NOT subscribers IS NULL"), @"NOT ({subscribers} IS NULL)");
-        insta::assert_snapshot!(p("subscribers IS NOT NULL"), @"NOT ({subscribers} IS NULL)");
-        insta::assert_snapshot!(p("NOT subscribers IS NOT NULL"), @"{subscribers} IS NULL");
-        insta::assert_snapshot!(p("subscribers  IS   NOT   NULL"), @"NOT ({subscribers} IS NULL)");
-
-        // Test EMPTY + NOT EMPTY
-        insta::assert_snapshot!(p("subscribers IS EMPTY"), @"{subscribers} IS EMPTY");
-        insta::assert_snapshot!(p("NOT subscribers IS EMPTY"), @"NOT ({subscribers} IS EMPTY)");
-        insta::assert_snapshot!(p("subscribers IS NOT EMPTY"), @"NOT ({subscribers} IS EMPTY)");
-        insta::assert_snapshot!(p("NOT subscribers IS NOT EMPTY"), @"{subscribers} IS EMPTY");
-        insta::assert_snapshot!(p("subscribers  IS   NOT   EMPTY"), @"NOT ({subscribers} IS EMPTY)");
-
-        // Test EXISTS + NOT EXITS
-        insta::assert_snapshot!(p("subscribers EXISTS"), @"{subscribers} EXISTS");
-        insta::assert_snapshot!(p("NOT subscribers EXISTS"), @"NOT ({subscribers} EXISTS)");
-        insta::assert_snapshot!(p("subscribers NOT EXISTS"), @"NOT ({subscribers} EXISTS)");
-        insta::assert_snapshot!(p("NOT subscribers NOT EXISTS"), @"{subscribers} EXISTS");
-        insta::assert_snapshot!(p("subscribers NOT   EXISTS"), @"NOT ({subscribers} EXISTS)");
-
-        // Test CONTAINS + NOT CONTAINS
-        insta::assert_snapshot!(p("subscribers CONTAINS 'hello'"), @"{subscribers} CONTAINS {hello}");
-        insta::assert_snapshot!(p("NOT subscribers CONTAINS 'hello'"), @"NOT ({subscribers} CONTAINS {hello})");
-        insta::assert_snapshot!(p("subscribers NOT CONTAINS hello"), @"NOT ({subscribers} CONTAINS {hello})");
-        insta::assert_snapshot!(p("NOT subscribers NOT CONTAINS 'hello'"), @"{subscribers} CONTAINS {hello}");
-        insta::assert_snapshot!(p("subscribers NOT   CONTAINS 'hello'"), @"NOT ({subscribers} CONTAINS {hello})");
-
-        // Test STARTS WITH + NOT STARTS WITH
-        insta::assert_snapshot!(p("subscribers STARTS WITH 'hel'"), @"{subscribers} STARTS WITH {hel}");
-        insta::assert_snapshot!(p("NOT subscribers STARTS WITH 'hel'"), @"NOT ({subscribers} STARTS WITH {hel})");
-        insta::assert_snapshot!(p("subscribers NOT STARTS WITH hel"), @"NOT ({subscribers} STARTS WITH {hel})");
-        insta::assert_snapshot!(p("NOT subscribers NOT STARTS WITH 'hel'"), @"{subscribers} STARTS WITH {hel}");
-        insta::assert_snapshot!(p("subscribers NOT   STARTS WITH 'hel'"), @"NOT ({subscribers} STARTS WITH {hel})");
-
-        // Test nested NOT
-        insta::assert_snapshot!(p("NOT NOT NOT NOT x = 5"), @"{x} = {5}");
-        insta::assert_snapshot!(p("NOT NOT (NOT NOT x = 5)"), @"{x} = {5}");
-
-        // Test geo radius
-        insta::assert_snapshot!(p("_geoRadius(12, 13, 14)"), @"_geoRadius({12}, {13}, {14})");
-        insta::assert_snapshot!(p("NOT _geoRadius(12, 13, 14)"), @"NOT (_geoRadius({12}, {13}, {14}))");
-        insta::assert_snapshot!(p("_geoRadius(12,13,14)"), @"_geoRadius({12}, {13}, {14})");
-
-        // Test geo bounding box
-        insta::assert_snapshot!(p("_geoBoundingBox([12, 13], [14, 15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
-        insta::assert_snapshot!(p("NOT _geoBoundingBox([12, 13], [14, 15])"), @"NOT (_geoBoundingBox([{12}, {13}], [{14}, {15}]))");
-        insta::assert_snapshot!(p("_geoBoundingBox([12,13],[14,15])"), @"_geoBoundingBox([{12}, {13}], [{14}, {15}])");
-
-        // Test OR + AND
-        insta::assert_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain'"), @"AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
-        insta::assert_snapshot!(p("channel = ponce OR 'dog race' != 'bernese mountain'"), @"OR[{channel} = {ponce}, {dog race} != {bernese mountain}, ]");
-        insta::assert_snapshot!(p("channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000"), @"OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, ]");
-        insta::assert_snapshot!(
-        p("channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000 OR colour = red OR colour = blue AND size = 7"),
-        @"OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, {colour} = {red}, AND[{colour} = {blue}, {size} = {7}, ], ]"
-        );
-
-        // Test parentheses
-        insta::assert_snapshot!(p("channel = ponce AND ( 'dog race' != 'bernese mountain' OR subscribers > 1000 )"), @"AND[{channel} = {ponce}, OR[{dog race} != {bernese mountain}, {subscribers} > {1000}, ], ]");
-        insta::assert_snapshot!(p("(channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000) AND _geoRadius(12, 13, 14)"), @"AND[OR[AND[{channel} = {ponce}, {dog race} != {bernese mountain}, ], {subscribers} > {1000}, ], _geoRadius({12}, {13}, {14}), ]");
-
-        // Test recursion
-        // This is the most that is allowed
-        insta::assert_snapshot!(
-            p("(((((((((((((((((((((((((((((((((((((((((((((((((x = 1)))))))))))))))))))))))))))))))))))))))))))))))))"),
-            @"{x} = {1}"
-        );
-        insta::assert_snapshot!(
-            p("NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT x = 1"),
-            @"NOT ({x} = {1})"
-        );
-
-        // Confusing keywords
-        insta::assert_snapshot!(p(r#"NOT "OR" EXISTS AND "EXISTS" NOT EXISTS"#), @"AND[NOT ({OR} EXISTS), NOT ({EXISTS} EXISTS), ]");
-    }
-
-    #[test]
-    fn error() {
-        use FilterCondition as Fc;
-
-        fn p(s: &str) -> impl std::fmt::Display + '_ {
-            Fc::parse(s).unwrap_err().to_string()
-        }
-
-        insta::assert_snapshot!(p("channel = Ponce = 12"), @r###"
-        Found unexpected characters at the end of the filter: `= 12`. You probably forgot an `OR` or an `AND` rule.
-        17:21 channel = Ponce = 12
-        "###);
-
-        insta::assert_snapshot!(p("channel =    "), @r###"
-        Was expecting a value but instead got nothing.
-        14:14 channel =
-        "###);
-
-        insta::assert_snapshot!(p("channel = 🐻"), @r###"
-        Was expecting a value but instead got `🐻`.
-        11:12 channel = 🐻
-        "###);
-
-        insta::assert_snapshot!(p("channel = 🐻 AND followers < 100"), @r###"
-        Was expecting a value but instead got `🐻`.
-        11:12 channel = 🐻 AND followers < 100
-        "###);
-
-        insta::assert_snapshot!(p("'OR'"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `\'OR\'`.
-        1:5 'OR'
-        "###);
-
-        insta::assert_snapshot!(p("OR"), @r###"
-        Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes.
-        1:3 OR
-        "###);
-
-        insta::assert_snapshot!(p("channel Ponce"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `channel Ponce`.
-        1:14 channel Ponce
-        "###);
-
-        insta::assert_snapshot!(p("channel = Ponce OR"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.
-        19:19 channel = Ponce OR
-        "###);
-
-        insta::assert_snapshot!(p("_geoRadius"), @r###"
-        The `_geoRadius` filter expects three arguments: `_geoRadius(latitude, longitude, radius)`.
-        1:11 _geoRadius
-        "###);
-
-        insta::assert_snapshot!(p("_geoRadius = 12"), @r###"
-        The `_geoRadius` filter expects three arguments: `_geoRadius(latitude, longitude, radius)`.
-        1:16 _geoRadius = 12
-        "###);
-
-        insta::assert_snapshot!(p("_geoBoundingBox"), @r###"
-        The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
-        1:16 _geoBoundingBox
-        "###);
-
-        insta::assert_snapshot!(p("_geoBoundingBox = 12"), @r###"
-        The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
-        1:21 _geoBoundingBox = 12
-        "###);
-
-        insta::assert_snapshot!(p("_geoBoundingBox(1.0, 1.0)"), @r###"
-        The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.
-        1:26 _geoBoundingBox(1.0, 1.0)
-        "###);
-
-        insta::assert_snapshot!(p("_geoPoint(12, 13, 14)"), @r###"
-        `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
-        1:22 _geoPoint(12, 13, 14)
-        "###);
-
-        insta::assert_snapshot!(p("position <= _geoPoint(12, 13, 14)"), @r###"
-        `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
-        13:34 position <= _geoPoint(12, 13, 14)
-        "###);
-
-        insta::assert_snapshot!(p("_geoDistance(12, 13, 14)"), @r###"
-        `_geoDistance` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
-        1:25 _geoDistance(12, 13, 14)
-        "###);
-
-        insta::assert_snapshot!(p("position <= _geoDistance(12, 13, 14)"), @r###"
-        `_geoDistance` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
-        13:37 position <= _geoDistance(12, 13, 14)
-        "###);
-
-        insta::assert_snapshot!(p("_geo(12, 13, 14)"), @r###"
-        `_geo` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
-        1:17 _geo(12, 13, 14)
-        "###);
-
-        insta::assert_snapshot!(p("position <= _geo(12, 13, 14)"), @r###"
-        `_geo` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
-        13:29 position <= _geo(12, 13, 14)
-        "###);
-
-        insta::assert_snapshot!(p("position <= _geoRadius(12, 13, 14)"), @r###"
-        The `_geoRadius` filter is an operation and can't be used as a value.
-        13:35 position <= _geoRadius(12, 13, 14)
-        "###);
-
-        insta::assert_snapshot!(p("channel = 'ponce"), @r###"
-        Expression `\'ponce` is missing the following closing delimiter: `'`.
-        11:17 channel = 'ponce
-        "###);
-
-        insta::assert_snapshot!(p("channel = \"ponce"), @r###"
-        Expression `\"ponce` is missing the following closing delimiter: `"`.
-        11:17 channel = "ponce
-        "###);
-
-        insta::assert_snapshot!(p("channel = mv OR (followers >= 1000"), @r###"
-        Expression `(followers >= 1000` is missing the following closing delimiter: `)`.
-        17:35 channel = mv OR (followers >= 1000
-        "###);
-
-        insta::assert_snapshot!(p("channel = mv OR followers >= 1000)"), @r###"
-        Found unexpected characters at the end of the filter: `)`. You probably forgot an `OR` or an `AND` rule.
-        34:35 channel = mv OR followers >= 1000)
-        "###);
-
-        insta::assert_snapshot!(p("colour NOT EXIST"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `colour NOT EXIST`.
-        1:17 colour NOT EXIST
-        "###);
-
-        insta::assert_snapshot!(p("subscribers 100 TO1000"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `subscribers 100 TO1000`.
-        1:23 subscribers 100 TO1000
-        "###);
-
-        insta::assert_snapshot!(p("channel = ponce ORdog != 'bernese mountain'"), @r###"
-        Found unexpected characters at the end of the filter: `ORdog != \'bernese mountain\'`. You probably forgot an `OR` or an `AND` rule.
-        17:44 channel = ponce ORdog != 'bernese mountain'
-        "###);
-
-        insta::assert_snapshot!(p("colour IN blue, green]"), @r###"
-        Expected `[` after `IN` keyword.
-        11:23 colour IN blue, green]
-        "###);
-
-        insta::assert_snapshot!(p("colour IN [blue, green, 'blue' > 2]"), @r###"
-        Expected only comma-separated field names inside `IN[..]` but instead found `> 2]`.
-        32:36 colour IN [blue, green, 'blue' > 2]
-        "###);
-
-        insta::assert_snapshot!(p("colour IN [blue, green, AND]"), @r###"
-        Expected only comma-separated field names inside `IN[..]` but instead found `AND]`.
-        25:29 colour IN [blue, green, AND]
-        "###);
-
-        insta::assert_snapshot!(p("colour IN [blue, green"), @r###"
-        Expected matching `]` after the list of field names given to `IN[`
-        23:23 colour IN [blue, green
-        "###);
-
-        insta::assert_snapshot!(p("colour IN ['blue, green"), @r###"
-        Expression `\'blue, green` is missing the following closing delimiter: `'`.
-        12:24 colour IN ['blue, green
-        "###);
-
-        insta::assert_snapshot!(p("x = EXISTS"), @r###"
-        Was expecting a value but instead got `EXISTS`, which is a reserved keyword. To use `EXISTS` as a field name or a value, surround it by quotes.
-        5:11 x = EXISTS
-        "###);
-
-        insta::assert_snapshot!(p("AND = 8"), @r###"
-        Was expecting a value but instead got `AND`, which is a reserved keyword. To use `AND` as a field name or a value, surround it by quotes.
-        1:4 AND = 8
-        "###);
-
-        insta::assert_snapshot!(p("((((((((((((((((((((((((((((((((((((((((((((((((((x = 1))))))))))))))))))))))))))))))))))))))))))))))))))"), @r###"
-        The filter exceeded the maximum depth limit. Try rewriting the filter so that it contains fewer nested conditions.
-        51:106 ((((((((((((((((((((((((((((((((((((((((((((((((((x = 1))))))))))))))))))))))))))))))))))))))))))))))))))
-        "###);
-
-        insta::assert_snapshot!(
-            p("NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT x = 1"),
-            @r###"
-        The filter exceeded the maximum depth limit. Try rewriting the filter so that it contains fewer nested conditions.
-        797:802 NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT NOT x = 1
-        "###
-        );
-
-        insta::assert_snapshot!(p(r#"NOT OR EXISTS AND EXISTS NOT EXISTS"#), @r###"
-        Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes.
-        5:7 NOT OR EXISTS AND EXISTS NOT EXISTS
-        "###);
-
-        insta::assert_snapshot!(p(r#"value NULL"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value NULL`.
-        1:11 value NULL
-        "###);
-        insta::assert_snapshot!(p(r#"value NOT NULL"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value NOT NULL`.
-        1:15 value NOT NULL
-        "###);
-        insta::assert_snapshot!(p(r#"value EMPTY"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value EMPTY`.
-        1:12 value EMPTY
-        "###);
-        insta::assert_snapshot!(p(r#"value NOT EMPTY"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value NOT EMPTY`.
-        1:16 value NOT EMPTY
-        "###);
-        insta::assert_snapshot!(p(r#"value IS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS`.
-        1:9 value IS
-        "###);
-        insta::assert_snapshot!(p(r#"value IS NOT"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT`.
-        1:13 value IS NOT
-        "###);
-        insta::assert_snapshot!(p(r#"value IS EXISTS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS EXISTS`.
-        1:16 value IS EXISTS
-        "###);
-        insta::assert_snapshot!(p(r#"value IS NOT EXISTS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `CONTAINS`, `NOT CONTAINS`, `STARTS WITH`, `NOT STARTS WITH`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT EXISTS`.
-        1:20 value IS NOT EXISTS
-        "###);
-    }
-
-    #[test]
-    fn depth() {
-        let filter = FilterCondition::parse("account_ids=1 OR account_ids=2 OR account_ids=3 OR account_ids=4 OR account_ids=5 OR account_ids=6").unwrap().unwrap();
-        assert!(filter.token_at_depth(1).is_some());
-        assert!(filter.token_at_depth(2).is_none());
-
-        let filter = FilterCondition::parse("(account_ids=1 OR (account_ids=2 AND account_ids=3) OR (account_ids=4 AND account_ids=5) OR account_ids=6)").unwrap().unwrap();
-        assert!(filter.token_at_depth(2).is_some());
-        assert!(filter.token_at_depth(3).is_none());
-
-        let filter = FilterCondition::parse("account_ids=1 OR account_ids=2 AND account_ids=3 OR account_ids=4 AND account_ids=5 OR account_ids=6").unwrap().unwrap();
-        assert!(filter.token_at_depth(2).is_some());
-        assert!(filter.token_at_depth(3).is_none());
-    }
-
-    #[test]
-    fn token_from_str() {
-        let s = "test string that should not be parsed";
-        let token: Token = s.into();
-        assert_eq!(token.value(), s);
-    }
-}

filter-parser/src/value.rs

@@ -7,8 +7,8 @@ use nom::{InputIter, InputLength, InputTake, Slice};
 
 use crate::error::{ExpectedValueKind, NomErrorExt};
 use crate::{
-    parse_geo, parse_geo_bounding_box, parse_geo_distance, parse_geo_point, parse_geo_radius,
-    Error, ErrorKind, IResult, Span, Token,
+    parse_geo_bounding_box, parse_geo_point, parse_geo_radius, Error, ErrorKind, IResult, Span,
+    Token,
 };
 
 /// This function goes through all characters in the [Span] if it finds any escaped character (`\`).
@@ -42,7 +42,7 @@ fn quoted_by(quote: char, input: Span) -> IResult<Token> {
                 )));
             }
         }
-        // if it was preceded by a `\` or if it was anything else we can continue to advance
+        // if it was preceeded by a `\` or if it was anything else we can continue to advance
     }
 
     Ok((
@@ -88,16 +88,11 @@ pub fn parse_value(input: Span) -> IResult<Token> {
     // then, we want to check if the user is misusing a geo expression
     // This expression can’t finish without error.
     // We want to return an error in case of failure.
-    let geo_reserved_parse_functions = [parse_geo_point, parse_geo_distance, parse_geo];
-
-    for parser in geo_reserved_parse_functions {
-        if let Err(err) = parser(input) {
-            if err.is_failure() {
-                return Err(err);
-            }
+    if let Err(err) = parse_geo_point(input) {
+        if err.is_failure() {
+            return Err(err);
         }
     }
-
     match parse_geo_radius(input) {
         Ok(_) => {
             return Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::MisusedGeoRadius)))
@@ -171,24 +166,7 @@ pub fn parse_value(input: Span) -> IResult<Token> {
         })
     })?;
 
-    match unescaper::unescape(value.value()) {
-        Ok(content) => {
-            if content.len() != value.value().len() {
-                Ok((input, Token::new(value.original_span(), Some(content))))
-            } else {
-                Ok((input, value))
-            }
-        }
-        Err(unescaper::Error::IncompleteStr(_)) => Err(nom::Err::Incomplete(nom::Needed::Unknown)),
-        Err(unescaper::Error::ParseIntError { .. }) => Err(nom::Err::Error(Error::new_from_kind(
-            value.original_span(),
-            ErrorKind::InvalidEscapedNumber,
-        ))),
-        Err(unescaper::Error::InvalidChar { .. }) => Err(nom::Err::Error(Error::new_from_kind(
-            value.original_span(),
-            ErrorKind::MalformedValue,
-        ))),
-    }
+    Ok((input, value))
 }
 
 fn is_value_component(c: char) -> bool {
@@ -200,23 +178,7 @@ fn is_syntax_component(c: char) -> bool {
 }
 
 fn is_keyword(s: &str) -> bool {
-    matches!(
-        s,
-        "AND"
-            | "OR"
-            | "IN"
-            | "NOT"
-            | "TO"
-            | "EXISTS"
-            | "IS"
-            | "NULL"
-            | "EMPTY"
-            | "CONTAINS"
-            | "STARTS"
-            | "WITH"
-            | "_geoRadius"
-            | "_geoBoundingBox"
-    )
+    matches!(s, "AND" | "OR" | "IN" | "NOT" | "TO" | "EXISTS" | "_geoRadius" | "_geoBoundingBox")
 }
 
 #[cfg(test)]
@@ -273,8 +235,8 @@ pub mod test {
             ("aaaa", "", rtok("", "aaaa"), "aaaa"),
             (r#"aa"aa"#, r#""aa"#, rtok("", "aa"), "aa"),
             (r#"aa\"aa"#, r#""#, rtok("", r#"aa\"aa"#), r#"aa"aa"#),
-            (r"aa\\\aa", r#""#, rtok("", r"aa\\\aa"), r"aa\\\aa"),
-            (r#"aa\\"\aa"#, r#""\aa"#, rtok("", r"aa\\"), r"aa\\"),
+            (r#"aa\\\aa"#, r#""#, rtok("", r#"aa\\\aa"#), r#"aa\\\aa"#),
+            (r#"aa\\"\aa"#, r#""\aa"#, rtok("", r#"aa\\"#), r#"aa\\"#),
             (r#"aa\\\"\aa"#, r#""#, rtok("", r#"aa\\\"\aa"#), r#"aa\\"\aa"#),
             (r#"\"\""#, r#""#, rtok("", r#"\"\""#), r#""""#),
         ];
@@ -304,12 +266,12 @@ pub mod test {
         );
         // simple quote
         assert_eq!(
-            unescape(Span::new_extra(r"Hello \'World\'", ""), '\''),
+            unescape(Span::new_extra(r#"Hello \'World\'"#, ""), '\''),
             r#"Hello 'World'"#.to_string()
         );
         assert_eq!(
-            unescape(Span::new_extra(r"Hello \\\'World\\\'", ""), '\''),
-            r"Hello \\'World\\'".to_string()
+            unescape(Span::new_extra(r#"Hello \\\'World\\\'"#, ""), '\''),
+            r#"Hello \\'World\\'"#.to_string()
         );
     }
 
@@ -338,19 +300,19 @@ pub mod test {
             ("\"cha'nnel\"", "cha'nnel", false),
             ("I'm tamo", "I", false),
             // escaped thing but not quote
-            (r#""\\""#, r"\", true),
-            (r#""\\\\\\""#, r"\\\", true),
-            (r#""aa\\aa""#, r"aa\aa", true),
+            (r#""\\""#, r#"\\"#, false),
+            (r#""\\\\\\""#, r#"\\\\\\"#, false),
+            (r#""aa\\aa""#, r#"aa\\aa"#, false),
             // with double quote
             (r#""Hello \"world\"""#, r#"Hello "world""#, true),
-            (r#""Hello \\\"world\\\"""#, r#"Hello \"world\""#, true),
+            (r#""Hello \\\"world\\\"""#, r#"Hello \\"world\\""#, true),
             (r#""I'm \"super\" tamo""#, r#"I'm "super" tamo"#, true),
             (r#""\"\"""#, r#""""#, true),
             // with simple quote
-            (r"'Hello \'world\''", r#"Hello 'world'"#, true),
-            (r"'Hello \\\'world\\\''", r"Hello \'world\'", true),
+            (r#"'Hello \'world\''"#, r#"Hello 'world'"#, true),
+            (r#"'Hello \\\'world\\\''"#, r#"Hello \\'world\\'"#, true),
             (r#"'I\'m "super" tamo'"#, r#"I'm "super" tamo"#, true),
-            (r"'\'\''", r#"''"#, true),
+            (r#"'\'\''"#, r#"''"#, true),
         ];
 
         for (input, expected, escaped) in test_case {
@@ -370,14 +332,7 @@ pub mod test {
                 "Filter `{}` was not supposed to be escaped",
                 input
             );
-            assert_eq!(
-                token.value(),
-                expected,
-                "Filter `{}` failed by giving `{}` instead of `{}`.",
-                input,
-                token.value(),
-                expected
-            );
+            assert_eq!(token.value(), expected, "Filter `{}` failed.", input);
         }
     }
 

flatten-serde-json/Cargo.toml

@@ -16,7 +16,7 @@ license.workspace = true
 serde_json = "1.0"
 
 [dev-dependencies]
-criterion = { version = "0.5.1", features = ["html_reports"] }
+criterion = { version = "0.4.0", features = ["html_reports"] }
 
 [[bench]]
 name = "benchmarks"

flatten-serde-json/src/lib.rs

@@ -4,56 +4,51 @@ use serde_json::{Map, Value};
 
 pub fn flatten(json: &Map<String, Value>) -> Map<String, Value> {
     let mut obj = Map::new();
-    let mut all_entries = vec![];
-    insert_object(&mut obj, None, json, &mut all_entries);
-    for (key, old_val) in all_entries {
-        obj.entry(key).or_insert(old_val.clone());
+    let mut all_keys = vec![];
+    insert_object(&mut obj, None, json, &mut all_keys);
+    for key in all_keys {
+        obj.entry(key).or_insert(Value::Array(vec![]));
     }
     obj
 }
 
-fn insert_object<'a>(
+fn insert_object(
     base_json: &mut Map<String, Value>,
     base_key: Option<&str>,
-    object: &'a Map<String, Value>,
-    all_entries: &mut Vec<(String, &'a Value)>,
+    object: &Map<String, Value>,
+    all_keys: &mut Vec<String>,
 ) {
     for (key, value) in object {
         let new_key = base_key.map_or_else(|| key.clone(), |base_key| format!("{base_key}.{key}"));
-        all_entries.push((new_key.clone(), value));
+        all_keys.push(new_key.clone());
         if let Some(array) = value.as_array() {
-            insert_array(base_json, &new_key, array, all_entries);
+            insert_array(base_json, &new_key, array, all_keys);
         } else if let Some(object) = value.as_object() {
-            insert_object(base_json, Some(&new_key), object, all_entries);
+            insert_object(base_json, Some(&new_key), object, all_keys);
         } else {
-            insert_value(base_json, &new_key, value.clone(), false);
+            insert_value(base_json, &new_key, value.clone());
         }
     }
 }
 
-fn insert_array<'a>(
+fn insert_array(
     base_json: &mut Map<String, Value>,
     base_key: &str,
-    array: &'a Vec<Value>,
-    all_entries: &mut Vec<(String, &'a Value)>,
+    array: &Vec<Value>,
+    all_keys: &mut Vec<String>,
 ) {
     for value in array {
         if let Some(object) = value.as_object() {
-            insert_object(base_json, Some(base_key), object, all_entries);
+            insert_object(base_json, Some(base_key), object, all_keys);
         } else if let Some(sub_array) = value.as_array() {
-            insert_array(base_json, base_key, sub_array, all_entries);
+            insert_array(base_json, base_key, sub_array, all_keys);
         } else {
-            insert_value(base_json, base_key, value.clone(), true);
+            insert_value(base_json, base_key, value.clone());
         }
     }
 }
 
-fn insert_value(
-    base_json: &mut Map<String, Value>,
-    key: &str,
-    to_insert: Value,
-    came_from_array: bool,
-) {
+fn insert_value(base_json: &mut Map<String, Value>, key: &str, to_insert: Value) {
     debug_assert!(!to_insert.is_object());
     debug_assert!(!to_insert.is_array());
 
@@ -68,8 +63,6 @@ fn insert_value(
             base_json[key] = Value::Array(vec![value, to_insert]);
         }
         // if it does not exist we can push the value untouched
-    } else if came_from_array {
-        base_json.insert(key.to_string(), Value::Array(vec![to_insert]));
     } else {
         base_json.insert(key.to_string(), to_insert);
     }
@@ -120,11 +113,7 @@ mod tests {
         assert_eq!(
             &flat,
             json!({
-                "a": {
-                  "b": "c",
-                  "d": "e",
-                  "f": "g"
-                },
+                "a": [],
                 "a.b": "c",
                 "a.d": "e",
                 "a.f": "g"
@@ -175,7 +164,7 @@ mod tests {
         assert_eq!(
             &flat,
             json!({
-                "a": [42],
+                "a": 42,
                 "a.b": ["c", "d", "e"],
             })
             .as_object()
@@ -197,7 +186,7 @@ mod tests {
         assert_eq!(
             &flat,
             json!({
-                "a": [null],
+                "a": null,
                 "a.b": ["c", "d", "e"],
             })
             .as_object()
@@ -219,9 +208,7 @@ mod tests {
         assert_eq!(
             &flat,
             json!({
-                "a": {
-                    "b": "c"
-                },
+                "a": [],
                 "a.b": ["c", "d"],
             })
             .as_object()
@@ -247,7 +234,7 @@ mod tests {
             json!({
                 "a.b": ["c", "d", "f"],
                 "a.c": "e",
-                "a": [35],
+                "a": 35,
             })
             .as_object()
             .unwrap()
@@ -315,53 +302,4 @@ mod tests {
             .unwrap()
         );
     }
-
-    #[test]
-    fn flatten_nested_values_keep_original_values() {
-        let mut base: Value = json!({
-            "tags": {
-                "t1": "v1"
-            },
-            "prices": {
-                "p1": [null],
-                "p1000": {"tamo": {"le": {}}}
-            },
-            "kiki": [[]]
-        });
-        let json = std::mem::take(base.as_object_mut().unwrap());
-        let flat = flatten(&json);
-
-        println!("{}", serde_json::to_string_pretty(&flat).unwrap());
-
-        assert_eq!(
-            &flat,
-            json!({
-              "prices": {
-                "p1": [null],
-                "p1000": {
-                  "tamo": {
-                    "le": {}
-                  }
-                }
-              },
-              "prices.p1": [null],
-              "prices.p1000": {
-                "tamo": {
-                  "le": {}
-                }
-              },
-              "prices.p1000.tamo": {
-                "le": {}
-              },
-              "prices.p1000.tamo.le": {},
-              "tags": {
-                "t1": "v1"
-              },
-              "tags.t1": "v1",
-              "kiki": [[]]
-            })
-            .as_object()
-            .unwrap()
-        );
-    }
 }

fuzzers/Cargo.toml

@@ -1,20 +0,0 @@
-[package]
-name = "fuzzers"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
-[dependencies]
-arbitrary = { version = "1.3.2", features = ["derive"] }
-clap = { version = "4.5.9", features = ["derive"] }
-fastrand = "2.1.0"
-milli = { path = "../milli" }
-serde = { version = "1.0.204", features = ["derive"] }
-serde_json = { version = "1.0.120", features = ["preserve_order"] }
-tempfile = "3.10.1"

fuzzers/README.md

@@ -1,3 +0,0 @@
-# Fuzzers
-
-The purpose of this crate is to contains all the handmade "fuzzer" we may need.

fuzzers/src/bin/fuzz-indexing.rs

@@ -1,152 +0,0 @@
-use std::num::NonZeroUsize;
-use std::path::PathBuf;
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
-use std::time::Duration;
-
-use arbitrary::{Arbitrary, Unstructured};
-use clap::Parser;
-use fuzzers::Operation;
-use milli::heed::EnvOpenOptions;
-use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig};
-use milli::Index;
-use tempfile::TempDir;
-
-#[derive(Debug, Arbitrary)]
-struct Batch([Operation; 5]);
-
-#[derive(Debug, Clone, Parser)]
-struct Opt {
-    /// The number of fuzzer to run in parallel.
-    #[clap(long)]
-    par: Option<NonZeroUsize>,
-    // We need to put a lot of newlines in the following documentation or else everything gets collapsed on one line
-    /// The path in which the databases will be created.
-    /// Using a ramdisk is recommended.
-    ///
-    /// Linux:
-    ///
-    /// sudo mount -t tmpfs -o size=2g tmpfs ramdisk # to create it
-    ///
-    /// sudo umount ramdisk # to remove it
-    ///
-    /// MacOS:
-    ///
-    /// diskutil erasevolume HFS+ 'RAM Disk' `hdiutil attach -nobrowse -nomount ram://4194304 # create it
-    ///
-    /// hdiutil detach /dev/:the_disk
-    #[clap(long)]
-    path: Option<PathBuf>,
-}
-
-fn main() {
-    let opt = Opt::parse();
-    let progression: &'static AtomicUsize = Box::leak(Box::new(AtomicUsize::new(0)));
-    let stop: &'static AtomicBool = Box::leak(Box::new(AtomicBool::new(false)));
-
-    let par = opt.par.unwrap_or_else(|| std::thread::available_parallelism().unwrap()).get();
-    let mut handles = Vec::with_capacity(par);
-
-    for _ in 0..par {
-        let opt = opt.clone();
-
-        let handle = std::thread::spawn(move || {
-            let mut options = EnvOpenOptions::new();
-            options.map_size(1024 * 1024 * 1024 * 1024);
-            let tempdir = match opt.path {
-                Some(path) => TempDir::new_in(path).unwrap(),
-                None => TempDir::new().unwrap(),
-            };
-            let index = Index::new(options, tempdir.path()).unwrap();
-            let indexer_config = IndexerConfig::default();
-            let index_documents_config = IndexDocumentsConfig::default();
-
-            std::thread::scope(|s| {
-                loop {
-                    if stop.load(Ordering::Relaxed) {
-                        return;
-                    }
-                    let v: Vec<u8> =
-                        std::iter::repeat_with(|| fastrand::u8(..)).take(1000).collect();
-
-                    let mut data = Unstructured::new(&v);
-                    let batches = <[Batch; 5]>::arbitrary(&mut data).unwrap();
-                    // will be used to display the error once a thread crashes
-                    let dbg_input = format!("{:#?}", batches);
-
-                    let handle = s.spawn(|| {
-                        let mut wtxn = index.write_txn().unwrap();
-
-                        for batch in batches {
-                            let mut builder = IndexDocuments::new(
-                                &mut wtxn,
-                                &index,
-                                &indexer_config,
-                                index_documents_config.clone(),
-                                |_| (),
-                                || false,
-                            )
-                            .unwrap();
-
-                            for op in batch.0 {
-                                match op {
-                                    Operation::AddDoc(doc) => {
-                                        let documents =
-                                            milli::documents::objects_from_json_value(doc.to_d());
-                                        let documents =
-                                            milli::documents::documents_batch_reader_from_objects(
-                                                documents,
-                                            );
-                                        let (b, _added) = builder.add_documents(documents).unwrap();
-                                        builder = b;
-                                    }
-                                    Operation::DeleteDoc(id) => {
-                                        let (b, _removed) =
-                                            builder.remove_documents(vec![id.to_s()]).unwrap();
-                                        builder = b;
-                                    }
-                                }
-                            }
-                            builder.execute().unwrap();
-
-                            // after executing a batch we check if the database is corrupted
-                            let res = index.search(&wtxn).execute().unwrap();
-                            index.documents(&wtxn, res.documents_ids).unwrap();
-                            progression.fetch_add(1, Ordering::Relaxed);
-                        }
-                        wtxn.abort();
-                    });
-                    if let err @ Err(_) = handle.join() {
-                        stop.store(true, Ordering::Relaxed);
-                        err.expect(&dbg_input);
-                    }
-                }
-            });
-        });
-        handles.push(handle);
-    }
-
-    std::thread::spawn(|| {
-        let mut last_value = 0;
-        let start = std::time::Instant::now();
-        loop {
-            let total = progression.load(Ordering::Relaxed);
-            let elapsed = start.elapsed().as_secs();
-            if elapsed > 3600 {
-                // after 1 hour, stop the fuzzer, success
-                std::process::exit(0);
-            }
-            println!(
-                "Has been running for {:?} seconds. Tested {} new values for a total of {}.",
-                elapsed,
-                total - last_value,
-                total
-            );
-            last_value = total;
-            std::thread::sleep(Duration::from_secs(1));
-        }
-    });
-
-    for handle in handles {
-        handle.join().unwrap();
-    }
-}

fuzzers/src/lib.rs

@@ -1,46 +0,0 @@
-use arbitrary::Arbitrary;
-use serde_json::{json, Value};
-
-#[derive(Debug, Arbitrary)]
-pub enum Document {
-    One,
-    Two,
-    Three,
-    Four,
-    Five,
-    Six,
-}
-
-impl Document {
-    pub fn to_d(&self) -> Value {
-        match self {
-            Document::One => json!({ "id": 0, "doggo": "bernese" }),
-            Document::Two => json!({ "id": 0, "doggo": "golden" }),
-            Document::Three => json!({ "id": 0, "catto": "jorts" }),
-            Document::Four => json!({ "id": 1, "doggo": "bernese" }),
-            Document::Five => json!({ "id": 1, "doggo": "golden" }),
-            Document::Six => json!({ "id": 1, "catto": "jorts" }),
-        }
-    }
-}
-
-#[derive(Debug, Arbitrary)]
-pub enum DocId {
-    Zero,
-    One,
-}
-
-impl DocId {
-    pub fn to_s(&self) -> String {
-        match self {
-            DocId::Zero => "0".to_string(),
-            DocId::One => "1".to_string(),
-        }
-    }
-}
-
-#[derive(Debug, Arbitrary)]
-pub enum Operation {
-    AddDoc(Document),
-    DeleteDoc(DocId),
-}

index-scheduler/Cargo.toml

@@ -11,38 +11,28 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.86"
+anyhow = "1.0.64"
 bincode = "1.3.3"
-csv = "1.3.0"
-derive_builder = "0.20.0"
+csv = "1.1.6"
+derive_builder = "0.11.2"
 dump = { path = "../dump" }
-enum-iterator = "2.1.0"
+enum-iterator = "1.1.3"
 file-store = { path = "../file-store" }
-flate2 = "1.0.30"
-meilisearch-auth = { path = "../meilisearch-auth" }
+log = "0.4.14"
 meilisearch-types = { path = "../meilisearch-types" }
-page_size = "0.6.0"
-rayon = "1.10.0"
-roaring = { version = "0.10.6", features = ["serde"] }
-serde = { version = "1.0.204", features = ["derive"] }
-serde_json = { version = "1.0.120", features = ["preserve_order"] }
+page_size = "0.5.0"
+roaring = { version = "0.10.0", features = ["serde"] }
+serde = { version = "1.0.136", features = ["derive"] }
+serde_json = { version = "1.0.85", features = ["preserve_order"] }
 synchronoise = "1.0.1"
-tempfile = "3.10.1"
-thiserror = "1.0.61"
-time = { version = "0.3.36", features = [
-    "serde-well-known",
-    "formatting",
-    "parsing",
-    "macros",
-] }
-tracing = "0.1.40"
-ureq = "2.10.0"
-uuid = { version = "1.10.0", features = ["serde", "v4"] }
+tempfile = "3.3.0"
+thiserror = "1.0.30"
+time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+uuid = { version = "1.1.2", features = ["serde", "v4"] }
 
 [dev-dependencies]
-arroy = "0.4.0"
 big_s = "1.0.2"
-crossbeam = "0.8.4"
-insta = { version = "1.39.0", features = ["json", "redactions"] }
-maplit = "1.0.2"
+crossbeam = "0.8.2"
+insta = { version = "1.19.1", features = ["json", "redactions"] }
 meili-snap = { path = "../meili-snap" }
+nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"}

index-scheduler/src/autobatcher.rs

@@ -24,10 +24,7 @@ enum AutobatchKind {
         allow_index_creation: bool,
         primary_key: Option<String>,
     },
-    DocumentEdition,
-    DocumentDeletion {
-        by_filter: bool,
-    },
+    DocumentDeletion,
     DocumentClear,
     Settings {
         allow_index_creation: bool,
@@ -65,14 +62,8 @@ impl From<KindWithContent> for AutobatchKind {
                 primary_key,
                 ..
             } => AutobatchKind::DocumentImport { method, allow_index_creation, primary_key },
-            KindWithContent::DocumentEdition { .. } => AutobatchKind::DocumentEdition,
-            KindWithContent::DocumentDeletion { .. } => {
-                AutobatchKind::DocumentDeletion { by_filter: false }
-            }
+            KindWithContent::DocumentDeletion { .. } => AutobatchKind::DocumentDeletion,
             KindWithContent::DocumentClear { .. } => AutobatchKind::DocumentClear,
-            KindWithContent::DocumentDeletionByFilter { .. } => {
-                AutobatchKind::DocumentDeletion { by_filter: true }
-            }
             KindWithContent::SettingsUpdate { allow_index_creation, is_deletion, .. } => {
                 AutobatchKind::Settings {
                     allow_index_creation: allow_index_creation && !is_deletion,
@@ -103,12 +94,8 @@ pub enum BatchKind {
         primary_key: Option<String>,
         operation_ids: Vec<TaskId>,
     },
-    DocumentEdition {
-        id: TaskId,
-    },
     DocumentDeletion {
         deletion_ids: Vec<TaskId>,
-        includes_by_filter: bool,
     },
     ClearAndSettings {
         other: Vec<TaskId>,
@@ -166,7 +153,7 @@ impl BatchKind {
 impl BatchKind {
     /// Returns a `ControlFlow::Break` if you must stop right now.
     /// The boolean tell you if an index has been created by the batched task.
-    /// To ease the writing of the code. `true` can be returned when you don't need to create an index
+    /// To ease the writting of the code. `true` can be returned when you don't need to create an index
     /// but false can't be returned if you needs to create an index.
     // TODO use an AutoBatchKind as input
     pub fn new(
@@ -205,14 +192,9 @@ impl BatchKind {
                 }),
                 allow_index_creation,
             ),
-            K::DocumentEdition => (Break(BatchKind::DocumentEdition { id: task_id }), false),
-            K::DocumentDeletion { by_filter: includes_by_filter } => (
-                Continue(BatchKind::DocumentDeletion {
-                    deletion_ids: vec![task_id],
-                    includes_by_filter,
-                }),
-                false,
-            ),
+            K::DocumentDeletion => {
+                (Continue(BatchKind::DocumentDeletion { deletion_ids: vec![task_id] }), false)
+            }
             K::Settings { allow_index_creation } => (
                 Continue(BatchKind::Settings { allow_index_creation, settings_ids: vec![task_id] }),
                 allow_index_creation,
@@ -222,7 +204,7 @@ impl BatchKind {
 
     /// Returns a `ControlFlow::Break` if you must stop right now.
     /// The boolean tell you if an index has been created by the batched task.
-    /// To ease the writing of the code. `true` can be returned when you don't need to create an index
+    /// To ease the writting of the code. `true` can be returned when you don't need to create an index
     /// but false can't be returned if you needs to create an index.
     #[rustfmt::skip]
     fn accumulate(self, id: TaskId, kind: AutobatchKind, index_already_exists: bool, primary_key: Option<&str>) -> ControlFlow<BatchKind, BatchKind> {
@@ -230,7 +212,7 @@ impl BatchKind {
 
         match (self, kind) {
             // We don't batch any of these operations
-            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentEdition) => Break(this),
+            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap) => Break(this),
             // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists.
             (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => {
                 Break(this)
@@ -266,7 +248,7 @@ impl BatchKind {
             // The index deletion can batch with everything but must stop after
             (
                 BatchKind::DocumentClear { mut ids }
-                | BatchKind::DocumentDeletion { deletion_ids: mut ids, includes_by_filter: _ }
+                | BatchKind::DocumentDeletion { deletion_ids: mut ids }
                 | BatchKind::DocumentOperation { method: _, allow_index_creation: _, primary_key: _, operation_ids: mut ids }
                 | BatchKind::Settings { allow_index_creation: _, settings_ids: mut ids },
                 K::IndexDeletion,
@@ -286,7 +268,7 @@ impl BatchKind {
 
             (
                 BatchKind::DocumentClear { mut ids },
-                K::DocumentClear | K::DocumentDeletion { by_filter: _ },
+                K::DocumentClear | K::DocumentDeletion,
             ) => {
                 ids.push(id);
                 Continue(BatchKind::DocumentClear { ids })
@@ -330,7 +312,7 @@ impl BatchKind {
             }
             (
                 BatchKind::DocumentOperation { method, allow_index_creation, primary_key, mut operation_ids },
-                K::DocumentDeletion { by_filter: false },
+                K::DocumentDeletion,
             ) => {
                 operation_ids.push(id);
 
@@ -341,13 +323,6 @@ impl BatchKind {
                     operation_ids,
                 })
             }
-            // We can't batch a document operation with a delete by filter
-            (
-                this @ BatchKind::DocumentOperation { .. },
-                K::DocumentDeletion { by_filter: true },
-            ) => {
-                Break(this)
-            }
             // but we can't autobatch documents if it's not the same kind
             // this match branch MUST be AFTER the previous one
             (
@@ -366,18 +341,13 @@ impl BatchKind {
                 operation_ids,
             }),
 
-            (BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: _ }, K::DocumentClear) => {
+            (BatchKind::DocumentDeletion { mut deletion_ids }, K::DocumentClear) => {
                 deletion_ids.push(id);
                 Continue(BatchKind::DocumentClear { ids: deletion_ids })
             }
-            // we can't autobatch the deletion and import if the document deletion contained a filter
-            (
-                this @ BatchKind::DocumentDeletion { deletion_ids: _, includes_by_filter: true },
-                K::DocumentImport { .. }
-            ) => Break(this),
             // we can autobatch the deletion and import if the index already exists
             (
-                BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: false },
+                BatchKind::DocumentDeletion { mut deletion_ids },
                 K::DocumentImport { method, allow_index_creation, primary_key }
             ) if index_already_exists => {
                 deletion_ids.push(id);
@@ -391,7 +361,7 @@ impl BatchKind {
             }
             // we can autobatch the deletion and import if both can't create an index
             (
-                BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter: false },
+                BatchKind::DocumentDeletion { mut deletion_ids },
                 K::DocumentImport { method, allow_index_creation, primary_key }
             ) if !allow_index_creation => {
                 deletion_ids.push(id);
@@ -410,9 +380,9 @@ impl BatchKind {
             ) => {
                 Break(this)
             }
-            (BatchKind::DocumentDeletion { mut deletion_ids, includes_by_filter }, K::DocumentDeletion { by_filter }) => {
+            (BatchKind::DocumentDeletion { mut deletion_ids }, K::DocumentDeletion) => {
                 deletion_ids.push(id);
-                Continue(BatchKind::DocumentDeletion { deletion_ids, includes_by_filter: includes_by_filter | by_filter })
+                Continue(BatchKind::DocumentDeletion { deletion_ids })
             }
             (this @ BatchKind::DocumentDeletion { .. }, K::Settings { .. }) => Break(this),
 
@@ -426,7 +396,7 @@ impl BatchKind {
             }),
             (
                 this @ BatchKind::Settings { .. },
-                K::DocumentImport { .. } | K::DocumentDeletion { .. },
+                K::DocumentImport { .. } | K::DocumentDeletion,
             ) => Break(this),
             (
                 BatchKind::Settings { mut settings_ids, allow_index_creation },
@@ -457,7 +427,7 @@ impl BatchKind {
                     settings_ids,
                     allow_index_creation,
                 },
-                K::DocumentDeletion { .. },
+                K::DocumentDeletion,
             ) => {
                 other.push(id);
                 Continue(BatchKind::ClearAndSettings {
@@ -519,7 +489,7 @@ impl BatchKind {
             // this MUST be AFTER the two previous branch
             (
                 this @ BatchKind::SettingsAndDocumentOperation { .. },
-                K::DocumentDeletion { .. } | K::DocumentImport { .. },
+                K::DocumentDeletion | K::DocumentImport { .. },
             ) => Break(this),
             (
                 BatchKind::SettingsAndDocumentOperation { mut settings_ids, method, allow_index_creation,primary_key, operation_ids },
@@ -538,8 +508,7 @@ impl BatchKind {
                 BatchKind::IndexCreation { .. }
                 | BatchKind::IndexDeletion { .. }
                 | BatchKind::IndexUpdate { .. }
-                | BatchKind::IndexSwap { .. }
-                | BatchKind::DocumentEdition { .. },
+                | BatchKind::IndexSwap { .. },
                 _,
             ) => {
                 unreachable!()
@@ -629,13 +598,6 @@ mod tests {
         }
     }
 
-    fn doc_del_fil() -> KindWithContent {
-        KindWithContent::DocumentDeletionByFilter {
-            index_uid: String::from("doggo"),
-            filter_expr: serde_json::json!("cuteness > 100"),
-        }
-    }
-
     fn doc_clr() -> KindWithContent {
         KindWithContent::DocumentClear { index_uid: String::from("doggo") }
     }
@@ -696,16 +658,10 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1, 2] }, false))");
 
         // we can autobatch one or multiple DocumentDeletion together
-        debug_snapshot!(autobatch_from(true, None, [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: false }, false))");
-
-        // we can autobatch one or multiple DocumentDeletionByFilter together
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_del_fil(), doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del_fil(), doc_del_fil(), doc_del_fil()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false,None,  [doc_del(), doc_del(), doc_del()]), @"Some((DocumentDeletion { deletion_ids: [0, 1, 2] }, false))");
 
         // we can autobatch one or multiple Settings together
         debug_snapshot!(autobatch_from(true, None, [settings(true)]), @"Some((Settings { allow_index_creation: true, settings_ids: [0] }, true))");
@@ -748,63 +704,25 @@ mod tests {
         debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
         debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, false, Some("catto"))]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0, 1] }, false))"###);
-
-        // But we can't autobatch document addition with document deletion by filter
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, true, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: Some("catto"), operation_ids: [0] }, true))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(ReplaceDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
-        debug_snapshot!(autobatch_from(false, None, [doc_imp(UpdateDocuments, false, Some("catto")), doc_del_fil()]), @r###"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: Some("catto"), operation_ids: [0] }, false))"###);
-        // And the other way around
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(ReplaceDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del_fil(), doc_imp(UpdateDocuments, false, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
     }
 
     #[test]
     fn simple_document_operation_dont_autobatch_with_other() {
-        // addition, updates and deletion by filter can't batch together
+        // addition, updates and deletion can't batch together
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), doc_del_fil()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
 
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_create()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_create()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_create()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
 
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_update()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_update()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_update()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
 
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, true, None), idx_swap()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, true, None), idx_swap()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: true, primary_key: None, operation_ids: [0] }, true))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: true }, false))");
+        debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_swap()]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
     }
 
     #[test]
@@ -871,7 +789,6 @@ mod tests {
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_del()]), @"Some((IndexDeletion { ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_del_fil()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), doc_clr()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), settings(true)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(true, None, [idx_del(), settings(false)]), @"Some((IndexDeletion { ids: [0] }, false))");
@@ -881,7 +798,6 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_imp(ReplaceDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_imp(UpdateDocuments, false, None)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_del()]), @"Some((IndexDeletion { ids: [0] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_del_fil()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), doc_clr()]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), settings(true)]), @"Some((IndexDeletion { ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [idx_del(), settings(false)]), @"Some((IndexDeletion { ids: [0] }, false))");
@@ -893,7 +809,6 @@ mod tests {
         debug_snapshot!(autobatch_from(true, None, [doc_imp(ReplaceDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [doc_imp(UpdateDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [doc_del(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(true, None, [doc_del_fil(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(true, None, [settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
         debug_snapshot!(autobatch_from(true, None, [settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
@@ -903,7 +818,6 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_del(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
-        debug_snapshot!(autobatch_from(false,None,  [doc_del_fil(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [settings(true), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, true))");
         debug_snapshot!(autobatch_from(false,None,  [settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 1] }, false))");
@@ -945,7 +859,7 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), settings(false), idx_del()]), @"Some((IndexDeletion { ids: [0, 2, 1] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments,false, None), settings(false), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), settings(false), doc_clr(), idx_del()]), @"Some((IndexDeletion { ids: [1, 3, 0, 2] }, false))");
-        // The third and final case is when the first task doesn't create an index but is directly followed by a task creating an index. In this case we can't batch whit what
+        // The third and final case is when the first task doesn't create an index but is directly followed by a task creating an index. In this case we can't batch whith what
         // follows because we first need to process the erronous batch.
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments,false, None), settings(true), idx_del()]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(UpdateDocuments, false, None), settings(true), idx_del()]), @"Some((DocumentOperation { method: UpdateDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
@@ -969,10 +883,10 @@ mod tests {
         debug_snapshot!(autobatch_from(false,None,  [doc_imp(ReplaceDocuments, false, None), settings(true)]), @"Some((DocumentOperation { method: ReplaceDocuments, allow_index_creation: false, primary_key: None, operation_ids: [0] }, false))");
 
         // batch deletion and addition
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
-        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0], includes_by_filter: false }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, Some("catto"))]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(ReplaceDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
+        debug_snapshot!(autobatch_from(false, None, [doc_del(), doc_imp(UpdateDocuments, true, None)]), @"Some((DocumentDeletion { deletion_ids: [0] }, false))");
     }
 
     #[test]