Merge #3501

3501: Update milli 0.41.2 r=curquiza a=ManyTheFish

# Pull Request

- [Update milli 0.41.2](51d144bb28)

- [update meilisearch v1.0.1](9e5ef14a25)

Fixes #3483


Co-authored-by: ManyTheFish <many@meilisearch.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -23,8 +23,7 @@ A clear and concise description of what you expected to happen.
 **Screenshots**
 If applicable, add screenshots to help explain your problem.
 
-**Meilisearch version:**
-[e.g. v0.20.0]
+**Meilisearch version:** [e.g. v0.20.0]
 
 **Additional context**
 Additional information that may be relevant to the issue.

.github/ISSUE_TEMPLATE/sprint_issue.md

@@ -1,32 +0,0 @@
----
-name: New sprint issue
-about: ⚠️ Should only be used by the engine team ⚠️
-title: ''
-labels: ''
-assignees: ''
-
----
-
-Related product team resources: [PRD]() (_internal only_)
-Related product discussion:
-Related spec: WIP
-
-## Motivation
-
-<!---Copy/paste the information in PRD or briefly detail the product motivation. Ask product team if any hesitation.-->
-
-## Usage
-
-<!---Link to the public part of the PRD, or to the related product discussion for experimental features-->
-
-## TODO
-
-<!---Feel free to adapt this list with more technical/product steps-->
-
-- [ ] Release a prototype
-- [ ] If prototype validated, merge changes into `main`
-- [ ] Update the spec
-
-## Impacted teams
-
-<!---Ping the related teams. Ask for the engine manager if any hesitation-->

.github/dependabot.yml

@@ -2,6 +2,7 @@
 
 version: 2
 updates:
+
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/scripts/check-release.sh

@@ -1,41 +1,28 @@
-#!/usr/bin/env bash
-set -eu -o pipefail
+#!/bin/bash
 
-check_tag() {
-    local expected=$1
-    local actual=$2
-    local filename=$3
-
-    if [[ $actual != $expected ]]; then
-        echo >&2 "Error: the current tag does not match the version in $filename: found $actual, expected $expected"
-        return 1
-    fi
+# check_tag $current_tag $file_tag $file_name
+function check_tag {
+  if [[ "$1" != "$2" ]]; then
+      echo "Error: the current tag does not match the version in $3: found $2 - expected $1"
+      ret=1
+  fi
 }
 
-read_version() {
-    grep '^version = ' | cut -d \" -f 2
-}
-
-if [[ -z "${GITHUB_REF:-}" ]]; then
-    echo >&2 "Error: GITHUB_REF is not set"
-    exit 1
-fi
-
-if [[ ! "$GITHUB_REF" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+(-[a-z0-9]+)?$ ]]; then
-    echo >&2 "Error: GITHUB_REF is not a valid tag: $GITHUB_REF"
-    exit 1
-fi
-
-current_tag=${GITHUB_REF#refs/tags/v}
 ret=0
+current_tag=${GITHUB_REF#'refs/tags/v'}
 
-toml_tag="$(cat Cargo.toml | read_version)"
-check_tag "$current_tag" "$toml_tag" Cargo.toml || ret=1
+toml_files='*/Cargo.toml'
+for toml_file in $toml_files;
+do
+    file_tag="$(grep '^version = ' $toml_file | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')"
+    check_tag $current_tag $file_tag $toml_file
+done
 
-lock_tag=$(grep -A 1 '^name = "meilisearch-auth"' Cargo.lock | read_version)
-check_tag "$current_tag" "$lock_tag" Cargo.lock || ret=1
+lock_file='Cargo.lock'
+lock_tag=$(grep -A 1 'name = "meilisearch-auth"' $lock_file | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')
+check_tag $current_tag $lock_tag $lock_file
 
-if (( ret == 0 )); then
-    echo 'OK'
+if [[ "$ret" -eq 0 ]] ; then
+  echo 'OK'
 fi
 exit $ret

.github/workflows/benchmarks-manual.yml

@@ -1,77 +0,0 @@
-name: Benchmarks (manual)
-
-on:
-  workflow_dispatch:
-    inputs:
-      dataset_name:
-        description: 'The name of the dataset used to benchmark (search_songs, search_wiki, search_geo or indexing)'
-        required: false
-        default: 'search_songs'
-
-env:
-  BENCH_NAME: ${{ github.event.inputs.dataset_name }}
-
-jobs:
-  benchmarks:
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Helper
-      - name: 'README: compare with another benchmark'
-        run: |
-          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
-          echo 'How to compare this benchmark with another one?'
-          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scripts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/benchmarks-pr.yml

@@ -1,97 +0,0 @@
-name: Benchmarks (PR)
-on: issue_comment
-permissions:
-  issues: write
-
-env:
-  GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-
-jobs:
-  run-benchmarks-on-comment:
-    if: startsWith(github.event.comment.body, '/benchmark')
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      - name: Check for Command
-        id: command
-        uses: xt0rted/slash-command-action@v2
-        with:
-          command: benchmark
-          reaction-type: "eyes"
-          repo-token: ${{ env.GH_TOKEN }}
-
-      - uses: xt0rted/pull-request-comment-branch@v2
-        id: comment-branch
-        with:
-          repo_token: ${{ env.GH_TOKEN }}
-
-      - uses: actions/checkout@v3
-        if: success()
-        with:
-          fetch-depth: 0 # fetch full history to be able to get main commit sha
-          ref: ${{ steps.comment-branch.outputs.head_ref }}
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(git rev-parse --abbrev-ref HEAD | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${{ steps.command.outputs.command-arguments }}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${{ steps.command.outputs.command-arguments }} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${{ steps.command.outputs.command-arguments }} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Compute the diff of the benchmarks and send a message on the GitHub PR
-      - name: Compute and send a message in the PR
-        env:
-          GITHUB_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-        run: |
-          set -x
-          export base_ref=$(git merge-base origin/main ${{ steps.comment-branch.outputs.head_ref }} | head -c8)
-          export base_filename=$(echo ${{ steps.command.outputs.command-arguments }}_main_${base_ref}.json)
-          echo 'Here are your benchmarks diff 👊' >> body.txt
-          echo '```' >> body.txt
-          ./benchmarks/scripts/compare.sh $base_filename ${{ steps.file.outputs.basename }}.json >> body.txt
-          echo '```' >> body.txt
-          gh pr comment ${{ steps.current_branch.outputs.name }} --body-file body.txt

.github/workflows/benchmarks-push-indexing.yml

@@ -1,79 +0,0 @@
-name: Benchmarks of indexing (push)
-
-on:
-  push:
-    branches:
-      - main
-
-env:
-  INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
-  BENCH_NAME: "indexing"
-
-jobs:
-  benchmarks:
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Upload benchmarks to influxdb
-      - name: Upload ${{ steps.file.outputs.basename }}.json to influxDB
-        run: telegraf --config https://eu-central-1-1.aws.cloud2.influxdata.com/api/v2/telegrafs/08b52e34a370b000 --once --debug
-
-      # Helper
-      - name: 'README: compare with another benchmark'
-        run: |
-          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
-          echo 'How to compare this benchmark with another one?'
-          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/benchmarks-push-search-geo.yml

@@ -1,78 +0,0 @@
-name: Benchmarks of search for geo (push)
-
-on:
-  push:
-    branches:
-      - main
-
-env:
-  BENCH_NAME: "search_geo"
-  INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
-
-jobs:
-  benchmarks:
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Upload benchmarks to influxdb
-      - name: Upload ${{ steps.file.outputs.basename }}.json to influxDB
-        run: telegraf --config https://eu-central-1-1.aws.cloud2.influxdata.com/api/v2/telegrafs/08b52e34a370b000 --once --debug
-
-      # Helper
-      - name: 'README: compare with another benchmark'
-        run: |
-          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
-          echo 'How to compare this benchmark with another one?'
-          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/benchmarks-push-search-songs.yml

@@ -1,78 +0,0 @@
-name: Benchmarks of search for songs (push)
-
-on:
-  push:
-    branches:
-      - main
-
-env:
-  BENCH_NAME: "search_songs"
-  INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
-
-jobs:
-  benchmarks:
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Upload benchmarks to influxdb
-      - name: Upload ${{ steps.file.outputs.basename }}.json to influxDB
-        run: telegraf --config https://eu-central-1-1.aws.cloud2.influxdata.com/api/v2/telegrafs/08b52e34a370b000 --once --debug
-
-      # Helper
-      - name: 'README: compare with another benchmark'
-        run: |
-          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
-          echo 'How to compare this benchmark with another one?'
-          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/benchmarks-push-search-wiki.yml

@@ -1,78 +0,0 @@
-name: Benchmarks of search for Wikipedia articles (push)
-
-on:
-  push:
-    branches:
-      - main
-
-env:
-  BENCH_NAME: "search_wiki"
-  INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
-
-jobs:
-  benchmarks:
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Upload benchmarks to influxdb
-      - name: Upload ${{ steps.file.outputs.basename }}.json to influxDB
-        run: telegraf --config https://eu-central-1-1.aws.cloud2.influxdata.com/api/v2/telegrafs/08b52e34a370b000 --once --debug
-
-      # Helper
-      - name: 'README: compare with another benchmark'
-        run: |
-          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
-          echo 'How to compare this benchmark with another one?'
-          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/create-issue-dependencies.yml

@@ -0,0 +1,23 @@
+name: Create issue to upgrade dependencies
+on:
+  schedule:
+    - cron: '0 0 1 */3 *'
+  workflow_dispatch:
+
+jobs:
+  create-issue:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Create an issue
+      uses: actions-ecosystem/action-create-issue@v1
+      with:
+        github_token: ${{ secrets.MEILI_BOT_GH_PAT }}
+        title: Upgrade dependencies
+        body: |
+          We need to update the dependencies of the Meilisearch repository, and, if possible, the dependencies of all the engine-team repositories that Meilisearch depends on (milli, charabia, heed...).
+
+          ⚠️ This issue should only be done at the beginning of the sprint!
+        labels: |
+          dependencies
+          maintenance

.github/workflows/dependency-issue.yml

@@ -1,24 +0,0 @@
-name: Create issue to upgrade dependencies
-
-on:
-  schedule:
-    # Run the first of the month, every 6 month
-    - cron: '0 0 1 */6 *'
-  workflow_dispatch:
-
-jobs:
-  create-issue:
-    runs-on: ubuntu-latest
-    env:
-      ISSUE_TEMPLATE: issue-template.md
-      GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-    steps:
-    - uses: actions/checkout@v3
-    - name: Download the issue template
-      run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/dependency-issue.md > $ISSUE_TEMPLATE
-    - name: Create issue
-      run: |
-        gh issue create \
-          --title 'Upgrade dependencies' \
-          --label 'dependencies,maintenance' \
-          --body-file $ISSUE_TEMPLATE

.github/workflows/flaky.yml

@@ -22,11 +22,5 @@ jobs:
         override: true
     - name: Install cargo-flaky
       run: cargo install cargo-flaky
-    - name: Run cargo flaky in the dumps
-      run: cd dump; cargo flaky -i 100 --release
-    - name: Run cargo flaky in the index-scheduler
-      run: cd index-scheduler; cargo flaky -i 100 --release
-    - name: Run cargo flaky in the auth
-      run: cd meilisearch-auth; cargo flaky -i 100 --release
-    - name: Run cargo flaky in meilisearch
-      run: cd meilisearch; cargo flaky -i 100 --release
+    - name: Run cargo flaky 100 times
+      run: cargo flaky -i 100 --release

.github/workflows/fuzzer-indexing.yml

@@ -1,24 +0,0 @@
-name: Run the indexing fuzzer
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  fuzz:
-    name: Setup the action
-    runs-on: ubuntu-latest
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Run benchmarks
-      - name: Run the fuzzer
-        run: |
-          cargo run --release --bin fuzz-indexing

.github/workflows/publish-binaries.yml

@@ -1,5 +1,3 @@
-name: Publish binaries to GitHub release
-
 on:
   workflow_dispatch:
   schedule:
@@ -7,6 +5,8 @@ on:
   release:
     types: [published]
 
+name: Publish binaries to release
+
 jobs:
   check-version:
     name: Check the version validity
@@ -54,7 +54,7 @@ jobs:
     # No need to upload binaries for dry run (cron)
     - name: Upload binaries to release
       if: github.event_name == 'release'
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.3.0
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/release/meilisearch
@@ -87,7 +87,7 @@ jobs:
     # No need to upload binaries for dry run (cron)
     - name: Upload binaries to release
       if: github.event_name == 'release'
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.3.0
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/release/${{ matrix.artifact_name }}
@@ -96,12 +96,14 @@ jobs:
 
   publish-macos-apple-silicon:
     name: Publish binary for macOS silicon
-    runs-on: macos-12
+    runs-on: ${{ matrix.os }}
     needs: check-version
     strategy:
+      fail-fast: false
       matrix:
         include:
-          - target: aarch64-apple-darwin
+          - os: macos-12
+            target: aarch64-apple-darwin
             asset_name: meilisearch-macos-apple-silicon
     steps:
       - name: Checkout repository
@@ -121,7 +123,7 @@ jobs:
       - name: Upload the binary to release
         # No need to upload binaries for dry run (cron)
         if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.3.0
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/${{ matrix.target }}/release/meilisearch
@@ -130,29 +132,21 @@ jobs:
 
   publish-aarch64:
     name: Publish binary for aarch64
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     needs: check-version
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27
-      image: ubuntu:18.04
     strategy:
+      fail-fast: false
       matrix:
         include:
-          - target: aarch64-unknown-linux-gnu
+          - build: aarch64
+            os: ubuntu-18.04
+            target: aarch64-unknown-linux-gnu
+            linker: gcc-aarch64-linux-gnu
+            use-cross: true
             asset_name: meilisearch-linux-aarch64
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update -y && apt upgrade -y
-          apt-get install -y curl build-essential gcc-aarch64-linux-gnu
-      - name: Set up Docker for cross compilation
-        run: |
-          apt-get install -y curl apt-transport-https ca-certificates software-properties-common
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
-          add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-          apt-get update -y && apt-get install -y docker-ce
       - name: Installing Rust toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -160,7 +154,15 @@ jobs:
           profile: minimal
           target: ${{ matrix.target }}
           override: true
+      - name: APT update
+        run: |
+          sudo apt update
+      - name: Install target specific tools
+        if: matrix.use-cross
+        run: |
+          sudo apt-get install -y ${{ matrix.linker }}
       - name: Configure target aarch64 GNU
+        if: matrix.target == 'aarch64-unknown-linux-gnu'
         ## Environment variable is not passed using env:
         ## LD gold won't work with MUSL
         # env:
@@ -174,16 +176,14 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: build
-          use-cross: true
+          use-cross: ${{ matrix.use-cross }}
           args: --release --target ${{ matrix.target }}
-        env:
-          CROSS_DOCKER_IN_DOCKER: true
       - name: List target output files
         run: ls -lR ./target
       - name: Upload the binary to release
         # No need to upload binaries for dry run (cron)
         if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.3.0
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/${{ matrix.target }}/release/meilisearch

.github/workflows/publish-deb-brew-pkg.yml

@@ -1,4 +1,4 @@
-name: Publish to APT & Homebrew
+name: Publish to APT repository & Homebrew
 
 on:
   release:
@@ -35,7 +35,7 @@ jobs:
     - name: Build deb package
       run: cargo deb -p meilisearch -o target/debian/meilisearch.deb
     - name: Upload debian pkg to release
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.3.0
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/debian/meilisearch.deb
@@ -53,6 +53,5 @@ jobs:
         uses: mislav/bump-homebrew-formula-action@v2
         with:
           formula-name: meilisearch
-          formula-path: Formula/m/meilisearch.rb
         env:
           COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }}

.github/workflows/publish-docker-images.yml

@@ -1,5 +1,4 @@
-name: Publish images to Docker Hub
-
+---
 on:
   push:
     # Will run for every tag pushed except `latest`
@@ -13,6 +12,8 @@ on:
     - cron: '0 23 * * *' # Every day at 11:00pm
   workflow_dispatch:
 
+name: Publish tagged images to Docker Hub
+
 jobs:
   docker:
     runs-on: docker
@@ -51,16 +52,19 @@ jobs:
       - name: Set build-args for Docker buildx
         id: build-metadata
         run: |
+          # Define ownership
+          git config --global --add safe.directory /home/meili/actions-runner/_work/meilisearch/meilisearch
+
           # Extract commit date
           commit_date=$(git show -s --format=%cd --date=iso-strict ${{ github.sha }})
 
           echo "date=$commit_date" >> $GITHUB_OUTPUT
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -70,7 +74,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v4
         with:
           images: getmeili/meilisearch
           # Prevent `latest` to be updated for each new tag pushed.
@@ -83,7 +87,7 @@ jobs:
             type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v3
         with:
           push: true
           platforms: linux/amd64,linux/arm64
@@ -91,7 +95,6 @@ jobs:
           build-args: |
             COMMIT_SHA=${{ github.sha }}
             COMMIT_DATE=${{ steps.build-metadata.outputs.date }}
-            GIT_TAG=${{ github.ref_name }}
 
       # /!\ Don't touch this without checking with Cloud team
       - name: Send CI information to Cloud team

.github/workflows/rust.yml

@@ -0,0 +1,127 @@
+name: Rust
+
+on:
+  workflow_dispatch:
+  pull_request:
+  push:
+    # trying and staging branches are for Bors config
+    branches:
+      - trying
+      - staging
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  RUSTFLAGS: "-D warnings"
+
+jobs:
+  test-linux:
+    name: Tests on ubuntu-18.04
+    runs-on: ubuntu-latest
+    container:
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install needed dependencies
+      run: |
+        apt-get update && apt-get install -y curl
+        apt-get install build-essential -y
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
+    - name: Cache dependencies
+      uses: Swatinem/rust-cache@v2.2.0
+    - name: Run cargo check without any default features
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --locked --release --no-default-features
+    - name: Run cargo test
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --locked --release
+
+  test-others:
+    name: Tests on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-12, windows-2022]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Cache dependencies
+      uses: Swatinem/rust-cache@v2.2.0
+    - name: Run cargo check without any default features
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --locked --release --no-default-features
+    - name: Run cargo test
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --locked --release
+
+  # We run tests in debug also, to make sure that the debug_assertions are hit
+  test-debug:
+    name: Run tests in debug
+    runs-on: ubuntu-latest
+    container:
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install needed dependencies
+        run: |
+          apt-get update && apt-get install -y curl
+          apt-get install build-essential -y
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v2.2.0
+      - name: Run tests in debug
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --locked
+
+  clippy:
+    name: Run Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+          components: clippy
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v2.2.0
+      - name: Run cargo clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          args: --all-targets -- --deny warnings --allow clippy::uninlined_format_args
+
+  fmt:
+    name: Run Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v2.2.0
+      - name: Run cargo fmt
+        run: cargo fmt --all -- --check

.github/workflows/sdks-tests.yml

@@ -1,386 +0,0 @@
-# If any test fails, the engine team should ensure the "breaking" changes are expected and contact the integration team
-name: SDKs tests
-
-on:
-  workflow_dispatch:
-    inputs:
-      docker_image:
-        description: 'The Meilisearch Docker image used'
-        required: false
-        default: nightly
-  schedule:
-    - cron: "0 6 * * MON" # Every Monday at 6:00AM
-
-env:
-  MEILI_MASTER_KEY: 'masterKey'
-  MEILI_NO_ANALYTICS: 'true'
-  DISABLE_COVERAGE: 'true'
-
-jobs:
-  define-docker-image:
-    runs-on: ubuntu-latest
-    outputs:
-      docker-image: ${{ steps.define-image.outputs.docker-image }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Define the Docker image we need to use
-        id: define-image
-        run: |
-          event=${{ github.event_name }}
-          echo "docker-image=nightly" >> $GITHUB_OUTPUT
-          if [[ $event == 'workflow_dispatch' ]]; then
-            echo "docker-image=${{ github.event.inputs.docker_image }}" >> $GITHUB_OUTPUT
-          fi
-      - name: Docker image is ${{ steps.define-image.outputs.docker-image }}
-        run: echo "Docker image is ${{ steps.define-image.outputs.docker-image }}"
-
-##########
-## SDKs ##
-##########
-
-  meilisearch-dotnet-tests:
-    needs: define-docker-image
-    name: .NET SDK tests
-    runs-on: ubuntu-latest
-    env:
-      MEILISEARCH_VERSION: ${{ needs.define-docker-image.outputs.docker-image }}
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-dotnet
-      - name: Setup .NET Core
-        uses: actions/setup-dotnet@v3
-        with:
-          dotnet-version: "6.0.x"
-      - name: Install dependencies
-        run: dotnet restore
-      - name: Build
-        run: dotnet build --configuration Release --no-restore
-      - name: Meilisearch (latest version) setup with Docker
-        run: docker compose up -d
-      - name: Run tests
-        run: dotnet test --no-restore --verbosity normal
-
-  meilisearch-dart-tests:
-    needs: define-docker-image
-    name: Dart SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-dart
-      - uses: dart-lang/setup-dart@v1
-        with:
-          sdk: 3.1.1
-      - name: Install dependencies
-        run: dart pub get
-      - name: Run integration tests
-        run: dart test --concurrency=4
-
-  meilisearch-go-tests:
-    needs: define-docker-image
-    name: Go SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-go
-      - name: Get dependencies
-        run: |
-          go get -v -t -d ./...
-          if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-          fi
-      - name: Run integration tests
-        run: go test -v ./...
-
-  meilisearch-java-tests:
-    needs: define-docker-image
-    name: Java SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-java
-      - name: Set up Java
-        uses: actions/setup-java@v3
-        with:
-          java-version: 8
-          distribution: 'zulu'
-          cache: gradle
-      - name: Grant execute permission for gradlew
-        run: chmod +x gradlew
-      - name: Build and run unit and integration tests
-        run: ./gradlew build integrationTest
-
-  meilisearch-js-tests:
-    needs: define-docker-image
-    name: JS SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-js
-      - name: Setup node
-        uses: actions/setup-node@v3
-        with:
-          cache: 'yarn'
-      - name: Install dependencies
-        run: yarn --dev
-      - name: Run tests
-        run: yarn test
-      - name: Build project
-        run: yarn build
-      - name: Run ESM env
-        run: yarn test:env:esm
-      - name: Run Node.js env
-        run: yarn test:env:nodejs
-      - name: Run node typescript env
-        run: yarn test:env:node-ts
-      - name: Run Browser env
-        run: yarn test:env:browser
-
-  meilisearch-php-tests:
-    needs: define-docker-image
-    name: PHP SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-php
-      - name: Install PHP
-        uses: shivammathur/setup-php@v2
-      - name: Validate composer.json and composer.lock
-        run: composer validate
-      - name: Install dependencies
-        run: |
-          composer remove --dev friendsofphp/php-cs-fixer --no-update --no-interaction
-          composer update --prefer-dist --no-progress
-      - name: Run test suite - default HTTP client (Guzzle 7)
-        run: |
-          sh scripts/tests.sh
-          composer remove --dev guzzlehttp/guzzle http-interop/http-factory-guzzle
-
-  meilisearch-python-tests:
-    needs: define-docker-image
-    name: Python SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-python
-      - name: Set up Python
-        uses: actions/setup-python@v4
-      - name: Install pipenv
-        uses: dschep/install-pipenv-action@v1
-      - name: Install dependencies
-        run: pipenv install --dev --python=${{ matrix.python-version }}
-      - name: Test with pytest
-        run: pipenv run pytest
-
-  meilisearch-ruby-tests:
-    needs: define-docker-image
-    name: Ruby SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-ruby
-      - name: Set up Ruby 3
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3
-      - name: Install ruby dependencies
-        run: bundle install --with test
-      - name: Run test suite
-        run: bundle exec rspec
-
-  meilisearch-rust-tests:
-    needs: define-docker-image
-    name: Rust SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-rust
-      - name: Build
-        run: cargo build --verbose
-      - name: Run tests
-        run: cargo test --verbose
-
-  meilisearch-swift-tests:
-    needs: define-docker-image
-    name: Swift SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-swift
-      - name: Run tests
-        run: swift test
-
-########################
-## FRONT-END PLUGINS ##
-########################
-
-  meilisearch-js-plugins-tests:
-    needs: define-docker-image
-    name: meilisearch-js-plugins tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-js-plugins
-      - name: Setup node
-        uses: actions/setup-node@v3
-        with:
-          cache: yarn
-      - name: Install dependencies
-        run: yarn install
-      - name: Run tests
-        run: yarn test
-      - name: Build all the playgrounds and the packages
-        run: yarn build
-
-########################
-## BACK-END PLUGINS ###
-########################
-
-  meilisearch-rails-tests:
-    needs: define-docker-image
-    name: meilisearch-rails tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-rails
-      - name: Set up Ruby 3
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3
-          bundler-cache: true
-      - name: Run tests
-        run: bundle exec rspec
-
-  meilisearch-symfony-tests:
-    needs: define-docker-image
-    name: meilisearch-symfony tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-symfony
-      - name: Install PHP
-        uses: shivammathur/setup-php@v2
-        with:
-          tools: composer:v2, flex
-      - name: Validate composer.json and composer.lock
-        run: composer validate
-      - name: Install dependencies
-        run: composer install --prefer-dist --no-progress --quiet
-      - name: Remove doctrine/annotations
-        run: composer remove --dev doctrine/annotations
-      - name: Run test suite
-        run: composer test:unit

.github/workflows/test-suite.yml

@@ -1,197 +0,0 @@
-name: Test suite
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Everyday at 5:00am
-    - cron: '0 5 * * *'
-  pull_request:
-  push:
-    # trying and staging branches are for Bors config
-    branches:
-      - trying
-      - staging
-
-env:
-  CARGO_TERM_COLOR: always
-  RUST_BACKTRACE: 1
-  RUSTFLAGS: "-D warnings"
-
-jobs:
-  test-linux:
-    name: Tests on ubuntu-18.04
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update && apt-get install -y curl
-          apt-get install build-essential -y
-      - name: Setup test with Rust stable
-        if: github.event_name != 'schedule'
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-      - name: Setup test with Rust nightly
-        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
-      - name: Run cargo check without any default features
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --locked --release --no-default-features --all
-      - name: Run cargo test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked --release --all
-
-  test-others:
-    name: Tests on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-12, windows-2022]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
-      - name: Run cargo check without any default features
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --locked --release --no-default-features --all
-      - name: Run cargo test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked --release --all
-
-  test-all-features:
-    name: Tests all features
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update
-          apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-      - name: Run cargo build with all features
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --workspace --locked --release --all-features
-      - name: Run cargo test with all features
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace --locked --release --all-features
-
-  test-disabled-tokenization:
-    name: Test disabled tokenization
-    runs-on: ubuntu-latest
-    container:
-      image: ubuntu:18.04
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update
-          apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-      - name: Run cargo tree without default features and check lindera is not present
-        run: |
-          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -vqz lindera; then
-            echo "lindera has been found in the sources and it shouldn't"
-            exit 1
-          fi
-      - name: Run cargo tree with default features and check lindera is pressent
-        run: |
-          cargo tree -f '{p} {f}' -e normal | grep lindera -qz
-
-  # We run tests in debug also, to make sure that the debug_assertions are hit
-  test-debug:
-    name: Run tests in debug
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update && apt-get install -y curl
-          apt-get install build-essential -y
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
-      - name: Run tests in debug
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked --all
-
-  clippy:
-    name: Run Clippy
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: 1.71.1
-          override: true
-          components: clippy
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
-      - name: Run cargo clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all-targets -- --deny warnings
-
-  fmt:
-    name: Run Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly
-          override: true
-          components: rustfmt
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
-      - name: Run cargo fmt
-        # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
-        # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate
-        # we are going to create an empty file where rustfmt expects it.
-        run: |
-          echo -ne "\n" > benchmarks/benches/datasets_paths.rs
-          cargo fmt --all -- --check

.github/workflows/update-cargo-toml-version.yml

@@ -1,4 +1,4 @@
-name: Update Meilisearch version in Cargo.toml
+name: Update Meilisearch version in all Cargo.toml files
 
 on:
   workflow_dispatch:
@@ -13,8 +13,9 @@ env:
   GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
 
 jobs:
+
   update-version-cargo-toml:
-    name: Update version in Cargo.toml
+    name: Update version in Cargo.toml files
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -25,23 +26,23 @@ jobs:
           override: true
       - name: Install sd
         run: cargo install sd
-      - name: Update Cargo.toml file
+      - name: Update Cargo.toml files
         run: |
           raw_new_version=$(echo $NEW_VERSION | cut -d 'v' -f 2)
           new_string="version = \"$raw_new_version\""
-          sd '^version = "\d+.\d+.\w+"$' "$new_string" Cargo.toml
+          sd '^version = "\d+.\d+.\w+"$' "$new_string" */Cargo.toml
       - name: Build Meilisearch to update Cargo.lock
         run: cargo build
       - name: Commit and push the changes to the ${{ env.NEW_BRANCH }} branch
         uses: EndBug/add-and-commit@v9
         with:
-          message: "Update version for the next release (${{ env.NEW_VERSION }}) in Cargo.toml"
+          message: "Update version for the next release (${{ env.NEW_VERSION }}) in Cargo.toml files"
           new_branch: ${{ env.NEW_BRANCH }}
       - name: Create the PR pointing to ${{ github.ref_name }}
         run: |
           gh pr create \
-            --title "Update version for the next release ($NEW_VERSION) in Cargo.toml" \
-            --body '⚠️ This PR is automatically generated. Check the new version is the expected one and Cargo.lock has been updated before merging.' \
+            --title "Update version for the next release ($NEW_VERSION) in Cargo.toml files" \
+            --body '⚠️ This PR is automatically generated. Check the new version is the expected one before merging.' \
             --label 'skip changelog' \
             --milestone $NEW_VERSION \
             --base $GITHUB_REF_NAME

.gitignore

@@ -1,5 +1,3 @@
-.idea/
-.vscode/
 /target
 **/*.csv
 **/*.json_lines
@@ -10,11 +8,9 @@
 /snapshots
 /dumps
 
+
 # Snapshots
 ## ... large
 *.full.snap
 ##  ... unreviewed
 *.snap.new
-
-# Fuzzcheck data for the facet indexing fuzz test
-milli/fuzz/update::facet::incremental::fuzz::fuzz/

CONTRIBUTING.md

@@ -18,9 +18,9 @@ If Meilisearch does not offer optimized support for your language, please consid
 
 ## Assumptions
 
-1. **You're familiar with [GitHub](https://github.com) and the [Pull Requests (PR)](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests) workflow.**
-2. **You've read the Meilisearch [documentation](https://www.meilisearch.com/docs).**
-3. **You know about the [Meilisearch community on Discord](https://discord.meilisearch.com).
+1. **You're familiar with [GitHub](https://github.com) and the [Pull Requests](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)(PR) workflow.**
+2. **You've read the Meilisearch [documentation](https://docs.meilisearch.com).**
+3. **You know about the [Meilisearch community](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html).
    Please use this for help.**
 
 ## How to Contribute
@@ -52,23 +52,6 @@ cargo test
 
 This command will be triggered to each PR as a requirement for merging it.
 
-#### Snapshot-based tests
-
-We are using [insta](https://insta.rs) to perform snapshot-based testing.
-We recommend using the insta tooling (such as `cargo-insta`) to update the snapshots if they change following a PR.
-
-New tests should use insta where possible rather than manual `assert` statements.
-
-Furthermore, we provide some macros on top of insta, notably a way to use snapshot hashes instead of inline snapshots, saving a lot of space in the repository.
-
-To effectively debug snapshot-based hashes, we recommend you export the `MEILI_TEST_FULL_SNAPS` environment variable so that snapshot are fully created locally:
-
-```
-export MEILI_TEST_FULL_SNAPS=true # add this to your .bashrc, .zshrc, ...
-```
-
-#### Test troubleshooting
-
 If you get a "Too many open files" error you might want to increase the open file limit using this command:
 
 ```bash
@@ -116,14 +99,6 @@ _[Read more about this](https://github.com/meilisearch/integration-guides/blob/m
 
 The full Meilisearch release process is described in [this guide](https://github.com/meilisearch/engine-team/blob/main/resources/meilisearch-release.md). Please follow it carefully before doing any release.
 
-### How to publish a prototype
-
-Depending on the developed feature, you might need to provide a prototyped version of Meilisearch to make it easier to test by the users.
-
-This happens in two steps:
-- [Release the prototype](https://github.com/meilisearch/engine-team/blob/main/resources/prototypes.md#how-to-publish-a-prototype)
-- [Communicate about it](https://github.com/meilisearch/engine-team/blob/main/resources/prototypes.md#communication)
-
 ### Release assets
 
 For each release, the following assets are created:

Cargo.toml

@@ -9,51 +9,13 @@ members = [
     "dump",
     "file-store",
     "permissive-json-pointer",
-    "milli",
-    "filter-parser",
-    "flatten-serde-json",
-    "json-depth-checker",
-    "benchmarks",
-    "fuzzers",
 ]
 
-[workspace.package]
-version = "1.4.1"
-authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
-description = "Meilisearch HTTP server"
-homepage = "https://meilisearch.com"
-readme = "README.md"
-edition = "2021"
-license = "MIT"
-
 [profile.release]
 codegen-units = 1
 
 [profile.dev.package.flate2]
 opt-level = 3
 
-[profile.dev.package.grenad]
-opt-level = 3
-[profile.dev.package.roaring]
-opt-level = 3
-
-[profile.dev.package.lindera-ipadic-builder]
-opt-level = 3
-[profile.dev.package.encoding]
-opt-level = 3
-[profile.dev.package.yada]
-opt-level = 3
-
-[profile.release.package.lindera-ipadic-builder]
-opt-level = 3
-[profile.release.package.encoding]
-opt-level = 3
-[profile.release.package.yada]
-opt-level = 3
-
-[profile.bench.package.lindera-ipadic-builder]
-opt-level = 3
-[profile.bench.package.encoding]
-opt-level = 3
-[profile.bench.package.yada]
+[profile.dev.package.milli]
 opt-level = 3

Dockerfile

@@ -7,8 +7,7 @@ WORKDIR /meilisearch
 
 ARG     COMMIT_SHA
 ARG     COMMIT_DATE
-ARG     GIT_TAG
-ENV     VERGEN_GIT_SHA=${COMMIT_SHA} VERGEN_GIT_COMMIT_TIMESTAMP=${COMMIT_DATE} VERGEN_GIT_SEMVER_LIGHTWEIGHT=${GIT_TAG}
+ENV     VERGEN_GIT_SHA=${COMMIT_SHA} VERGEN_GIT_COMMIT_TIMESTAMP=${COMMIT_DATE}
 ENV     RUSTFLAGS="-C target-feature=-crt-static"
 
 COPY    . .

PROFILING.md

@@ -1,19 +0,0 @@
-# Profiling Meilisearch
-
-Search engine technologies are complex pieces of software that require thorough profiling tools. We chose to use [Puffin](https://github.com/EmbarkStudios/puffin), which the Rust gaming industry uses extensively. You can export and import the profiling reports using the top bar's _File_ menu options [in Puffin Viewer](https://github.com/embarkstudios/puffin#ui).
-
-![An example profiling with Puffin viewer](assets/profiling-example.png)
-
-## Profiling the Indexing Process
-
-When you enable [the `exportPuffinReports` experimental feature](https://www.meilisearch.com/docs/learn/experimental/overview) of Meilisearch, Puffin reports with the `.puffin` extension will be automatically exported to disk. When this option is enabled, the engine will automatically create a "frame" whenever it executes the `IndexScheduler::tick` method.
-
-[Puffin Viewer](https://github.com/EmbarkStudios/puffin/tree/main/puffin_viewer) is used to analyze the reports. Those reports show areas where Meilisearch spent time during indexing.
-
-Another piece of advice on the Puffin viewer UI interface is to consider the _Merge children with same ID_ option. It can hide the exact actual timings at which events were sent. Please turn it off when you see strange gaps on the Flamegraph. It can help.
-
-## Profiling the Search Process
-
-We still need to take the time to profile the search side of the engine with Puffin. It would require time to profile the filtering phase, query parsing, creation, and execution. We could even profile the Actix HTTP server.
-
-The only issue we see is the framing system. Puffin requires a global frame-based profiling phase, which collides with Meilisearch's ability to accept and answer multiple requests on different threads simultaneously.

README.md

@@ -1,105 +1,95 @@
 <p align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-light-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
-  </a>
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-dark-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
-  </a>
+  <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
+  <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
 </p>
 
 <h4 align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Website</a> |
+  <a href="https://www.meilisearch.com">Website</a> |
   <a href="https://roadmap.meilisearch.com/tabs/1-under-consideration">Roadmap</a> |
-  <a href="https://www.meilisearch.com/pricing?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Meilisearch Cloud</a> |
-  <a href="https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Blog</a> |
-  <a href="https://www.meilisearch.com/docs?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Documentation</a> |
-  <a href="https://www.meilisearch.com/docs/faq?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">FAQ</a> |
-  <a href="https://discord.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Discord</a>
+  <a href="https://blog.meilisearch.com">Blog</a> |
+  <a href="https://docs.meilisearch.com">Documentation</a> |
+  <a href="https://docs.meilisearch.com/faq/">FAQ</a> |
+  <a href="https://discord.meilisearch.com">Discord</a>
 </h4>
 
 <p align="center">
+  <a href="https://github.com/meilisearch/meilisearch/actions"><img src="https://github.com/meilisearch/meilisearch/workflows/Cargo%20test/badge.svg" alt="Build Status"></a>
   <a href="https://deps.rs/repo/github/meilisearch/meilisearch"><img src="https://deps.rs/repo/github/meilisearch/meilisearch/status.svg" alt="Dependency status"></a>
   <a href="https://github.com/meilisearch/meilisearch/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
-  <a href="https://ms-bors.herokuapp.com/repositories/52"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
+  <a href="https://app.bors.tech/repositories/26457"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
 </p>
 
 <p align="center">⚡ A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow 🔍</p>
 
----
-
-### 🔥 On November 2nd, we are hosting our first-ever live demo and product updates for [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). Make sure to [register here](https://us06web.zoom.us/meeting/register/tZMlc-mqrjIsH912-HTRe-AaT-pp41bDe81a#/registration) and bring your questions for live Q&A!
-
----
-
 Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow.
 
 <p align="center" name="demo">
-  <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-light-mode-only" target="_blank">
+  <a href="https://where2watch.meilisearch.com/#gh-light-mode-only" target="_blank">
     <img src="assets/demo-light.gif#gh-light-mode-only" alt="A bright colored application for finding movies screening near the user">
   </a>
-  <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-dark-mode-only" target="_blank">
+  <a href="https://where2watch.meilisearch.com/#gh-dark-mode-only" target="_blank">
     <img src="assets/demo-dark.gif#gh-dark-mode-only" alt="A dark colored application for finding movies screening near the user">
   </a>
 </p>
 
-🔥 [**Try it!**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-link) 🔥
+🔥 [**Try it!**](https://where2watch.meilisearch.com/) 🔥
 
 ## ✨ Features
 
 - **Search-as-you-type:** find search results in less than 50 milliseconds
-- **[Typo tolerance](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#typo-tolerance):** get relevant matches even when queries contain typos and misspellings
-- **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code
-- **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** sort results based on price, date, or pretty much anything else your users need
-- **[Synonym support](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#synonyms):** configure synonyms to include more relevant content in your search results
-- **[Geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** filter and sort documents based on geographic data
-- **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
-- **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** control which users can access what data with API keys that allow fine-grained permissions handling
-- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** personalize search results for any number of application tenants
+- **[Typo tolerance](https://docs.meilisearch.com/learn/getting_started/customizing_relevancy.html#typo-tolerance):** get relevant matches even when queries contain typos and misspellings
+- **[Filtering and faceted search](https://docs.meilisearch.com/learn/advanced/filtering_and_faceted_search.html):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code
+- **[Sorting](https://docs.meilisearch.com/learn/advanced/sorting.html):** sort results based on price, date, or pretty much anything else your users need
+- **[Synonym support](https://docs.meilisearch.com/learn/getting_started/customizing_relevancy.html#synonyms):** configure synonyms to include more relevant content in your search results
+- **[Geosearch](https://docs.meilisearch.com/learn/advanced/geosearch.html):** filter and sort documents based on geographic data
+- **[Extensive language support](https://docs.meilisearch.com/learn/what_is_meilisearch/language.html):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
+- **[Security management](https://docs.meilisearch.com/learn/security/master_api_keys.html):** control which users can access what data with API keys that allow fine-grained permissions handling
+- **[Multi-Tenancy](https://docs.meilisearch.com/learn/security/tenant_tokens.html):** personalize search results for any number of application tenants
 - **Highly Customizable:** customize Meilisearch to your specific needs or use our out-of-the-box and hassle-free presets
-- **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** integrate Meilisearch in your technical stack with our plugins and SDKs
+- **[RESTful API](https://docs.meilisearch.com/reference/api/overview.html):** integrate Meilisearch in your technical stack with our plugins and SDKs
 - **Easy to install, deploy, and maintain**
 
 ## 📖 Documentation
 
-You can consult Meilisearch's documentation at [https://www.meilisearch.com/docs](https://www.meilisearch.com/docs/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=docs).
+You can consult Meilisearch's documentation at [https://docs.meilisearch.com](https://docs.meilisearch.com/).
 
 ## 🚀 Getting started
 
-For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
+For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://docs.meilisearch.com/learn/getting_started/quick_start.html) guide.
 
-You may also want to check out [Meilisearch 101](https://www.meilisearch.com/docs/learn/getting_started/filtering_and_sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) for an introduction to some of Meilisearch's most popular features.
+You may also want to check out [Meilisearch 101](https://docs.meilisearch.com/learn/getting_started/filtering_and_sorting.html) for an introduction to some of Meilisearch's most popular features.
 
-## ⚡ Supercharge your Meilisearch experience
+## ☁️ Meilisearch cloud
 
-Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). No credit card required.
+Let us manage your infrastructure so you can focus on integrating a great search experience. Try [Meilisearch Cloud](https://meilisearch.com/pricing) today.
 
 ## 🧰 SDKs & integration tools
 
 Install one of our SDKs in your project for seamless integration between Meilisearch and your favorite language or framework!
 
-Take a look at the complete [Meilisearch integration list](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-link).
+Take a look at the complete [Meilisearch integration list](https://docs.meilisearch.com/learn/what_is_meilisearch/sdks.html).
 
-[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-logos)
+![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)
 
 ## ⚙️ Advanced usage
 
-Experienced users will want to keep our [API Reference](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) close at hand.
+Experienced users will want to keep our [API Reference](https://docs.meilisearch.com/reference/api) close at hand.
 
-We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [API keys](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), and [tenant tokens](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
+We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://docs.meilisearch.com/learn/advanced/filtering_and_faceted_search.html), [sorting](https://docs.meilisearch.com/learn/advanced/sorting.html), [geosearch](https://docs.meilisearch.com/learn/advanced/geosearch.html), [API keys](https://docs.meilisearch.com/learn/security/master_api_keys.html), and [tenant tokens](https://docs.meilisearch.com/learn/security/tenant_tokens.html).
 
-Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
+Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://docs.meilisearch.com/learn/core_concepts/documents.html) and [indexes](https://docs.meilisearch.com/learn/core_concepts/indexes.html).
 
 ## 📊 Telemetry
 
-Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
+Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html#how-to-disable-data-collection) whenever you want.
 
-To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
+To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
 
-If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) of our documentation.
+If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html) of our documentation.
 
 ## 📫 Get in touch!
 
-Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
+Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/)
 
 🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
 
@@ -108,17 +98,6 @@ Meilisearch is a search engine created by [Meili](https://www.welcometothejungle
 - For feature requests, please visit our [product repository](https://github.com/meilisearch/product/discussions)
 - Found a bug? Open an [issue](https://github.com/meilisearch/meilisearch/issues)!
 - Want to be part of our Discord community? [Join us!](https://discord.gg/meilisearch)
+- For everything else, please check [this page listing some of the other places where you can find us](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html)
 
 Thank you for your support!
-
-## 👩‍💻 Contributing
-
-Meilisearch is, and will always be, open-source! If you want to contribute to the project, please take a look at [our contribution guidelines](CONTRIBUTING.md).
-
-## 📦 Versioning
-
-Meilisearch releases and their associated binaries are available [in this GitHub page](https://github.com/meilisearch/meilisearch/releases).
-
-The binaries are versioned following [SemVer conventions](https://semver.org/). To know more, read our [versioning policy](https://github.com/meilisearch/engine-team/blob/main/resources/versioning-policy.md).
-
-Differently from the binaries, crates in this repository are not currently available on [crates.io](https://crates.io/) and do not follow [SemVer conventions](https://semver.org).

assets/milli-logo.svg

@@ -1,6 +0,0 @@
-<svg width="277" height="236" viewBox="0 0 277 236" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M213.085 190L242.907 86H276.196L246.375 190H213.085Z" fill="#494949"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M0 190L29.8215 86H63.1111L33.2896 190H0Z" fill="#494949"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M124.986 0L57.5772 235.083L60.7752 236H90.6038L158.276 0H124.986Z" fill="#494949"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M195.273 0L127.601 236H160.891L228.563 0H195.273Z" fill="#494949"/>
-</svg>

assets/prometheus-basic-scraper.yml

@@ -1,19 +0,0 @@
-global:
-  scrape_interval:     15s # By default, scrape targets every 15 seconds.
-
-  # Attach these labels to any time series or alerts when communicating with
-  # external systems (federation, remote storage, Alertmanager).
-  external_labels:
-    monitor: 'codelab-monitor'
-
-# A scrape configuration containing exactly one endpoint to scrape:
-# Here it's Prometheus itself.
-scrape_configs:
-  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
-  - job_name: 'meilisearch'
-
-    # Override the global default and scrape targets from this job every 5 seconds.
-    scrape_interval: 5s
-
-    static_configs:
-      - targets: ['localhost:7700']

benchmarks/.gitignore

@@ -1 +0,0 @@
-benches/datasets_paths.rs

benchmarks/Cargo.toml

@@ -1,50 +0,0 @@
-[package]
-name = "benchmarks"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
-[dependencies]
-anyhow = "1.0.70"
-csv = "1.2.1"
-milli = { path = "../milli" }
-mimalloc = { version = "0.1.37", default-features = false }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
-
-[dev-dependencies]
-criterion = { version = "0.5.1", features = ["html_reports"] }
-rand = "0.8.5"
-rand_chacha = "0.3.1"
-roaring = "0.10.1"
-
-[build-dependencies]
-anyhow = "1.0.70"
-bytes = "1.4.0"
-convert_case = "0.6.0"
-flate2 = "1.0.25"
-reqwest = { version = "0.11.16", features = ["blocking", "rustls-tls"], default-features = false }
-
-[features]
-default = ["milli/all-tokenizations"]
-
-[[bench]]
-name = "search_songs"
-harness = false
-
-[[bench]]
-name = "search_wiki"
-harness = false
-
-[[bench]]
-name = "search_geo"
-harness = false
-
-[[bench]]
-name = "indexing"
-harness = false

benchmarks/README.md

@@ -1,138 +0,0 @@
-Benchmarks
-==========
-
-## TOC
-
-- [Run the benchmarks](#run-the-benchmarks)
-- [Comparison between benchmarks](#comparison-between-benchmarks)
-- [Datasets](#datasets)
-
-## Run the benchmarks
-
-### On our private server
-
-The Meili team has self-hosted his own GitHub runner to run benchmarks on our dedicated bare metal server.
-
-To trigger the benchmark workflow:
-- Go to the `Actions` tab of this repository.
-- Select the `Benchmarks` workflow on the left.
-- Click on `Run workflow` in the blue banner.
-- Select the branch on which you want to run the benchmarks and select the dataset you want (default: `songs`).
-- Finally, click on `Run workflow`.
-
-This GitHub workflow will run the benchmarks and push the `critcmp` report to a DigitalOcean Space (= S3).
-
-The name of the uploaded file is displayed in the workflow.
-
-_[More about critcmp](https://github.com/BurntSushi/critcmp)._
-
-💡 To compare the just-uploaded benchmark with another one, check out the [next section](#comparison-between-benchmarks).
-
-### On your machine
-
-To run all the benchmarks (~5h):
-
-```bash
-cargo bench
-```
-
-To run only the `search_songs` (~1h), `search_wiki` (~3h), `search_geo` (~20m) or `indexing` (~2h) benchmark:
-
-```bash
-cargo bench --bench <dataset name>
-```
-
-By default, the benchmarks will be downloaded and uncompressed automatically in the target directory.<br>
-If you don't want to download the datasets every time you update something on the code, you can specify a custom directory with the environment variable `MILLI_BENCH_DATASETS_PATH`:
-
-```bash
-mkdir ~/datasets
-MILLI_BENCH_DATASETS_PATH=~/datasets cargo bench --bench search_songs # the four datasets are downloaded
-touch build.rs
-MILLI_BENCH_DATASETS_PATH=~/datasets cargo bench --bench songs # the code is compiled again but the datasets are not downloaded
-```
-
-## Comparison between benchmarks
-
-The benchmark reports we push are generated with `critcmp`. Thus, we use `critcmp` to show the result of a benchmark, or compare results between multiple benchmarks.
-
-We provide a script to download and display the comparison report.
-
-Requirements:
-- `grep`
-- `curl`
-- [`critcmp`](https://github.com/BurntSushi/critcmp)
-
-List the available file in the DO Space:
-
-```bash
-./benchmarks/script/list.sh
-```
-```bash
-songs_main_09a4321.json
-songs_geosearch_24ec456.json
-search_songs_main_cb45a10b.json
-```
-
-Run the comparison script:
-
-```bash
-# we get the result of ONE benchmark, this give you an idea of how much time an operation took
-./benchmarks/scripts/compare.sh son songs_geosearch_24ec456.json
-# we compare two benchmarks
-./benchmarks/scripts/compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json
-# we compare three benchmarks
-./benchmarks/scripts/compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json search_songs_main_cb45a10b.json
-```
-
-## Datasets
-
-The benchmarks uses the following datasets:
-- `smol-songs`
-- `smol-wiki`
-- `movies`
-- `smol-all-countries`
-
-### Songs
-
-`smol-songs` is a subset of the [`songs.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/songs.csv.gz).
-
-It was generated with this command:
-
-```bash
-xsv sample --seed 42 1000000 songs.csv -o smol-songs.csv
-```
-
-_[Download the generated `smol-songs` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-songs.csv.gz)._
-
-### Wiki
-
-`smol-wiki` is a subset of the [`wikipedia-articles.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/wiki-articles.csv.gz).
-
-It was generated with the following command:
-
-```bash
-xsv sample --seed 42 500000 wiki-articles.csv -o smol-wiki-articles.csv
-```
-
-_[Download the `smol-wiki` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-wiki-articles.csv.gz)._
-
-### Movies
-
-`movies` is a really small dataset we uses as our example in the [getting started](https://www.meilisearch.com/docs/learn/getting_started/quick_start)
-
-_[Download the `movies` dataset](https://www.meilisearch.com/movies.json)._
-
-
-### All Countries
-
-`smol-all-countries` is a subset of the [`all-countries.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/all-countries.csv.gz)
-It has been converted to jsonlines and then edited so it matches our format for the `_geo` field.
-
-It was generated with the following command:
-```bash
-bat all-countries.csv.gz | gunzip | xsv sample --seed 42 1000000 | csv2json-lite | sd '"latitude":"(.*?)","longitude":"(.*?)"' '"_geo": { "lat": $1, "lng": $2 }' | sd '\[|\]|,$' '' | gzip > smol-all-countries.jsonl.gz
-```
-
-_[Download the `smol-all-countries` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-all-countries.jsonl.gz)._
-

benchmarks/benches/search_geo.rs

@@ -1,122 +0,0 @@
-mod datasets_paths;
-mod utils;
-
-use criterion::{criterion_group, criterion_main};
-use milli::update::Settings;
-use utils::Conf;
-
-#[global_allocator]
-static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
-
-fn base_conf(builder: &mut Settings) {
-    let displayed_fields =
-        ["geonameid", "name", "asciiname", "alternatenames", "_geo", "population"]
-            .iter()
-            .map(|s| s.to_string())
-            .collect();
-    builder.set_displayed_fields(displayed_fields);
-
-    let searchable_fields =
-        ["name", "alternatenames", "elevation"].iter().map(|s| s.to_string()).collect();
-    builder.set_searchable_fields(searchable_fields);
-
-    let filterable_fields =
-        ["_geo", "population", "elevation"].iter().map(|s| s.to_string()).collect();
-    builder.set_filterable_fields(filterable_fields);
-
-    let sortable_fields =
-        ["_geo", "population", "elevation"].iter().map(|s| s.to_string()).collect();
-    builder.set_sortable_fields(sortable_fields);
-}
-
-#[rustfmt::skip]
-const BASE_CONF: Conf = Conf {
-    dataset: datasets_paths::SMOL_ALL_COUNTRIES,
-    dataset_format: "jsonl",
-    queries: &[
-        "",
-    ],
-    configure: base_conf,
-    primary_key: Some("geonameid"),
-    ..Conf::BASE
-};
-
-fn bench_geo(c: &mut criterion::Criterion) {
-    #[rustfmt::skip]
-    let confs = &[
-        // A basic placeholder with no geo
-        utils::Conf {
-            group_name: "placeholder with no geo",
-            ..BASE_CONF
-        },
-        // Medium aglomeration: probably the most common usecase
-        utils::Conf {
-            group_name: "asc sort from Lille",
-            sort: Some(vec!["_geoPoint(50.62999333378238, 3.086269263384099):asc"]),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "desc sort from Lille",
-            sort: Some(vec!["_geoPoint(50.62999333378238, 3.086269263384099):desc"]),
-            ..BASE_CONF
-        },
-        // Big agglomeration: a lot of documents close to our point
-        utils::Conf {
-            group_name: "asc sort from Tokyo",
-            sort: Some(vec!["_geoPoint(35.749512532692144, 139.61664952543356):asc"]),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "desc sort from Tokyo",
-            sort: Some(vec!["_geoPoint(35.749512532692144, 139.61664952543356):desc"]),
-            ..BASE_CONF
-        },
-        // The furthest point from any civilization
-        utils::Conf {
-            group_name: "asc sort from Point Nemo",
-            sort: Some(vec!["_geoPoint(-48.87561645055408, -123.39275749319793):asc"]),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "desc sort from Point Nemo",
-            sort: Some(vec!["_geoPoint(-48.87561645055408, -123.39275749319793):desc"]),
-            ..BASE_CONF
-        },
-        // Filters
-        utils::Conf {
-            group_name: "filter of 100km from Lille",
-            filter: Some("_geoRadius(50.62999333378238, 3.086269263384099, 100000)"),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "filter of 1km from Lille",
-            filter: Some("_geoRadius(50.62999333378238, 3.086269263384099, 1000)"),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "filter of 100km from Tokyo",
-            filter: Some("_geoRadius(35.749512532692144, 139.61664952543356, 100000)"),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "filter of 1km from Tokyo",
-            filter: Some("_geoRadius(35.749512532692144, 139.61664952543356, 1000)"),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "filter of 100km from Point Nemo",
-            filter: Some("_geoRadius(-48.87561645055408, -123.39275749319793, 100000)"),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "filter of 1km from Point Nemo",
-            filter: Some("_geoRadius(-48.87561645055408, -123.39275749319793, 1000)"),
-            ..BASE_CONF
-        },
-    ];
-
-    utils::run_benches(c, confs);
-}
-
-criterion_group!(benches, bench_geo);
-criterion_main!(benches);

benchmarks/benches/search_songs.rs

@@ -1,196 +0,0 @@
-mod datasets_paths;
-mod utils;
-
-use criterion::{criterion_group, criterion_main};
-use milli::update::Settings;
-use utils::Conf;
-
-#[global_allocator]
-static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
-
-fn base_conf(builder: &mut Settings) {
-    let displayed_fields =
-        ["id", "title", "album", "artist", "genre", "country", "released", "duration"]
-            .iter()
-            .map(|s| s.to_string())
-            .collect();
-    builder.set_displayed_fields(displayed_fields);
-
-    let searchable_fields = ["title", "album", "artist"].iter().map(|s| s.to_string()).collect();
-    builder.set_searchable_fields(searchable_fields);
-
-    let faceted_fields = ["released-timestamp", "duration-float", "genre", "country", "artist"]
-        .iter()
-        .map(|s| s.to_string())
-        .collect();
-    builder.set_filterable_fields(faceted_fields);
-}
-
-#[rustfmt::skip]
-const BASE_CONF: Conf = Conf {
-    dataset: datasets_paths::SMOL_SONGS,
-    queries: &[
-        "john ",             // 9097
-        "david ",            // 4794
-        "charles ",          // 1957
-        "david bowie ",      // 1200
-        "michael jackson ",  // 600
-        "thelonious monk ",  // 303
-        "charles mingus ",   // 142
-        "marcus miller ",    // 60
-        "tamo ",             // 13
-        "Notstandskomitee ", // 4
-    ],
-    configure: base_conf,
-    primary_key: Some("id"),
-    ..Conf::BASE
-};
-
-fn bench_songs(c: &mut criterion::Criterion) {
-    let default_criterion: Vec<String> =
-        milli::default_criteria().iter().map(|criteria| criteria.to_string()).collect();
-    let default_criterion = default_criterion.iter().map(|s| s.as_str());
-    let asc_default: Vec<&str> =
-        std::iter::once("released-timestamp:asc").chain(default_criterion.clone()).collect();
-    let desc_default: Vec<&str> =
-        std::iter::once("released-timestamp:desc").chain(default_criterion.clone()).collect();
-
-    let basic_with_quote: Vec<String> = BASE_CONF
-        .queries
-        .iter()
-        .map(|s| {
-            s.trim().split(' ').map(|s| format!(r#""{}""#, s)).collect::<Vec<String>>().join(" ")
-        })
-        .collect();
-    let basic_with_quote: &[&str] =
-        &basic_with_quote.iter().map(|s| s.as_str()).collect::<Vec<&str>>();
-
-    #[rustfmt::skip]
-    let confs = &[
-        /* first we bench each criterion alone */
-        utils::Conf {
-            group_name: "proximity",
-            queries: &[
-                "black saint sinner lady ",
-                "les dangeureuses 1960 ",
-                "The Disneyland Sing-Along Chorus ",
-                "Under Great Northern Lights ",
-                "7000 Danses Un Jour Dans Notre Vie ",
-            ],
-            criterion: Some(&["proximity"]),
-            optional_words: false,
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "typo",
-            queries: &[
-                "mongus ",
-                "thelonius monk ",
-                "Disnaylande ",
-                "the white striper ",
-                "indochie ",
-                "indochien ",
-                "klub des loopers ",
-                "fear of the duck ",
-                "michel depech ",
-                "stromal ",
-                "dire straights ",
-                "Arethla Franklin ",
-            ],
-            criterion: Some(&["typo"]),
-            optional_words: false,
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "words",
-            queries: &[
-                "the black saint and the sinner lady and the good doggo ", // four words to pop
-                "les liaisons dangeureuses 1793 ",                         // one word to pop
-                "The Disneyland Children's Sing-Alone song ",              // two words to pop
-                "seven nation mummy ",                                     // one word to pop
-                "7000 Danses / Le Baiser / je me trompe de mots ",         // four words to pop
-                "Bring Your Daughter To The Slaughter but now this is not part of the title ", // nine words to pop
-                "whathavenotnsuchforth and a good amount of words to pop to match the first one ", // 13
-            ],
-            criterion: Some(&["words"]),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "asc",
-            criterion: Some(&["released-timestamp:desc"]),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "desc",
-            criterion: Some(&["released-timestamp:desc"]),
-            ..BASE_CONF
-        },
-
-        /* then we bench the asc and desc criterion on top of the default criterion */
-        utils::Conf {
-            group_name: "asc + default",
-            criterion: Some(&asc_default[..]),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "desc + default",
-            criterion: Some(&desc_default[..]),
-            ..BASE_CONF
-        },
-
-        /* we bench the filters with the default request */
-        utils::Conf {
-            group_name: "basic filter: <=",
-            filter: Some("released-timestamp <= 946728000"), // year 2000
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "basic filter: TO",
-            filter: Some("released-timestamp 946728000 TO 1262347200"), // year 2000 to 2010
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "big filter",
-            filter: Some("released-timestamp != 1262347200 AND (NOT (released-timestamp = 946728000)) AND (duration-float = 1 OR (duration-float 1.1 TO 1.5 AND released-timestamp > 315576000))"),
-            ..BASE_CONF
-        },
-
-        /* the we bench some global / normal search with all the default criterion in the default
-         * order */
-        utils::Conf {
-            group_name: "basic placeholder",
-            queries: &[""],
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "basic without quote",
-            queries: &BASE_CONF
-                .queries
-                .iter()
-                .map(|s| s.trim()) // we remove the space at the end of each request
-                .collect::<Vec<&str>>(),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "basic with quote",
-            queries: basic_with_quote,
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "prefix search",
-            queries: &[
-                "s", // 500k+ results
-                "a", //
-                "b", //
-                "i", //
-                "x", // only 7k results
-            ],
-            ..BASE_CONF
-        },
-    ];
-
-    utils::run_benches(c, confs);
-}
-
-criterion_group!(benches, bench_songs);
-criterion_main!(benches);

benchmarks/benches/search_wiki.rs

@@ -1,129 +0,0 @@
-mod datasets_paths;
-mod utils;
-
-use criterion::{criterion_group, criterion_main};
-use milli::update::Settings;
-use utils::Conf;
-
-#[global_allocator]
-static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
-
-fn base_conf(builder: &mut Settings) {
-    let displayed_fields = ["title", "body", "url"].iter().map(|s| s.to_string()).collect();
-    builder.set_displayed_fields(displayed_fields);
-
-    let searchable_fields = ["title", "body"].iter().map(|s| s.to_string()).collect();
-    builder.set_searchable_fields(searchable_fields);
-}
-
-#[rustfmt::skip]
-const BASE_CONF: Conf = Conf {
-    dataset: datasets_paths::SMOL_WIKI_ARTICLES,
-    queries: &[
-        "mingus ",        // 46 candidates
-        "miles davis ",   // 159
-        "rock and roll ", // 1007
-        "machine ",       // 3448
-        "spain ",         // 7002
-        "japan ",         // 10.593
-        "france ",        // 17.616
-        "film ",          // 24.959
-    ],
-    configure: base_conf,
-    ..Conf::BASE
-};
-
-fn bench_songs(c: &mut criterion::Criterion) {
-    let basic_with_quote: Vec<String> = BASE_CONF
-        .queries
-        .iter()
-        .map(|s| {
-            s.trim().split(' ').map(|s| format!(r#""{}""#, s)).collect::<Vec<String>>().join(" ")
-        })
-        .collect();
-    let basic_with_quote: &[&str] =
-        &basic_with_quote.iter().map(|s| s.as_str()).collect::<Vec<&str>>();
-
-    #[rustfmt::skip]
-    let confs = &[
-        /* first we bench each criterion alone */
-        utils::Conf {
-            group_name: "proximity",
-            queries: &[
-                "herald sings ",
-                "april paris ",
-                "tea two ",
-                "diesel engine ",
-            ],
-            criterion: Some(&["proximity"]),
-            optional_words: false,
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "typo",
-            queries: &[
-                "migrosoft ",
-                "linax ",
-                "Disnaylande ",
-                "phytogropher ",
-                "nympalidea ",
-                "aritmetric ",
-                "the fronce ",
-                "sisan ",
-            ],
-            criterion: Some(&["typo"]),
-            optional_words: false,
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "words",
-            queries: &[
-                "the black saint and the sinner lady and the good doggo ", // four words to pop, 27 results
-                "Kameya Tokujirō mingus monk ",                            // two words to pop, 55
-                "Ulrich Hensel meilisearch milli ",                        // two words to pop, 306
-                "Idaho Bellevue pizza ",                                   // one word to pop, 800
-                "Abraham machin ",                                         // one word to pop, 1141
-            ],
-            criterion: Some(&["words"]),
-            ..BASE_CONF
-        },
-        /* the we bench some global / normal search with all the default criterion in the default
-         * order */
-        utils::Conf {
-            group_name: "basic placeholder",
-            queries: &[""],
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "basic without quote",
-            queries: &BASE_CONF
-                .queries
-                .iter()
-                .map(|s| s.trim()) // we remove the space at the end of each request
-                .collect::<Vec<&str>>(),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "basic with quote",
-            queries: basic_with_quote,
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "prefix search",
-            queries: &[
-                "t", // 453k results
-                "c", // 405k
-                "g", // 318k
-                "j", // 227k
-                "q", // 71k
-                "x", // 17k
-            ],
-            ..BASE_CONF
-        },
-    ];
-
-    utils::run_benches(c, confs);
-}
-
-criterion_group!(benches, bench_songs);
-criterion_main!(benches);

benchmarks/benches/utils.rs

@@ -1,256 +0,0 @@
-#![allow(dead_code)]
-
-use std::fs::{create_dir_all, remove_dir_all, File};
-use std::io::{self, BufRead, BufReader, Cursor, Read, Seek};
-use std::num::ParseFloatError;
-use std::path::Path;
-use std::str::FromStr;
-
-use criterion::BenchmarkId;
-use milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
-use milli::heed::EnvOpenOptions;
-use milli::update::{
-    IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Settings,
-};
-use milli::{Criterion, Filter, Index, Object, TermsMatchingStrategy};
-use serde_json::Value;
-
-pub struct Conf<'a> {
-    /// where we are going to create our database.mmdb directory
-    /// each benchmark will first try to delete it and then recreate it
-    pub database_name: &'a str,
-    /// the dataset to be used, it must be an uncompressed csv
-    pub dataset: &'a str,
-    /// The format of the dataset
-    pub dataset_format: &'a str,
-    pub group_name: &'a str,
-    pub queries: &'a [&'a str],
-    /// here you can change which criterion are used and in which order.
-    /// - if you specify something all the base configuration will be thrown out
-    /// - if you don't specify anything (None) the default configuration will be kept
-    pub criterion: Option<&'a [&'a str]>,
-    /// the last chance to configure your database as you want
-    pub configure: fn(&mut Settings),
-    pub filter: Option<&'a str>,
-    pub sort: Option<Vec<&'a str>>,
-    /// enable or disable the optional words on the query
-    pub optional_words: bool,
-    /// primary key, if there is None we'll auto-generate docids for every documents
-    pub primary_key: Option<&'a str>,
-}
-
-impl Conf<'_> {
-    pub const BASE: Self = Conf {
-        database_name: "benches.mmdb",
-        dataset_format: "csv",
-        dataset: "",
-        group_name: "",
-        queries: &[],
-        criterion: None,
-        configure: |_| (),
-        filter: None,
-        sort: None,
-        optional_words: true,
-        primary_key: None,
-    };
-}
-
-pub fn base_setup(conf: &Conf) -> Index {
-    match remove_dir_all(conf.database_name) {
-        Ok(_) => (),
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => (),
-        Err(e) => panic!("{}", e),
-    }
-    create_dir_all(conf.database_name).unwrap();
-
-    let mut options = EnvOpenOptions::new();
-    options.map_size(100 * 1024 * 1024 * 1024); // 100 GB
-    options.max_readers(10);
-    let index = Index::new(options, conf.database_name).unwrap();
-
-    let config = IndexerConfig::default();
-    let mut wtxn = index.write_txn().unwrap();
-    let mut builder = Settings::new(&mut wtxn, &index, &config);
-
-    if let Some(primary_key) = conf.primary_key {
-        builder.set_primary_key(primary_key.to_string());
-    }
-
-    if let Some(criterion) = conf.criterion {
-        builder.reset_filterable_fields();
-        builder.reset_criteria();
-        builder.reset_stop_words();
-
-        let criterion = criterion.iter().map(|s| Criterion::from_str(s).unwrap()).collect();
-        builder.set_criteria(criterion);
-    }
-
-    (conf.configure)(&mut builder);
-
-    builder.execute(|_| (), || false).unwrap();
-    wtxn.commit().unwrap();
-
-    let config = IndexerConfig::default();
-    let mut wtxn = index.write_txn().unwrap();
-    let indexing_config = IndexDocumentsConfig {
-        autogenerate_docids: conf.primary_key.is_none(),
-        update_method: IndexDocumentsMethod::ReplaceDocuments,
-        ..Default::default()
-    };
-    let builder =
-        IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| (), || false).unwrap();
-    let documents = documents_from(conf.dataset, conf.dataset_format);
-    let (builder, user_error) = builder.add_documents(documents).unwrap();
-    user_error.unwrap();
-    builder.execute().unwrap();
-    wtxn.commit().unwrap();
-
-    index
-}
-
-pub fn run_benches(c: &mut criterion::Criterion, confs: &[Conf]) {
-    for conf in confs {
-        let index = base_setup(conf);
-
-        let file_name = Path::new(conf.dataset).file_name().and_then(|f| f.to_str()).unwrap();
-        let name = format!("{}: {}", file_name, conf.group_name);
-        let mut group = c.benchmark_group(&name);
-
-        for &query in conf.queries {
-            group.bench_with_input(BenchmarkId::from_parameter(query), &query, |b, &query| {
-                b.iter(|| {
-                    let rtxn = index.read_txn().unwrap();
-                    let mut search = index.search(&rtxn);
-                    search.query(query).terms_matching_strategy(TermsMatchingStrategy::default());
-                    if let Some(filter) = conf.filter {
-                        let filter = Filter::from_str(filter).unwrap().unwrap();
-                        search.filter(filter);
-                    }
-                    if let Some(sort) = &conf.sort {
-                        let sort = sort.iter().map(|sort| sort.parse().unwrap()).collect();
-                        search.sort_criteria(sort);
-                    }
-                    let _ids = search.execute().unwrap();
-                });
-            });
-        }
-        group.finish();
-
-        index.prepare_for_closing().wait();
-    }
-}
-
-pub fn documents_from(filename: &str, filetype: &str) -> DocumentsBatchReader<impl BufRead + Seek> {
-    let reader = File::open(filename)
-        .unwrap_or_else(|_| panic!("could not find the dataset in: {}", filename));
-    let reader = BufReader::new(reader);
-    let documents = match filetype {
-        "csv" => documents_from_csv(reader).unwrap(),
-        "json" => documents_from_json(reader).unwrap(),
-        "jsonl" => documents_from_jsonl(reader).unwrap(),
-        otherwise => panic!("invalid update format {:?}", otherwise),
-    };
-    DocumentsBatchReader::from_reader(Cursor::new(documents)).unwrap()
-}
-
-fn documents_from_jsonl(reader: impl BufRead) -> anyhow::Result<Vec<u8>> {
-    let mut documents = DocumentsBatchBuilder::new(Vec::new());
-
-    for result in serde_json::Deserializer::from_reader(reader).into_iter::<Object>() {
-        let object = result?;
-        documents.append_json_object(&object)?;
-    }
-
-    documents.into_inner().map_err(Into::into)
-}
-
-fn documents_from_json(reader: impl BufRead) -> anyhow::Result<Vec<u8>> {
-    let mut documents = DocumentsBatchBuilder::new(Vec::new());
-
-    documents.append_json_array(reader)?;
-
-    documents.into_inner().map_err(Into::into)
-}
-
-fn documents_from_csv(reader: impl BufRead) -> anyhow::Result<Vec<u8>> {
-    let csv = csv::Reader::from_reader(reader);
-
-    let mut documents = DocumentsBatchBuilder::new(Vec::new());
-    documents.append_csv(csv)?;
-
-    documents.into_inner().map_err(Into::into)
-}
-
-enum AllowedType {
-    String,
-    Number,
-}
-
-fn parse_csv_header(header: &str) -> (String, AllowedType) {
-    // if there are several separators we only split on the last one.
-    match header.rsplit_once(':') {
-        Some((field_name, field_type)) => match field_type {
-            "string" => (field_name.to_string(), AllowedType::String),
-            "number" => (field_name.to_string(), AllowedType::Number),
-            // we may return an error in this case.
-            _otherwise => (header.to_string(), AllowedType::String),
-        },
-        None => (header.to_string(), AllowedType::String),
-    }
-}
-
-struct CSVDocumentDeserializer<R>
-where
-    R: Read,
-{
-    documents: csv::StringRecordsIntoIter<R>,
-    headers: Vec<(String, AllowedType)>,
-}
-
-impl<R: Read> CSVDocumentDeserializer<R> {
-    fn from_reader(reader: R) -> io::Result<Self> {
-        let mut records = csv::Reader::from_reader(reader);
-
-        let headers = records.headers()?.into_iter().map(parse_csv_header).collect();
-
-        Ok(Self { documents: records.into_records(), headers })
-    }
-}
-
-impl<R: Read> Iterator for CSVDocumentDeserializer<R> {
-    type Item = anyhow::Result<Object>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let csv_document = self.documents.next()?;
-
-        match csv_document {
-            Ok(csv_document) => {
-                let mut document = Object::new();
-
-                for ((field_name, field_type), value) in
-                    self.headers.iter().zip(csv_document.into_iter())
-                {
-                    let parsed_value: Result<Value, ParseFloatError> = match field_type {
-                        AllowedType::Number => {
-                            value.parse::<f64>().map(Value::from).map_err(Into::into)
-                        }
-                        AllowedType::String => Ok(Value::String(value.to_string())),
-                    };
-
-                    match parsed_value {
-                        Ok(value) => drop(document.insert(field_name.to_string(), value)),
-                        Err(_e) => {
-                            return Some(Err(anyhow::anyhow!(
-                                "Value '{}' is not a valid number",
-                                value
-                            )))
-                        }
-                    }
-                }
-
-                Some(Ok(document))
-            }
-            Err(e) => Some(Err(anyhow::anyhow!("Error parsing csv document: {}", e))),
-        }
-    }
-}

benchmarks/build.rs

@@ -1,115 +0,0 @@
-use std::fs::File;
-use std::io::{Cursor, Read, Seek, Write};
-use std::path::{Path, PathBuf};
-use std::{env, fs};
-
-use bytes::Bytes;
-use convert_case::{Case, Casing};
-use flate2::read::GzDecoder;
-use reqwest::IntoUrl;
-
-const BASE_URL: &str = "https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets";
-
-const DATASET_SONGS: (&str, &str) = ("smol-songs", "csv");
-const DATASET_SONGS_1_2: (&str, &str) = ("smol-songs-1_2", "csv");
-const DATASET_SONGS_3_4: (&str, &str) = ("smol-songs-3_4", "csv");
-const DATASET_SONGS_4_4: (&str, &str) = ("smol-songs-4_4", "csv");
-const DATASET_WIKI: (&str, &str) = ("smol-wiki-articles", "csv");
-const DATASET_WIKI_1_2: (&str, &str) = ("smol-wiki-articles-1_2", "csv");
-const DATASET_WIKI_3_4: (&str, &str) = ("smol-wiki-articles-3_4", "csv");
-const DATASET_WIKI_4_4: (&str, &str) = ("smol-wiki-articles-4_4", "csv");
-const DATASET_MOVIES: (&str, &str) = ("movies", "json");
-const DATASET_MOVIES_1_2: (&str, &str) = ("movies-1_2", "json");
-const DATASET_MOVIES_3_4: (&str, &str) = ("movies-3_4", "json");
-const DATASET_MOVIES_4_4: (&str, &str) = ("movies-4_4", "json");
-const DATASET_NESTED_MOVIES: (&str, &str) = ("nested_movies", "json");
-const DATASET_GEO: (&str, &str) = ("smol-all-countries", "jsonl");
-
-const ALL_DATASETS: &[(&str, &str)] = &[
-    DATASET_SONGS,
-    DATASET_SONGS_1_2,
-    DATASET_SONGS_3_4,
-    DATASET_SONGS_4_4,
-    DATASET_WIKI,
-    DATASET_WIKI_1_2,
-    DATASET_WIKI_3_4,
-    DATASET_WIKI_4_4,
-    DATASET_MOVIES,
-    DATASET_MOVIES_1_2,
-    DATASET_MOVIES_3_4,
-    DATASET_MOVIES_4_4,
-    DATASET_NESTED_MOVIES,
-    DATASET_GEO,
-];
-
-/// The name of the environment variable used to select the path
-/// of the directory containing the datasets
-const BASE_DATASETS_PATH_KEY: &str = "MILLI_BENCH_DATASETS_PATH";
-
-fn main() -> anyhow::Result<()> {
-    let out_dir = PathBuf::from(env::var(BASE_DATASETS_PATH_KEY).unwrap_or(env::var("OUT_DIR")?));
-
-    let benches_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).join("benches");
-    let mut manifest_paths_file = File::create(benches_dir.join("datasets_paths.rs"))?;
-    write!(
-        manifest_paths_file,
-        r#"//! This file is generated by the build script.
-//! Do not modify by hand, use the build.rs file.
-#![allow(dead_code)]
-"#
-    )?;
-    writeln!(manifest_paths_file)?;
-
-    for (dataset, extension) in ALL_DATASETS {
-        let out_path = out_dir.join(dataset);
-        let out_file = out_path.with_extension(extension);
-
-        writeln!(
-            &mut manifest_paths_file,
-            r#"pub const {}: &str = {:?};"#,
-            dataset.to_case(Case::ScreamingSnake),
-            out_file.display(),
-        )?;
-
-        if out_file.exists() {
-            eprintln!(
-                "The dataset {} already exists on the file system and will not be downloaded again",
-                out_path.display(),
-            );
-            continue;
-        }
-        let url = format!("{}/{}.{}.gz", BASE_URL, dataset, extension);
-        eprintln!("downloading: {}", url);
-        let bytes = retry(|| download_dataset(url.clone()), 10)?;
-        eprintln!("{} downloaded successfully", url);
-        eprintln!("uncompressing in {}", out_file.display());
-        uncompress_in_file(bytes, &out_file)?;
-    }
-
-    Ok(())
-}
-
-fn retry<Ok, Err>(fun: impl Fn() -> Result<Ok, Err>, times: usize) -> Result<Ok, Err> {
-    for _ in 0..times {
-        if let ok @ Ok(_) = fun() {
-            return ok;
-        }
-    }
-    fun()
-}
-
-fn download_dataset<U: IntoUrl>(url: U) -> anyhow::Result<Cursor<Bytes>> {
-    let bytes =
-        reqwest::blocking::Client::builder().timeout(None).build()?.get(url).send()?.bytes()?;
-    Ok(Cursor::new(bytes))
-}
-
-fn uncompress_in_file<R: Read + Seek, P: AsRef<Path>>(bytes: R, path: P) -> anyhow::Result<()> {
-    let path = path.as_ref();
-    let mut gz = GzDecoder::new(bytes);
-    let mut dataset = Vec::new();
-    gz.read_to_end(&mut dataset)?;
-
-    fs::write(path, dataset)?;
-    Ok(())
-}

benchmarks/scripts/compare.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-# Requirements:
-# - critcmp. See: https://github.com/BurntSushi/critcmp
-# - curl
-
-# Usage
-# $ bash compare.sh json_file1 json_file1
-# ex: bash compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json
-
-# Checking that critcmp is installed
-command -v critcmp > /dev/null 2>&1
-if [[ "$?" -ne 0 ]]; then
-    echo 'You must install critcmp to make this script work.'
-    echo 'See: https://github.com/BurntSushi/critcmp'
-    echo '  $ cargo install critcmp'
-    exit 1
-fi
-
-s3_url='https://milli-benchmarks.fra1.digitaloceanspaces.com/critcmp_results'
-
-for file in $@
-do
-    file_s3_url="$s3_url/$file"
-    file_local_path="/tmp/$file"
-
-    if [[ ! -f $file_local_path ]]; then
-        curl $file_s3_url --output $file_local_path --silent
-        if [[ "$?" -ne 0 ]]; then
-            echo 'curl command failed.'
-            exit 1
-        fi
-    fi
-done
-
-path_list=$(echo " $@" | sed 's/ / \/tmp\//g')
-
-critcmp $path_list

benchmarks/scripts/list.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-
-# Requirements:
-# - curl
-# - grep
-
-res=$(curl -s https://milli-benchmarks.fra1.digitaloceanspaces.com | grep -o '<Key>[^<]\+' | cut -c 5- | grep critcmp_results/ | cut -c 18-)
-
-for pattern in "$@"
-do
-	res=$(echo "$res" | grep $pattern)
-done
-
-echo "$res"

benchmarks/src/lib.rs

@@ -1,5 +0,0 @@
-//! This library is only used to isolate the benchmarks
-//! from the original milli library.
-//!
-//! It does not include interesting functions for milli library
-//! users only for milli contributors.

config.toml

@@ -1,131 +1,120 @@
 # This file shows the default configuration of Meilisearch.
-# All variables are defined here: https://www.meilisearch.com/docs/learn/configuration/instance_options#environment-variables
+# All variables are defined here: https://docs.meilisearch.com/learn/configuration/instance_options.html#environment-variables
 
-# Designates the location where database files will be created and retrieved.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#database-path
 db_path = "./data.ms"
+# Designates the location where database files will be created and retrieved.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#database-path
 
-# Configures the instance's environment. Value must be either `production` or `development`.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#environment
 env = "development"
+# Configures the instance's environment. Value must be either `production` or `development`.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#environment
 
-# The address on which the HTTP server will listen.
 http_addr = "localhost:7700"
+# The address on which the HTTP server will listen.
 
-# Sets the instance's master key, automatically protecting all routes except GET /health.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#master-key
 # master_key = "YOUR_MASTER_KEY_VALUE"
+# Sets the instance's master key, automatically protecting all routes except GET /health.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#master-key
 
+# no_analytics = true
 # Deactivates Meilisearch's built-in telemetry when provided.
 # Meilisearch automatically collects data from all instances that do not opt out using this flag.
 # All gathered data is used solely for the purpose of improving Meilisearch, and can be deleted at any time.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#disable-analytics
-# no_analytics = true
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#disable-analytics
 
-# Sets the maximum size of accepted payloads.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#payload-limit-size
 http_payload_size_limit = "100 MB"
+# Sets the maximum size of accepted payloads.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#payload-limit-size
 
+log_level = "INFO"
 # Defines how much detail should be present in Meilisearch's logs.
 # Meilisearch currently supports six log levels, listed in order of increasing verbosity:  `OFF`, `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#log-level
-log_level = "INFO"
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#log-level
 
-# Sets the maximum amount of RAM Meilisearch can use when indexing.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#max-indexing-memory
 # max_indexing_memory = "2 GiB"
+# Sets the maximum amount of RAM Meilisearch can use when indexing.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#max-indexing-memory
 
-# Sets the maximum number of threads Meilisearch can use during indexing.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#max-indexing-threads
 # max_indexing_threads = 4
+# Sets the maximum number of threads Meilisearch can use during indexing.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#max-indexing-threads
 
 #############
 ### DUMPS ###
 #############
 
-# Sets the directory where Meilisearch will create dump files.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#dump-directory
 dump_dir = "dumps/"
+# Sets the directory where Meilisearch will create dump files.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#dumps-destination
 
-# Imports the dump file located at the specified path. Path must point to a .dump file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#import-dump
 # import_dump = "./path/to/my/file.dump"
+# Imports the dump file located at the specified path. Path must point to a .dump file.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#import-dump
 
-# Prevents Meilisearch from throwing an error when `import_dump` does not point to a valid dump file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-missing-dump
 ignore_missing_dump = false
+# Prevents Meilisearch from throwing an error when `import_dump` does not point to a valid dump file.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ignore-missing-dump
 
-# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_dump`.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-dump-if-db-exists
 ignore_dump_if_db_exists = false
+# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_dump`.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ignore-dump-if-db-exists
 
 
 #################
 ### SNAPSHOTS ###
 #################
 
+schedule_snapshot = false
 # Enables scheduled snapshots when true, disable when false (the default).
 # If the value is given as an integer, then enables the scheduled snapshot with the passed value as the interval
 # between each snapshot, in seconds.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#schedule-snapshot-creation
-schedule_snapshot = false
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#schedule-snapshot-creation
 
-# Sets the directory where Meilisearch will store snapshots.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#snapshot-destination
 snapshot_dir = "snapshots/"
+# Sets the directory where Meilisearch will store snapshots.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-destination
 
-# Launches Meilisearch after importing a previously-generated snapshot at the given filepath.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#import-snapshot
 # import_snapshot = "./path/to/my/snapshot"
+# Launches Meilisearch after importing a previously-generated snapshot at the given filepath.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#import-snapshot
 
-# Prevents a Meilisearch instance from throwing an error when `import_snapshot` does not point to a valid snapshot file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-missing-snapshot
 ignore_missing_snapshot = false
+# Prevents a Meilisearch instance from throwing an error when `import_snapshot` does not point to a valid snapshot file.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ignore-missing-snapshot
 
-# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_snapshot`.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-snapshot-if-db-exists
 ignore_snapshot_if_db_exists = false
+# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_snapshot`.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ignore-snapshot-if-db-exists
 
 
 ###########
 ### SSL ###
 ###########
 
-# Enables client authentication in the specified path.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-authentication-path
 # ssl_auth_path = "./path/to/root"
+# Enables client authentication in the specified path.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-authentication-path
 
-# Sets the server's SSL certificates.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-certificates-path
 # ssl_cert_path = "./path/to/certfile"
+# Sets the server's SSL certificates.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-certificates-path
 
-# Sets the server's SSL key files.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-key-path
 # ssl_key_path = "./path/to/private-key"
+# Sets the server's SSL key files.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-key-path
 
-# Sets the server's OCSP file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-ocsp-path
 # ssl_ocsp_path = "./path/to/ocsp-file"
+# Sets the server's OCSP file.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-ocsp-path
 
-# Makes SSL authentication mandatory.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-require-auth
 ssl_require_auth = false
+# Makes SSL authentication mandatory.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-require-auth
 
-# Activates SSL session resumption.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-resumption
 ssl_resumption = false
+# Activates SSL session resumption.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-resumption
 
-# Activates SSL tickets.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-tickets
 ssl_tickets = false
-
-#############################
-### Experimental features ###
-#############################
-
-# Experimental metrics feature. For more information, see: <https://github.com/meilisearch/meilisearch/discussions/3518>
-# Enables the Prometheus metrics on the `GET /metrics` endpoint.
-experimental_enable_metrics = false
-
-# Experimental RAM reduction during indexing, do not use in production, see: <https://github.com/meilisearch/product/discussions/652>
-experimental_reduce_indexing_memory_usage = false
+# Activates SSL tickets.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-tickets

download-latest.sh

@@ -103,7 +103,7 @@ not_available_failure_usage() {
     printf "$RED%s\n$DEFAULT" 'ERROR: Meilisearch binary is not available for your OS distribution or your architecture yet.'
     echo ''
     echo 'However, you can easily compile the binary from the source files.'
-    echo 'Follow the steps at the page ("Source" tab): https://www.meilisearch.com/docs/learn/getting_started/installation'
+    echo 'Follow the steps at the page ("Source" tab): https://docs.meilisearch.com/learn/getting_started/installation.html'
 }
 
 fetch_release_failure_usage() {

dump/Cargo.toml

@@ -1,32 +1,25 @@
 [package]
 name = "dump"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-edition.workspace = true
-homepage.workspace = true
-readme.workspace = true
-license.workspace = true
+version = "1.0.1"
+edition = "2021"
 
 [dependencies]
-anyhow = "1.0.70"
-flate2 = "1.0.25"
-http = "0.2.9"
+anyhow = "1.0.65"
+flate2 = "1.0.22"
+http = "0.2.8"
 log = "0.4.17"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-once_cell = "1.17.1"
-regex = "1.7.3"
-roaring = { version = "0.10.1", features = ["serde"] }
-serde = { version = "1.0.160", features = ["derive"] }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
+once_cell = "1.15.0"
+regex = "1.6.0"
+roaring = { version = "0.10.0", features = ["serde"] }
+serde = { version = "1.0.136", features = ["derive"] }
+serde_json = { version = "1.0.85", features = ["preserve_order"] }
 tar = "0.4.38"
-tempfile = "3.5.0"
-thiserror = "1.0.40"
-time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-uuid = { version = "1.3.1", features = ["serde", "v4"] }
+tempfile = "3.3.0"
+thiserror = "1.0.30"
+time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+uuid = { version = "1.1.2", features = ["serde", "v4"] }
 
 [dev-dependencies]
 big_s = "1.0.2"

dump/src/lib.rs

@@ -101,9 +101,6 @@ pub enum KindDump {
         documents_ids: Vec<String>,
     },
     DocumentClear,
-    DocumentDeletionByFilter {
-        filter: serde_json::Value,
-    },
     Settings {
         settings: Box<meilisearch_types::settings::Settings<Unchecked>>,
         is_deletion: bool,
@@ -169,9 +166,6 @@ impl From<KindWithContent> for KindDump {
             KindWithContent::DocumentDeletion { documents_ids, .. } => {
                 KindDump::DocumentDeletion { documents_ids }
             }
-            KindWithContent::DocumentDeletionByFilter { filter_expr, .. } => {
-                KindDump::DocumentDeletionByFilter { filter: filter_expr }
-            }
             KindWithContent::DocumentClear { .. } => KindDump::DocumentClear,
             KindWithContent::SettingsUpdate {
                 new_settings,
@@ -208,14 +202,13 @@ pub(crate) mod test {
     use std::str::FromStr;
 
     use big_s::S;
-    use maplit::{btreemap, btreeset};
-    use meilisearch_types::facet_values_sort::FacetValuesSort;
-    use meilisearch_types::features::RuntimeTogglableFeatures;
-    use meilisearch_types::index_uid_pattern::IndexUidPattern;
+    use maplit::btreeset;
+    use meilisearch_types::index_uid::IndexUid;
     use meilisearch_types::keys::{Action, Key};
-    use meilisearch_types::milli;
     use meilisearch_types::milli::update::Setting;
-    use meilisearch_types::settings::{Checked, FacetingSettings, Settings};
+    use meilisearch_types::milli::{self};
+    use meilisearch_types::settings::{Checked, Settings};
+    use meilisearch_types::star_or::StarOr;
     use meilisearch_types::tasks::{Details, Status};
     use serde_json::{json, Map, Value};
     use time::macros::datetime;
@@ -262,18 +255,10 @@ pub(crate) mod test {
             sortable_attributes: Setting::Set(btreeset! { S("age") }),
             ranking_rules: Setting::NotSet,
             stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
             synonyms: Setting::NotSet,
             distinct_attribute: Setting::NotSet,
             typo_tolerance: Setting::NotSet,
-            faceting: Setting::Set(FacetingSettings {
-                max_values_per_facet: Setting::Set(111),
-                sort_facet_values_by: Setting::Set(
-                    btreemap! { S("age") => FacetValuesSort::Count },
-                ),
-            }),
+            faceting: Setting::NotSet,
             pagination: Setting::NotSet,
             _kind: std::marker::PhantomData,
         };
@@ -356,7 +341,7 @@ pub(crate) mod test {
                 name: Some(S("doggos_key")),
                 uid: Uuid::from_str("9f8a34da-b6b2-42f0-939b-dbd4c3448655").unwrap(),
                 actions: vec![Action::DocumentsAll],
-                indexes: vec![IndexUidPattern::from_str("doggos").unwrap()],
+                indexes: vec![StarOr::Other(IndexUid::from_str("doggos").unwrap())],
                 expires_at: Some(datetime!(4130-03-14 12:21 UTC)),
                 created_at: datetime!(1960-11-15 0:00 UTC),
                 updated_at: datetime!(2022-11-10 0:00 UTC),
@@ -366,7 +351,7 @@ pub(crate) mod test {
                 name: Some(S("master_key")),
                 uid: Uuid::from_str("4622f717-1c00-47bb-a494-39d76a49b591").unwrap(),
                 actions: vec![Action::All],
-                indexes: vec![IndexUidPattern::all()],
+                indexes: vec![StarOr::Star],
                 expires_at: None,
                 created_at: datetime!(0000-01-01 00:01 UTC),
                 updated_at: datetime!(1964-05-04 17:25 UTC),
@@ -422,11 +407,6 @@ pub(crate) mod test {
         }
         keys.flush().unwrap();
 
-        // ========== experimental features
-        let features = create_test_features();
-
-        dump.create_experimental_features(features).unwrap();
-
         // create the dump
         let mut file = tempfile::tempfile().unwrap();
         dump.persist_to(&mut file).unwrap();
@@ -435,10 +415,6 @@ pub(crate) mod test {
         file
     }
 
-    fn create_test_features() -> RuntimeTogglableFeatures {
-        RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
-    }
-
     #[test]
     fn test_creating_and_read_dump() {
         let mut file = create_test_dump();
@@ -483,9 +459,5 @@ pub(crate) mod test {
         for (key, expected) in dump.keys().unwrap().zip(create_test_api_keys()) {
             assert_eq!(key.unwrap(), expected);
         }
-
-        // ==== checking the features
-        let expected = create_test_features();
-        assert_eq!(dump.features().unwrap().unwrap(), expected);
     }
 }

dump/src/reader/compat/v2_to_v3.rs

@@ -25,6 +25,7 @@ impl CompatV2ToV3 {
             CompatV2ToV3::Compat(compat) => compat.index_uuid(),
         };
         v2_uuids
+            .into_iter()
             .into_iter()
             .map(|index| v3::meta::IndexUuid { uid: index.uid, uuid: index.uuid })
             .collect()

dump/src/reader/compat/v5_to_v6.rs

@@ -181,8 +181,10 @@ impl CompatV5ToV6 {
                     .indexes
                     .into_iter()
                     .map(|index| match index {
-                        v5::StarOr::Star => v6::IndexUidPattern::all(),
-                        v5::StarOr::Other(uid) => v6::IndexUidPattern::new_unchecked(uid.as_str()),
+                        v5::StarOr::Star => v6::StarOr::Star,
+                        v5::StarOr::Other(uid) => {
+                            v6::StarOr::Other(v6::IndexUid::new_unchecked(uid.as_str()))
+                        }
                     })
                     .collect(),
                 expires_at: key.expires_at,
@@ -191,10 +193,6 @@ impl CompatV5ToV6 {
             })
         })))
     }
-
-    pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
-        Ok(None)
-    }
 }
 
 pub enum CompatIndexV5ToV6 {
@@ -340,9 +338,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                 }
             },
             stop_words: settings.stop_words.into(),
-            non_separator_tokens: v6::Setting::NotSet,
-            separator_tokens: v6::Setting::NotSet,
-            dictionary: v6::Setting::NotSet,
             synonyms: settings.synonyms.into(),
             distinct_attribute: settings.distinct_attribute.into(),
             typo_tolerance: match settings.typo_tolerance {
@@ -365,7 +360,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
             faceting: match settings.faceting {
                 v5::Setting::Set(faceting) => v6::Setting::Set(v6::FacetingSettings {
                     max_values_per_facet: faceting.max_values_per_facet.into(),
-                    sort_facet_values_by: v6::Setting::NotSet,
                 }),
                 v5::Setting::Reset => v6::Setting::Reset,
                 v5::Setting::NotSet => v6::Setting::NotSet,

dump/src/reader/mod.rs

@@ -107,13 +107,6 @@ impl DumpReader {
             DumpReader::Compat(compat) => compat.keys(),
         }
     }
-
-    pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
-        match self {
-            DumpReader::Current(current) => Ok(current.features()),
-            DumpReader::Compat(compat) => compat.features(),
-        }
-    }
 }
 
 impl From<V6Reader> for DumpReader {
@@ -195,53 +188,6 @@ pub(crate) mod test {
     use meili_snap::insta;
 
     use super::*;
-    use crate::reader::v6::RuntimeTogglableFeatures;
-
-    #[test]
-    fn import_dump_v6_experimental() {
-        let dump = File::open("tests/assets/v6-with-experimental.dump").unwrap();
-        let mut dump = DumpReader::open(dump).unwrap();
-
-        // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-07-06 7:10:27.21958 +00:00:00");
-        insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
-
-        // tasks
-        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"d45cd8571703e58ae53c7bd7ce3f5c22");
-        assert_eq!(update_files.len(), 2);
-        assert!(update_files[0].is_none()); // the dump creation
-        assert!(update_files[1].is_none()); // the processed document addition
-
-        // keys
-        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"13c2da155e9729c2344688cab29af71d");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut test = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        insta::assert_json_snapshot!(test.metadata(), @r###"
-        {
-          "uid": "test",
-          "primaryKey": "id",
-          "createdAt": "2023-07-06T07:07:41.364694Z",
-          "updatedAt": "2023-07-06T07:07:41.396114Z"
-        }
-        "###);
-
-        assert_eq!(test.documents().unwrap().count(), 1);
-
-        assert_eq!(
-            dump.features().unwrap().unwrap(),
-            RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
-        );
-    }
 
     #[test]
     fn import_dump_v5() {
@@ -319,8 +265,6 @@ pub(crate) mod test {
         let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
-
-        assert_eq!(dump.features().unwrap(), None);
     }
 
     #[test]
@@ -526,12 +470,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2022-10-09T20:27:22.688964637Z",
-          "updatedAt": "2022-10-09T20:27:23.951017769Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -541,12 +485,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2022-10-09T20:27:22.197788495Z",
-          "updatedAt": "2022-10-09T20:28:01.93111053Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -571,12 +515,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2022-10-09T20:27:24.242683494Z",
-          "updatedAt": "2022-10-09T20:27:24.312809641Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -617,12 +561,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2023-01-30T16:25:56.595257Z",
-          "updatedAt": "2023-01-30T16:25:58.70348Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -632,12 +576,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2023-01-30T16:25:56.192178Z",
-          "updatedAt": "2023-01-30T16:25:56.455714Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -647,12 +591,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"0227598af846e574139ee0b80e03a720");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2023-01-30T16:25:58.876405Z",
-          "updatedAt": "2023-01-30T16:25:59.079906Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 

dump/src/reader/v2/mod.rs

@@ -46,7 +46,6 @@ pub type Checked = settings::Checked;
 pub type Unchecked = settings::Unchecked;
 
 pub type Task = updates::UpdateEntry;
-pub type Kind = updates::UpdateMeta;
 
 // everything related to the errors
 pub type ResponseError = errors::ResponseError;
@@ -108,11 +107,8 @@ impl V2Reader {
     pub fn indexes(&self) -> Result<impl Iterator<Item = Result<V2IndexReader>> + '_> {
         Ok(self.index_uuid.iter().map(|index| -> Result<_> {
             V2IndexReader::new(
+                index.uid.clone(),
                 &self.dump.path().join("indexes").join(format!("index-{}", index.uuid)),
-                index,
-                BufReader::new(
-                    File::open(self.dump.path().join("updates").join("data.jsonl")).unwrap(),
-                ),
             )
         }))
     }
@@ -147,41 +143,16 @@ pub struct V2IndexReader {
 }
 
 impl V2IndexReader {
-    pub fn new(path: &Path, index_uuid: &IndexUuid, tasks: BufReader<File>) -> Result<Self> {
+    pub fn new(name: String, path: &Path) -> Result<Self> {
         let meta = File::open(path.join("meta.json"))?;
         let meta: DumpMeta = serde_json::from_reader(meta)?;
 
-        let mut created_at = None;
-        let mut updated_at = None;
-
-        for line in tasks.lines() {
-            let task: Task = serde_json::from_str(&line?)?;
-            if !(task.uuid == index_uuid.uuid && task.is_finished()) {
-                continue;
-            }
-
-            let new_created_at = match task.update.meta() {
-                Kind::DocumentsAddition { .. } | Kind::Settings(_) => task.update.finished_at(),
-                _ => None,
-            };
-            let new_updated_at = task.update.finished_at();
-
-            if created_at.is_none() || created_at > new_created_at {
-                created_at = new_created_at;
-            }
-
-            if updated_at.is_none() || updated_at < new_updated_at {
-                updated_at = new_updated_at;
-            }
-        }
-
-        let current_time = OffsetDateTime::now_utc();
-
         let metadata = IndexMetadata {
-            uid: index_uuid.uid.clone(),
+            uid: name,
             primary_key: meta.primary_key,
-            created_at: created_at.unwrap_or(current_time),
-            updated_at: updated_at.unwrap_or(current_time),
+            // FIXME: Iterate over the whole task queue to find the creation and last update date.
+            created_at: OffsetDateTime::now_utc(),
+            updated_at: OffsetDateTime::now_utc(),
         };
 
         let ret = V2IndexReader {
@@ -277,12 +248,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2022-10-09T20:27:22.688964637Z",
-          "updatedAt": "2022-10-09T20:27:23.951017769Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -292,12 +263,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2022-10-09T20:27:22.197788495Z",
-          "updatedAt": "2022-10-09T20:28:01.93111053Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -322,12 +293,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2022-10-09T20:27:24.242683494Z",
-          "updatedAt": "2022-10-09T20:27:24.312809641Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -369,12 +340,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2023-01-30T16:25:56.595257Z",
-          "updatedAt": "2023-01-30T16:25:58.70348Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -384,12 +355,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2023-01-30T16:25:56.192178Z",
-          "updatedAt": "2023-01-30T16:25:56.455714Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -399,12 +370,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"0227598af846e574139ee0b80e03a720");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2023-01-30T16:25:58.876405Z",
-          "updatedAt": "2023-01-30T16:25:59.079906Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 

dump/src/reader/v2/updates.rs

@@ -227,14 +227,4 @@ impl UpdateStatus {
             _ => None,
         }
     }
-
-    pub fn finished_at(&self) -> Option<OffsetDateTime> {
-        match self {
-            UpdateStatus::Processing(_) => None,
-            UpdateStatus::Enqueued(_) => None,
-            UpdateStatus::Processed(u) => Some(u.processed_at),
-            UpdateStatus::Aborted(_) => None,
-            UpdateStatus::Failed(u) => Some(u.failed_at),
-        }
-    }
 }

dump/src/reader/v3/mod.rs

@@ -112,11 +112,8 @@ impl V3Reader {
     pub fn indexes(&self) -> Result<impl Iterator<Item = Result<V3IndexReader>> + '_> {
         Ok(self.index_uuid.iter().map(|index| -> Result<_> {
             V3IndexReader::new(
+                index.uid.clone(),
                 &self.dump.path().join("indexes").join(index.uuid.to_string()),
-                index,
-                BufReader::new(
-                    File::open(self.dump.path().join("updates").join("data.jsonl")).unwrap(),
-                ),
             )
         }))
     }
@@ -158,42 +155,16 @@ pub struct V3IndexReader {
 }
 
 impl V3IndexReader {
-    pub fn new(path: &Path, index_uuid: &IndexUuid, tasks: BufReader<File>) -> Result<Self> {
+    pub fn new(name: String, path: &Path) -> Result<Self> {
         let meta = File::open(path.join("meta.json"))?;
         let meta: DumpMeta = serde_json::from_reader(meta)?;
 
-        let mut created_at = None;
-        let mut updated_at = None;
-
-        for line in tasks.lines() {
-            let task: Task = serde_json::from_str(&line?)?;
-
-            if !(task.uuid == index_uuid.uuid && task.is_finished()) {
-                continue;
-            }
-
-            let new_created_at = match task.update.meta() {
-                Kind::DocumentAddition { .. } | Kind::Settings(_) => task.update.finished_at(),
-                _ => None,
-            };
-            let new_updated_at = task.update.finished_at();
-
-            if created_at.is_none() || created_at > new_created_at {
-                created_at = new_created_at;
-            }
-
-            if updated_at.is_none() || updated_at < new_updated_at {
-                updated_at = new_updated_at;
-            }
-        }
-
-        let current_time = OffsetDateTime::now_utc();
-
         let metadata = IndexMetadata {
-            uid: index_uuid.uid.clone(),
+            uid: name,
             primary_key: meta.primary_key,
-            created_at: created_at.unwrap_or(current_time),
-            updated_at: updated_at.unwrap_or(current_time),
+            // FIXME: Iterate over the whole task queue to find the creation and last update date.
+            created_at: OffsetDateTime::now_utc(),
+            updated_at: OffsetDateTime::now_utc(),
         };
 
         let ret = V3IndexReader {
@@ -292,12 +263,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2022-10-07T11:38:54.74389899Z",
-          "updatedAt": "2022-10-07T11:38:55.963185778Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -307,12 +278,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2022-10-07T11:38:54.026649575Z",
-          "updatedAt": "2022-10-07T11:39:04.188852537Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -337,12 +308,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2022-10-07T11:38:56.265951133Z",
-          "updatedAt": "2022-10-07T11:38:56.521004328Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 

dump/src/reader/v3/updates.rs

@@ -74,26 +74,6 @@ impl UpdateStatus {
             _ => None,
         }
     }
-
-    pub fn enqueued_at(&self) -> Option<OffsetDateTime> {
-        match self {
-            UpdateStatus::Processing(u) => Some(u.from.enqueued_at),
-            UpdateStatus::Enqueued(u) => Some(u.enqueued_at),
-            UpdateStatus::Processed(u) => Some(u.from.from.enqueued_at),
-            UpdateStatus::Aborted(u) => Some(u.from.enqueued_at),
-            UpdateStatus::Failed(u) => Some(u.from.from.enqueued_at),
-        }
-    }
-
-    pub fn finished_at(&self) -> Option<OffsetDateTime> {
-        match self {
-            UpdateStatus::Processing(_) => None,
-            UpdateStatus::Enqueued(_) => None,
-            UpdateStatus::Processed(u) => Some(u.processed_at),
-            UpdateStatus::Aborted(_) => None,
-            UpdateStatus::Failed(u) => Some(u.failed_at),
-        }
-    }
 }
 
 #[derive(Debug, Deserialize, Clone)]

dump/src/reader/v6/mod.rs

@@ -2,7 +2,6 @@ use std::fs::{self, File};
 use std::io::{BufRead, BufReader, ErrorKind};
 use std::path::Path;
 
-use log::debug;
 pub use meilisearch_types::milli;
 use tempfile::TempDir;
 use time::OffsetDateTime;
@@ -19,7 +18,6 @@ pub type Unchecked = meilisearch_types::settings::Unchecked;
 
 pub type Task = crate::TaskDump;
 pub type Key = meilisearch_types::keys::Key;
-pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures;
 
 // ===== Other types to clarify the code of the compat module
 // everything related to the tasks
@@ -36,7 +34,8 @@ pub type PaginationSettings = meilisearch_types::settings::PaginationSettings;
 
 // everything related to the api keys
 pub type Action = meilisearch_types::keys::Action;
-pub type IndexUidPattern = meilisearch_types::index_uid_pattern::IndexUidPattern;
+pub type StarOr<T> = meilisearch_types::star_or::StarOr<T>;
+pub type IndexUid = meilisearch_types::index_uid::IndexUid;
 
 // everything related to the errors
 pub type ResponseError = meilisearch_types::error::ResponseError;
@@ -49,7 +48,6 @@ pub struct V6Reader {
     metadata: Metadata,
     tasks: BufReader<File>,
     keys: BufReader<File>,
-    features: Option<RuntimeTogglableFeatures>,
 }
 
 impl V6Reader {
@@ -61,29 +59,11 @@ impl V6Reader {
             Err(e) => return Err(e.into()),
         };
 
-        let feature_file = match fs::read(dump.path().join("experimental-features.json")) {
-            Ok(feature_file) => Some(feature_file),
-            Err(error) => match error.kind() {
-                // Allows the file to be missing, this will only result in all experimental features disabled.
-                ErrorKind::NotFound => {
-                    debug!("`experimental-features.json` not found in dump");
-                    None
-                }
-                _ => return Err(error.into()),
-            },
-        };
-        let features = if let Some(feature_file) = feature_file {
-            Some(serde_json::from_reader(&*feature_file)?)
-        } else {
-            None
-        };
-
         Ok(V6Reader {
             metadata: serde_json::from_reader(&*meta_file)?,
             instance_uid,
             tasks: BufReader::new(File::open(dump.path().join("tasks").join("queue.jsonl"))?),
             keys: BufReader::new(File::open(dump.path().join("keys.jsonl"))?),
-            features,
             dump,
         })
     }
@@ -150,10 +130,6 @@ impl V6Reader {
             (&mut self.keys).lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
         )
     }
-
-    pub fn features(&self) -> Option<RuntimeTogglableFeatures> {
-        self.features
-    }
 }
 
 pub struct UpdateFile {

dump/src/writer.rs

@@ -4,7 +4,6 @@ use std::path::PathBuf;
 
 use flate2::write::GzEncoder;
 use flate2::Compression;
-use meilisearch_types::features::RuntimeTogglableFeatures;
 use meilisearch_types::keys::Key;
 use meilisearch_types::settings::{Checked, Settings};
 use serde_json::{Map, Value};
@@ -54,13 +53,6 @@ impl DumpWriter {
         TaskWriter::new(self.dir.path().join("tasks"))
     }
 
-    pub fn create_experimental_features(&self, features: RuntimeTogglableFeatures) -> Result<()> {
-        Ok(std::fs::write(
-            self.dir.path().join("experimental-features.json"),
-            serde_json::to_string(&features)?,
-        )?)
-    }
-
     pub fn persist_to(self, mut writer: impl Write) -> Result<()> {
         let gz_encoder = GzEncoder::new(&mut writer, Compression::default());
         let mut tar_encoder = tar::Builder::new(gz_encoder);
@@ -292,7 +284,6 @@ pub(crate) mod test {
         │    ├---- update_files/
         │    │    └---- 1.jsonl
         │    └---- queue.jsonl
-        ├---- experimental-features.json
         ├---- instance_uid.uuid
         ├---- keys.jsonl
         └---- metadata.json

file-store/Cargo.toml

@@ -1,19 +1,12 @@
 [package]
 name = "file-store"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
+version = "1.0.1"
+edition = "2021"
 
 [dependencies]
-tempfile = "3.5.0"
-thiserror = "1.0.40"
-uuid = { version = "1.3.1", features = ["serde", "v4"] }
+tempfile = "3.3.0"
+thiserror = "1.0.30"
+uuid = { version = "1.1.2", features = ["serde", "v4"] }
 
 [dev-dependencies]
-faux = "0.1.9"
+faux = "0.1.8"

file-store/src/lib.rs

@@ -116,20 +116,10 @@ impl FileStore {
 
     /// List the Uuids of the files in the FileStore
     pub fn all_uuids(&self) -> Result<impl Iterator<Item = Result<Uuid>>> {
-        Ok(self.path.read_dir()?.filter_map(|entry| {
-            let file_name = match entry {
-                Ok(entry) => entry.file_name(),
-                Err(e) => return Some(Err(e.into())),
-            };
-            let file_name = match file_name.to_str() {
-                Some(file_name) => file_name,
-                None => return Some(Err(Error::CouldNotParseFileNameAsUtf8)),
-            };
-            if file_name.starts_with('.') {
-                None
-            } else {
-                Some(Uuid::from_str(file_name).map_err(|e| e.into()))
-            }
+        Ok(self.path.read_dir()?.map(|entry| {
+            Ok(Uuid::from_str(
+                entry?.file_name().to_str().ok_or(Error::CouldNotParseFileNameAsUtf8)?,
+            )?)
         }))
     }
 }
@@ -145,34 +135,3 @@ impl File {
         Ok(())
     }
 }
-
-#[cfg(test)]
-mod test {
-    use std::io::Write;
-
-    use tempfile::TempDir;
-
-    use super::*;
-
-    #[test]
-    fn all_uuids() {
-        let dir = TempDir::new().unwrap();
-        let fs = FileStore::new(dir.path()).unwrap();
-        let (uuid, mut file) = fs.new_update().unwrap();
-        file.write_all(b"Hello world").unwrap();
-        file.persist().unwrap();
-        let all_uuids = fs.all_uuids().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(all_uuids, vec![uuid]);
-
-        let (uuid2, file) = fs.new_update().unwrap();
-        let all_uuids = fs.all_uuids().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(all_uuids, vec![uuid]);
-
-        file.persist().unwrap();
-        let mut all_uuids = fs.all_uuids().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        all_uuids.sort();
-        let mut expected = vec![uuid, uuid2];
-        expected.sort();
-        assert_eq!(all_uuids, expected);
-    }
-}

filter-parser/Cargo.toml

@@ -1,20 +0,0 @@
-[package]
-name = "filter-parser"
-description = "The parser for the Meilisearch filter syntax"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-# description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
-[dependencies]
-nom = "7.1.3"
-nom_locate = "4.1.0"
-unescaper = "0.1.2"
-
-[dev-dependencies]
-insta = "1.29.0"

filter-parser/README.md

@@ -1,36 +0,0 @@
-# Filter parser
-
-This workspace is dedicated to the parsing of the Meilisearch filters.
-
-Most of the code and explanation are in the [`lib.rs`](./src/lib.rs). Especially, the BNF of the filters at the top of this file.
-
-The parser use [nom](https://docs.rs/nom/) to do most of its work and [nom-locate](https://docs.rs/nom_locate/) to keep track of what we were doing when we encountered an error.
-
-## Cli
-A simple main is provided to quick-test if a filter can be parsed or not without bringing milli.
-It takes one argument and try to parse it.
-```
-cargo run -- 'field = value' # success
-cargo run -- 'field = "doggo' # error => missing closing delimiter "
-```
-
-## Fuzz
-The workspace have been fuzzed with [cargo-fuzz](https://rust-fuzz.github.io/book/cargo-fuzz.html).
-
-### Setup
-You'll need rust-nightly to execute the fuzzer.
-
-```
-cargo install cargo-fuzz
-```
-
-### Run
-When the filter parser is executed by the fuzzer it's triggering a stackoverflow really fast. We can avoid this problem by limiting the `max_len` of [libfuzzer](https://llvm.org/docs/LibFuzzer.html) at 500 characters.
-```
-cargo fuzz run parse -- -max_len=500
-```
-
-## What to do if you find a bug in the parser
-
-- Write a test at the end of the [`lib.rs`](./src/lib.rs) to ensure it never happens again.
-- Add a file in [the corpus directory](./fuzz/corpus/parse/) with your filter to help the fuzzer find new bugs. Since this directory is going to be heavily polluted by the execution of the fuzzer it's in the gitignore and you'll need to force push your new test.

filter-parser/fuzz/.gitignore

@@ -1,3 +0,0 @@
-/corpus/
-/artifacts/
-/target/

filter-parser/fuzz/Cargo.toml

@@ -1,25 +0,0 @@
-[package]
-name = "filter-parser-fuzz"
-version = "0.0.0"
-authors = ["Automatically generated"]
-publish = false
-edition = "2018"
-
-[package.metadata]
-cargo-fuzz = true
-
-[dependencies]
-libfuzzer-sys = "0.4"
-
-[dependencies.filter-parser]
-path = ".."
-
-# Prevent this from interfering with workspaces
-[workspace]
-members = ["."]
-
-[[bin]]
-name = "parse"
-path = "fuzz_targets/parse.rs"
-test = false
-doc = false

filter-parser/fuzz/corpus/parse/test_1

@@ -1 +0,0 @@
-channel = Ponce

filter-parser/fuzz/corpus/parse/test_10

@@ -1 +0,0 @@
-channel != ponce

filter-parser/fuzz/corpus/parse/test_11

@@ -1 +0,0 @@
-NOT channel = ponce

filter-parser/fuzz/corpus/parse/test_12

@@ -1 +0,0 @@
-subscribers < 1000

filter-parser/fuzz/corpus/parse/test_13

@@ -1 +0,0 @@
-subscribers > 1000

filter-parser/fuzz/corpus/parse/test_14

@@ -1 +0,0 @@
-subscribers <= 1000

filter-parser/fuzz/corpus/parse/test_15

@@ -1 +0,0 @@
-subscribers >= 1000

filter-parser/fuzz/corpus/parse/test_16

@@ -1 +0,0 @@
-NOT subscribers < 1000

filter-parser/fuzz/corpus/parse/test_17

@@ -1 +0,0 @@
-NOT subscribers > 1000

filter-parser/fuzz/corpus/parse/test_18

@@ -1 +0,0 @@
-NOT subscribers <= 1000

filter-parser/fuzz/corpus/parse/test_19

@@ -1 +0,0 @@
-NOT subscribers >= 1000

filter-parser/fuzz/corpus/parse/test_2

@@ -1 +0,0 @@
-subscribers = 12

filter-parser/fuzz/corpus/parse/test_20

@@ -1 +0,0 @@
-subscribers 100 TO 1000

filter-parser/fuzz/corpus/parse/test_21

@@ -1 +0,0 @@
-NOT subscribers 100 TO 1000

filter-parser/fuzz/corpus/parse/test_22

@@ -1 +0,0 @@
-_geoRadius(12, 13, 14)

filter-parser/fuzz/corpus/parse/test_23

@@ -1 +0,0 @@
-NOT _geoRadius(12, 13, 14)

filter-parser/fuzz/corpus/parse/test_24

@@ -1 +0,0 @@
-channel = ponce AND 'dog race' != 'bernese mountain'

filter-parser/fuzz/corpus/parse/test_25

@@ -1 +0,0 @@
-channel = ponce OR 'dog race' != 'bernese mountain'

filter-parser/fuzz/corpus/parse/test_26

@@ -1 +0,0 @@
-channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000

filter-parser/fuzz/corpus/parse/test_27

@@ -1 +0,0 @@
-channel = ponce AND ( 'dog race' != 'bernese mountain' OR subscribers > 1000 )

filter-parser/fuzz/corpus/parse/test_28

@@ -1 +0,0 @@
-(channel = ponce AND 'dog race' != 'bernese mountain' OR subscribers > 1000) AND _geoRadius(12, 13, 14)

filter-parser/fuzz/corpus/parse/test_29

@@ -1 +0,0 @@
-channel = Ponce = 12

filter-parser/fuzz/corpus/parse/test_3

@@ -1 +0,0 @@
-channel = 'Mister Mv'

filter-parser/fuzz/corpus/parse/test_30

@@ -1 +0,0 @@
-channel =

filter-parser/fuzz/corpus/parse/test_31

@@ -1 +0,0 @@
-channel = 🐻

filter-parser/fuzz/corpus/parse/test_32

@@ -1 +0,0 @@
-OR

filter-parser/fuzz/corpus/parse/test_33

@@ -1 +0,0 @@
-AND

filter-parser/fuzz/corpus/parse/test_34

@@ -1 +0,0 @@
-channel Ponce

filter-parser/fuzz/corpus/parse/test_35

@@ -1 +0,0 @@
-channel = Ponce OR

filter-parser/fuzz/corpus/parse/test_36

@@ -1 +0,0 @@
-_geoRadius

filter-parser/fuzz/corpus/parse/test_37

@@ -1 +0,0 @@
-_geoRadius = 12

filter-parser/fuzz/corpus/parse/test_38

@@ -1 +0,0 @@
-_geoPoint(12, 13, 14)

filter-parser/fuzz/corpus/parse/test_39

@@ -1 +0,0 @@
-position <= _geoPoint(12, 13, 14)