Introduce ContextMut and Context structs

This simplified the multiword_rewrite_matches function a little bit.

.dockerignore

@@ -1,4 +1,5 @@
 target
 Dockerfile
 .dockerignore
+.git
 .gitignore

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,31 +0,0 @@
----
-name: Bug report
-about: Create a report to help us improve
-title: ''
-labels: ''
-assignees: ''
-
----
-
-**Describe the bug**
-A clear and concise description of what the bug is.
-
-**To Reproduce**
-Steps to reproduce the behavior:
-1. Go to '...'
-2. Click on '....'
-3. Scroll down to '....'
-4. See error
-
-**Expected behavior**
-A clear and concise description of what you expected to happen.
-
-**Screenshots**
-If applicable, add screenshots to help explain your problem.
-
-**Meilisearch version:**
-[e.g. v0.20.0]
-
-**Additional context**
-Additional information that may be relevant to the issue.
-[e.g. architecture, device, OS, browser]

.github/ISSUE_TEMPLATE/config.yml

@@ -1,13 +0,0 @@
-contact_links:
-  - name: Support questions & other
-    url: https://github.com/meilisearch/meilisearch/discussions/new
-    about: For any other question, open a discussion in this repository
-  - name: Language support request & feedback
-    url: https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal?discussions_q=label%3Aproduct%3Acore%3Atokenizer+category%3A%22Feedback+%26+Feature+Proposal%22
-    about:  The requests and feedback regarding Language support are not managed in this repository. Please upvote the related discussion in our dedicated product repository or open a new one if it doesn't exist.
-  - name: Any other feature request & feedback
-    url: https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal
-    about: The feature requests and feedback regarding the already existing features are not managed in this repository. Please open a discussion in our dedicated product repository
-  - name: Documentation issue
-    url: https://github.com/meilisearch/documentation/issues/new
-    about: For documentation issues, open an issue or a PR in the documentation repository

.github/ISSUE_TEMPLATE/sprint_issue.md

@@ -1,34 +0,0 @@
----
-name: New sprint issue
-about: ⚠️ Should only be used by the engine team ⚠️
-title: ''
-labels: ''
-assignees: ''
-
----
-
-Related product team resources: [roadmap card]() (_internal only_) and [PRD]() (_internal only_)
-Related product discussion:
-Related spec: WIP
-
-## Motivation
-
-<!---Copy/paste the information in the roadmap resources or briefly detail the product motivation. Ask product team if any hesitation.-->
-
-## Usage
-
-<!---Write a quick description of the usage if the usage has already been defined-->
-
-Refer to the final spec to know the details and the final decisions about the usage.
-
-## TODO
-
-<!---Feel free to adapt this list with more technical/product steps-->
-
-- [ ] Release a prototype
-- [ ] If prototype validated, merge changes into `main`
-- [ ] Update the spec
-
-## Impacted teams
-
-<!---Ping the related teams. Ask for the engine manager if any hesitation-->

.github/dependabot.yml

@@ -1,12 +0,0 @@
-# Set update schedule for GitHub Actions only
-
-version: 2
-updates:
-  - package-ecosystem: "github-actions"
-    directory: "/"
-    schedule:
-      interval: "monthly"
-    labels:
-      - 'skip changelog'
-      - 'dependencies'
-    rebase-strategy: disabled

.github/scripts/check-release.sh

@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-set -eu -o pipefail
-
-check_tag() {
-    local expected=$1
-    local actual=$2
-    local filename=$3
-
-    if [[ $actual != $expected ]]; then
-        echo >&2 "Error: the current tag does not match the version in $filename: found $actual, expected $expected"
-        return 1
-    fi
-}
-
-read_version() {
-    grep '^version = ' | cut -d \" -f 2
-}
-
-if [[ -z "${GITHUB_REF:-}" ]]; then
-    echo >&2 "Error: GITHUB_REF is not set"
-    exit 1
-fi
-
-if [[ ! "$GITHUB_REF" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+(-[a-z0-9]+)?$ ]]; then
-    echo >&2 "Error: GITHUB_REF is not a valid tag: $GITHUB_REF"
-    exit 1
-fi
-
-current_tag=${GITHUB_REF#refs/tags/v}
-ret=0
-
-toml_tag="$(cat Cargo.toml | read_version)"
-check_tag "$current_tag" "$toml_tag" Cargo.toml || ret=1
-
-lock_tag=$(grep -A 1 '^name = "meilisearch-auth"' Cargo.lock | read_version)
-check_tag "$current_tag" "$lock_tag" Cargo.lock || ret=1
-
-if (( ret == 0 )); then
-    echo 'OK'
-fi
-exit $ret

.github/scripts/is-latest-release.sh

@@ -1,48 +0,0 @@
-#!/bin/sh
-
-# Used in our CIs to publish the latest Docker image.
-
-# Checks if the current tag ($GITHUB_REF) corresponds to the latest release tag on GitHub
-# Returns "true" or "false" (as a string).
-
-GITHUB_API='https://api.github.com/repos/meilisearch/meilisearch/releases'
-PNAME='meilisearch'
-
-# FUNCTIONS
-
-# Returns the version of the latest stable version of Meilisearch by setting the $latest variable.
-get_latest() {
-    # temp_file is needed because the grep would start before the download is over
-    temp_file=$(mktemp -q /tmp/$PNAME.XXXXXXXXX)
-    latest_release="$GITHUB_API/latest"
-
-    if [ $? -ne 0 ]; then
-        echo "$0: Can't create temp file."
-        exit 1
-    fi
-
-    if [ -z "$GITHUB_PAT" ]; then
-        curl -s "$latest_release" > "$temp_file" || return 1
-    else
-        curl -H "Authorization: token $GITHUB_PAT" -s "$latest_release" > "$temp_file" || return 1
-    fi
-
-    latest="$(cat "$temp_file" | grep '"tag_name":' | cut -d ':' -f2 | tr -d '"' | tr -d ',' | tr -d ' ')"
-
-    rm -f "$temp_file"
-    return 0
-}
-
-# MAIN
-current_tag="$(echo $GITHUB_REF | tr -d 'refs/tags/')"
-get_latest
-
-if [ "$current_tag" != "$latest" ]; then
-    # The current release tag is not the latest
-    echo "false"
-else
-    # The current release tag is the latest
-    echo "true"
-fi
-
-exit 0

.github/workflows/README.md

@@ -0,0 +1,11 @@
+# GitHub actions workflow for MeiliDB
+
+> **Note:**
+
+> - We do not use [cache](https://github.com/actions/cache) yet but we could use it to speed up CI
+
+## Workflow
+
+- On each pull request, we are triggering `cargo test`.
+- On each commit on master, we are building the latest docker image.
+- On each tag, we are building the tagged docker image and the binaries for MacOS & Ubuntu.

.github/workflows/benchmarks-manual.yml

@@ -1,77 +0,0 @@
-name: Benchmarks (manual)
-
-on:
-  workflow_dispatch:
-    inputs:
-      dataset_name:
-        description: 'The name of the dataset used to benchmark (search_songs, search_wiki, search_geo or indexing)'
-        required: false
-        default: 'search_songs'
-
-env:
-  BENCH_NAME: ${{ github.event.inputs.dataset_name }}
-
-jobs:
-  benchmarks:
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Helper
-      - name: 'README: compare with another benchmark'
-        run: |
-          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
-          echo 'How to compare this benchmark with another one?'
-          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/benchmarks-push-indexing.yml

@@ -1,79 +0,0 @@
-name: Benchmarks of indexing (push)
-
-on:
-  push:
-    branches:
-      - main
-
-env:
-  INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
-  BENCH_NAME: "indexing"
-
-jobs:
-  benchmarks:
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Upload benchmarks to influxdb
-      - name: Upload ${{ steps.file.outputs.basename }}.json to influxDB
-        run: telegraf --config https://eu-central-1-1.aws.cloud2.influxdata.com/api/v2/telegrafs/08b52e34a370b000 --once --debug
-
-      # Helper
-      - name: 'README: compare with another benchmark'
-        run: |
-          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
-          echo 'How to compare this benchmark with another one?'
-          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/benchmarks-push-search-geo.yml

@@ -1,78 +0,0 @@
-name: Benchmarks of search for geo (push)
-
-on:
-  push:
-    branches:
-      - main
-
-env:
-  BENCH_NAME: "search_geo"
-  INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
-
-jobs:
-  benchmarks:
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Upload benchmarks to influxdb
-      - name: Upload ${{ steps.file.outputs.basename }}.json to influxDB
-        run: telegraf --config https://eu-central-1-1.aws.cloud2.influxdata.com/api/v2/telegrafs/08b52e34a370b000 --once --debug
-
-      # Helper
-      - name: 'README: compare with another benchmark'
-        run: |
-          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
-          echo 'How to compare this benchmark with another one?'
-          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/benchmarks-push-search-songs.yml

@@ -1,78 +0,0 @@
-name: Benchmarks of search for songs (push)
-
-on:
-  push:
-    branches:
-      - main
-
-env:
-  BENCH_NAME: "search_songs"
-  INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
-
-jobs:
-  benchmarks:
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Upload benchmarks to influxdb
-      - name: Upload ${{ steps.file.outputs.basename }}.json to influxDB
-        run: telegraf --config https://eu-central-1-1.aws.cloud2.influxdata.com/api/v2/telegrafs/08b52e34a370b000 --once --debug
-
-      # Helper
-      - name: 'README: compare with another benchmark'
-        run: |
-          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
-          echo 'How to compare this benchmark with another one?'
-          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/benchmarks-push-search-wiki.yml

@@ -1,78 +0,0 @@
-name: Benchmarks of search for Wikipedia articles (push)
-
-on:
-  push:
-    branches:
-      - main
-
-env:
-  BENCH_NAME: "search_wiki"
-  INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
-
-jobs:
-  benchmarks:
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Upload benchmarks to influxdb
-      - name: Upload ${{ steps.file.outputs.basename }}.json to influxDB
-        run: telegraf --config https://eu-central-1-1.aws.cloud2.influxdata.com/api/v2/telegrafs/08b52e34a370b000 --once --debug
-
-      # Helper
-      - name: 'README: compare with another benchmark'
-        run: |
-          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
-          echo 'How to compare this benchmark with another one?'
-          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/dependency-issue.yml

@@ -1,24 +0,0 @@
-name: Create issue to upgrade dependencies
-
-on:
-  schedule:
-    # Run the first of the month, every 3 month
-    - cron: '0 0 1 */3 *'
-  workflow_dispatch:
-
-jobs:
-  create-issue:
-    runs-on: ubuntu-latest
-    env:
-      ISSUE_TEMPLATE: issue-template.md
-      GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-    steps:
-    - uses: actions/checkout@v3
-    - name: Download the issue template
-      run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/dependency-issue.md > $ISSUE_TEMPLATE
-    - name: Create issue
-      run: |
-        gh issue create \
-          --title 'Upgrade dependencies' \
-          --label 'dependencies,maintenance' \
-          --body-file $ISSUE_TEMPLATE

.github/workflows/flaky-tests.yml

@@ -1,32 +0,0 @@
-name: Look for flaky tests
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: "0 12 * * FRI" # Every Friday at 12:00PM
-
-jobs:
-  flaky:
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
-    steps:
-    - uses: actions/checkout@v3
-    - name: Install needed dependencies
-      run: |
-        apt-get update && apt-get install -y curl
-        apt-get install build-essential -y
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
-    - name: Install cargo-flaky
-      run: cargo install cargo-flaky
-    - name: Run cargo flaky in the dumps
-      run: cd dump; cargo flaky -i 100 --release
-    - name: Run cargo flaky in the index-scheduler
-      run: cd index-scheduler; cargo flaky -i 100 --release
-    - name: Run cargo flaky in the auth
-      run: cd meilisearch-auth; cargo flaky -i 100 --release
-    - name: Run cargo flaky in meilisearch
-      run: cd meilisearch; cargo flaky -i 100 --release

.github/workflows/fuzzer-indexing.yml

@@ -1,24 +0,0 @@
-name: Run the indexing fuzzer
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  fuzz:
-    name: Setup the action
-    runs-on: ubuntu-latest
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Run benchmarks
-      - name: Run the fuzzer
-        run: |
-          cargo run --release --bin fuzz-indexing

.github/workflows/latest-git-tag.yml

@@ -1,29 +0,0 @@
-# Create or update a latest git tag when releasing a stable version of Meilisearch
-name: Update latest git tag
-on:
-  workflow_dispatch:
-  release:
-    types: [released]
-
-jobs:
-  check-version:
-    name: Check the version validity
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check release validity
-        if: github.event_name == 'release'
-        run: bash .github/scripts/check-release.sh
-
-  update-latest-tag:
-    runs-on: ubuntu-latest
-    needs: check-version
-    steps:
-      - uses: actions/checkout@v3
-      - uses: rickstaa/action-create-tag@v1
-        with:
-          tag: "latest"
-          message: "Latest stable release of Meilisearch"
-          # Move the tag if `latest` already exists
-          force_push_tag: true
-          github_token: ${{ secrets.MEILI_BOT_GH_PAT }}

.github/workflows/milestone-workflow.yml

@@ -1,154 +0,0 @@
-name: Milestone's workflow
-
-# /!\ No git flow are handled here
-
-# For each Milestone created (not opened!), and if the release is NOT a patch release (only the patch changed)
-# - the roadmap issue is created, see https://github.com/meilisearch/engine-team/blob/main/issue-templates/roadmap-issue.md
-# - the changelog issue is created, see https://github.com/meilisearch/engine-team/blob/main/issue-templates/changelog-issue.md
-
-# For each Milestone closed
-# - the `release_version` label is created
-# - this label is applied to all issues/PRs in the Milestone
-
-on:
-  milestone:
-    types: [created, closed]
-
-env:
-  MILESTONE_VERSION: ${{ github.event.milestone.title }}
-  MILESTONE_URL: ${{ github.event.milestone.html_url }}
-  MILESTONE_DUE_ON: ${{ github.event.milestone.due_on }}
-  GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-
-jobs:
-
-# -----------------
-# MILESTONE CREATED
-# -----------------
-
-  get-release-version:
-    if: github.event.action == 'created'
-    runs-on: ubuntu-latest
-    outputs:
-      is-patch: ${{ steps.check-patch.outputs.is-patch }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check if this release is a patch release only
-        id: check-patch
-        run: |
-          echo version: $MILESTONE_VERSION
-          if [[ $MILESTONE_VERSION =~ ^v[0-9]+\.[0-9]+\.0$ ]]; then
-            echo 'This is NOT a patch release'
-            echo "is-patch=false" >> $GITHUB_OUTPUT
-          elif [[ $MILESTONE_VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo 'This is a patch release'
-            echo "is-patch=true" >> $GITHUB_OUTPUT
-          else
-            echo "Not a valid format of release, check the Milestone's title."
-            echo 'Should be vX.Y.Z'
-            exit 1
-          fi
-
-  create-roadmap-issue:
-    needs: get-release-version
-    # Create the roadmap issue if the release is not only a patch release
-    if: github.event.action == 'created' && needs.get-release-version.outputs.is-patch == 'false'
-    runs-on: ubuntu-latest
-    env:
-      ISSUE_TEMPLATE: issue-template.md
-    steps:
-      - uses: actions/checkout@v3
-      - name: Download the issue template
-        run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/roadmap-issue.md > $ISSUE_TEMPLATE
-      - name: Replace all empty occurrences in the templates
-        run: |
-          # Replace all <<version>> occurrences
-          sed -i "s/<<version>>/$MILESTONE_VERSION/g" $ISSUE_TEMPLATE
-
-          # Replace all <<milestone_id>> occurrences
-          milestone_id=$(echo $MILESTONE_URL | cut -d '/' -f 7)
-          sed -i "s/<<milestone_id>>/$milestone_id/g" $ISSUE_TEMPLATE
-
-          # Replace release date if exists
-          if [[ ! -z $MILESTONE_DUE_ON ]]; then
-            date=$(echo $MILESTONE_DUE_ON | cut -d 'T' -f 1)
-            sed -i "s/Release date\: 20XX-XX-XX/Release date\: $date/g" $ISSUE_TEMPLATE
-          fi
-      - name: Create the issue
-        run: |
-          gh issue create \
-            --title "$MILESTONE_VERSION ROADMAP" \
-            --label 'epic,impacts docs,impacts integrations,impacts cloud' \
-            --body-file $ISSUE_TEMPLATE \
-            --milestone $MILESTONE_VERSION
-
-  create-changelog-issue:
-    needs: get-release-version
-    # Create the changelog issue if the release is not only a patch release
-    if: github.event.action == 'created' && needs.get-release-version.outputs.is-patch == 'false'
-    runs-on: ubuntu-latest
-    env:
-      ISSUE_TEMPLATE: issue-template.md
-    steps:
-      - uses: actions/checkout@v3
-      - name: Download the issue template
-        run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/changelog-issue.md > $ISSUE_TEMPLATE
-      - name: Replace all empty occurrences in the templates
-        run: |
-          # Replace all <<version>> occurrences
-          sed -i "s/<<version>>/$MILESTONE_VERSION/g" $ISSUE_TEMPLATE
-
-          # Replace all <<milestone_id>> occurrences
-          milestone_id=$(echo $MILESTONE_URL | cut -d '/' -f 7)
-          sed -i "s/<<milestone_id>>/$milestone_id/g" $ISSUE_TEMPLATE
-      - name: Create the issue
-        run: |
-          gh issue create \
-            --title "Create release changelogs for $MILESTONE_VERSION" \
-            --label 'impacts docs,documentation' \
-            --body-file $ISSUE_TEMPLATE \
-            --milestone $MILESTONE_VERSION \
-            --assignee curquiza
-
-# ----------------
-# MILESTONE CLOSED
-# ----------------
-
-  create-release-label:
-    if: github.event.action == 'closed'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Create the ${{ env.MILESTONE_VERSION }} label
-        run: |
-          label_description="PRs/issues solved in $MILESTONE_VERSION"
-          if [[ ! -z $MILESTONE_DUE_ON ]]; then
-            date=$(echo $MILESTONE_DUE_ON | cut -d 'T' -f 1)
-            label_description="$label_description released on $date"
-          fi
-
-          gh api repos/meilisearch/meilisearch/labels \
-            --method POST \
-            -H "Accept: application/vnd.github+json" \
-            -f name="$MILESTONE_VERSION" \
-            -f description="$label_description" \
-            -f color='ff5ba3'
-
-  labelize-all-milestone-content:
-    if: github.event.action == 'closed'
-    needs: create-release-label
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Add label ${{ env.MILESTONE_VERSION }} to all PRs in the Milestone
-        run: |
-          prs=$(gh pr list --search milestone:"$MILESTONE_VERSION" --limit 1000 --state all --json number --template '{{range .}}{{tablerow (printf "%v" .number)}}{{end}}')
-          for pr in $prs; do
-              gh pr edit $pr --add-label $MILESTONE_VERSION
-          done
-      - name: Add label ${{ env.MILESTONE_VERSION }} to all issues in the Milestone
-        run: |
-          issues=$(gh issue list --search milestone:"$MILESTONE_VERSION" --limit 1000 --state all --json number --template '{{range .}}{{tablerow (printf "%v" .number)}}{{end}}')
-          for issue in $issues; do
-              gh issue edit $issue --add-label $MILESTONE_VERSION
-          done

.github/workflows/publish-apt-brew-pkg.yml

@@ -1,57 +0,0 @@
-name: Publish to APT & Homebrew
-
-on:
-  release:
-    types: [released]
-
-jobs:
-  check-version:
-    name: Check the version validity
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Check release validity
-        run: bash .github/scripts/check-release.sh
-
-  debian:
-    name: Publish debian packagge
-    runs-on: ubuntu-latest
-    needs: check-version
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27
-      image: ubuntu:18.04
-    steps:
-    - name: Install needed dependencies
-      run: |
-        apt-get update && apt-get install -y curl
-        apt-get install build-essential -y
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
-    - name: Install cargo-deb
-      run: cargo install cargo-deb
-    - uses: actions/checkout@v3
-    - name: Build deb package
-      run: cargo deb -p meilisearch -o target/debian/meilisearch.deb
-    - name: Upload debian pkg to release
-      uses: svenstaro/upload-release-action@2.6.1
-      with:
-        repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
-        file: target/debian/meilisearch.deb
-        asset_name: meilisearch.deb
-        tag: ${{ github.ref }}
-    - name: Upload debian pkg to apt repository
-      run: curl -F package=@target/debian/meilisearch.deb https://${{ secrets.GEMFURY_PUSH_TOKEN }}@push.fury.io/meilisearch/
-
-  homebrew:
-    name: Bump Homebrew formula
-    runs-on: ubuntu-latest
-    needs: check-version
-    steps:
-      - name: Create PR to Homebrew
-        uses: mislav/bump-homebrew-formula-action@v2
-        with:
-          formula-name: meilisearch
-        env:
-          COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }}

.github/workflows/publish-binaries.yml

@@ -1,191 +1,36 @@
 name: Publish binaries to GitHub release
 
 on:
-  workflow_dispatch:
-  schedule:
-    - cron: '0 2 * * *' # Every day at 2:00am
-  release:
-    types: [published]
+  push:
+    tags:
+      - '*'
 
 jobs:
-  check-version:
-    name: Check the version validity
-    runs-on: ubuntu-latest
-    # No need to check the version for dry run (cron)
-    steps:
-      - uses: actions/checkout@v3
-      # Check if the tag has the v<nmumber>.<number>.<number> format.
-      # If yes, it means we are publishing an official release.
-      # If no, we are releasing a RC, so no need to check the version.
-      - name: Check tag format
-        if: github.event_name == 'release'
-        id: check-tag-format
-        run: |
-          escaped_tag=$(printf "%q" ${{ github.ref_name }})
-
-          if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo "stable=true" >> $GITHUB_OUTPUT
-          else
-            echo "stable=false" >> $GITHUB_OUTPUT
-          fi
-      - name: Check release validity
-        if: github.event_name == 'release' && steps.check-tag-format.outputs.stable == 'true'
-        run: bash .github/scripts/check-release.sh
-
-  publish-linux:
-    name: Publish binary for Linux
-    runs-on: ubuntu-latest
-    needs: check-version
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27
-      image: ubuntu:18.04
-    steps:
-    - uses: actions/checkout@v3
-    - name: Install needed dependencies
-      run: |
-        apt-get update && apt-get install -y curl
-        apt-get install build-essential -y
-    - uses: actions-rs/toolchain@v1
-      with:
-        toolchain: stable
-        override: true
-    - name: Build
-      run: cargo build --release --locked
-    # No need to upload binaries for dry run (cron)
-    - name: Upload binaries to release
-      if: github.event_name == 'release'
-      uses: svenstaro/upload-release-action@2.6.1
-      with:
-        repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
-        file: target/release/meilisearch
-        asset_name: meilisearch-linux-amd64
-        tag: ${{ github.ref }}
-
-  publish-macos-windows:
-    name: Publish binary for ${{ matrix.os }}
+  publish:
+    name: Publish for ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
-    needs: check-version
     strategy:
-      fail-fast: false
       matrix:
-        os: [macos-12, windows-2022]
+        os: [ubuntu-latest, macos-latest]
         include:
-          - os: macos-12
+          - os: ubuntu-latest
+            artifact_name: meilisearch
+            asset_name: meilisearch-linux-amd64
+          - os: macos-latest
             artifact_name: meilisearch
             asset_name: meilisearch-macos-amd64
-          - os: windows-2022
-            artifact_name: meilisearch.exe
-            asset_name: meilisearch-windows-amd64.exe
+
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions-rs/toolchain@v1
+    - uses: hecrj/setup-rust-action@master
       with:
-        toolchain: stable
-        override: true
+        rust-version: stable
+    - uses: actions/checkout@v1
     - name: Build
       run: cargo build --release --locked
-    # No need to upload binaries for dry run (cron)
     - name: Upload binaries to release
-      if: github.event_name == 'release'
-      uses: svenstaro/upload-release-action@2.6.1
+      uses: svenstaro/upload-release-action@v1-release
       with:
-        repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
+        repo_token: ${{ secrets.GITHUB_TOKEN }}
         file: target/release/${{ matrix.artifact_name }}
         asset_name: ${{ matrix.asset_name }}
         tag: ${{ github.ref }}
-
-  publish-macos-apple-silicon:
-    name: Publish binary for macOS silicon
-    runs-on: macos-12
-    needs: check-version
-    strategy:
-      matrix:
-        include:
-          - target: aarch64-apple-darwin
-            asset_name: meilisearch-macos-apple-silicon
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Installing Rust toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          target: ${{ matrix.target }}
-          override: true
-      - name: Cargo build
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --release --target ${{ matrix.target }}
-      - name: Upload the binary to release
-        # No need to upload binaries for dry run (cron)
-        if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.6.1
-        with:
-          repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
-          file: target/${{ matrix.target }}/release/meilisearch
-          asset_name: ${{ matrix.asset_name }}
-          tag: ${{ github.ref }}
-
-  publish-aarch64:
-    name: Publish binary for aarch64
-    runs-on: ubuntu-latest
-    needs: check-version
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27
-      image: ubuntu:18.04
-    strategy:
-      matrix:
-        include:
-          - target: aarch64-unknown-linux-gnu
-            asset_name: meilisearch-linux-aarch64
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update -y && apt upgrade -y
-          apt-get install -y curl build-essential gcc-aarch64-linux-gnu
-      - name: Set up Docker for cross compilation
-        run: |
-          apt-get install -y curl apt-transport-https ca-certificates software-properties-common
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
-          add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-          apt-get update -y && apt-get install -y docker-ce
-      - name: Installing Rust toolchain
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          profile: minimal
-          target: ${{ matrix.target }}
-          override: true
-      - name: Configure target aarch64 GNU
-        ## Environment variable is not passed using env:
-        ## LD gold won't work with MUSL
-        # env:
-        #   JEMALLOC_SYS_WITH_LG_PAGE: 16
-        #   RUSTFLAGS: '-Clink-arg=-fuse-ld=gold'
-        run: |
-          echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
-          echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
-          echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
-      - name: Cargo build
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          use-cross: true
-          args: --release --target ${{ matrix.target }}
-        env:
-          CROSS_DOCKER_IN_DOCKER: true
-      - name: List target output files
-        run: ls -lR ./target
-      - name: Upload the binary to release
-        # No need to upload binaries for dry run (cron)
-        if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.6.1
-        with:
-          repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
-          file: target/${{ matrix.target }}/release/meilisearch
-          asset_name: ${{ matrix.asset_name }}
-          tag: ${{ github.ref }}

.github/workflows/publish-docker-images.yml

@@ -1,105 +0,0 @@
-name: Publish images to Docker Hub
-
-on:
-  push:
-    # Will run for every tag pushed except `latest`
-    # When the `latest` git tag is created with this [CI](../latest-git-tag.yml)
-    # we don't need to create a Docker `latest` image again.
-    # The `latest` Docker image push is already done in this CI when releasing a stable version of Meilisearch.
-    tags-ignore:
-      - latest
-  # Both `schedule` and `workflow_dispatch` build the nightly tag
-  schedule:
-    - cron: '0 23 * * *' # Every day at 11:00pm
-  workflow_dispatch:
-
-jobs:
-  docker:
-    runs-on: docker
-    steps:
-      - uses: actions/checkout@v3
-
-      # If we are running a cron or manual job ('schedule' or 'workflow_dispatch' event), it means we are publishing the `nightly` tag, so not considered stable.
-      # If we have pushed a tag, and the tag has the v<nmumber>.<number>.<number> format, it means we are publishing an official release, so considered stable.
-      # In this situation, we need to set `output.stable` to create/update the following tags (additionally to the `vX.Y.Z` Docker tag):
-      # - a `vX.Y` (without patch version) Docker tag
-      # - a `latest` Docker tag
-      # For any other tag pushed, this is not considered stable.
-      - name: Define if stable and latest release
-        id: check-tag-format
-        env:
-          # To avoid request limit with the .github/scripts/is-latest-release.sh script
-          GITHUB_PATH: ${{ secrets.MEILI_BOT_GH_PAT }}
-        run: |
-          escaped_tag=$(printf "%q" ${{ github.ref_name }})
-          echo "latest=false" >> $GITHUB_OUTPUT
-
-          if [[ ${{ github.event_name }} != 'push' ]]; then
-            echo "stable=false" >> $GITHUB_OUTPUT
-          elif [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo "stable=true" >> $GITHUB_OUTPUT
-            echo "latest=$(sh .github/scripts/is-latest-release.sh)" >> $GITHUB_OUTPUT
-          else
-            echo "stable=false" >> $GITHUB_OUTPUT
-          fi
-
-      # Check only the validity of the tag for stable releases (not for pre-releases or other tags)
-      - name: Check release validity
-        if: steps.check-tag-format.outputs.stable == 'true'
-        run: bash .github/scripts/check-release.sh
-
-      - name: Set build-args for Docker buildx
-        id: build-metadata
-        run: |
-          # Extract commit date
-          commit_date=$(git show -s --format=%cd --date=iso-strict ${{ github.sha }})
-
-          echo "date=$commit_date" >> $GITHUB_OUTPUT
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
-
-      - name: Login to Docker Hub
-        uses: docker/login-action@v2
-        with:
-          username: ${{ secrets.DOCKERHUB_USERNAME }}
-          password: ${{ secrets.DOCKERHUB_TOKEN }}
-
-      - name: Docker meta
-        id: meta
-        uses: docker/metadata-action@v4
-        with:
-          images: getmeili/meilisearch
-          # Prevent `latest` to be updated for each new tag pushed.
-          # We need latest and `vX.Y` tags to only be pushed for the stable Meilisearch releases.
-          flavor: latest=false
-          tags: |
-            type=ref,event=tag
-            type=raw,value=nightly,enable=${{ github.event_name != 'push' }}
-            type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
-            type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
-
-      - name: Build and push
-        uses: docker/build-push-action@v4
-        with:
-          push: true
-          platforms: linux/amd64,linux/arm64
-          tags: ${{ steps.meta.outputs.tags }}
-          build-args: |
-            COMMIT_SHA=${{ github.sha }}
-            COMMIT_DATE=${{ steps.build-metadata.outputs.date }}
-            GIT_TAG=${{ github.ref_name }}
-
-      # /!\ Don't touch this without checking with Cloud team
-      - name: Send CI information to Cloud team
-        # Do not send if nightly build (i.e. 'schedule' or 'workflow_dispatch' event)
-        if: github.event_name == 'push'
-        uses: peter-evans/repository-dispatch@v2
-        with:
-          token: ${{ secrets.MEILI_BOT_GH_PAT }}
-          repository: meilisearch/meilisearch-cloud
-          event-type: cloud-docker-build
-          client-payload: '{ "meilisearch_version": "${{ github.ref_name }}", "stable": "${{ steps.check-tag-format.outputs.stable }}" }'

.github/workflows/publish-docker-latest.yml

@@ -0,0 +1,19 @@
+---
+on:
+  push:
+    branches:
+      - master
+
+name: Publish latest image to Docker Hub
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Publish to Registry
+        uses: elgohr/Publish-Docker-Github-Action@master
+        with:
+          name: getmeili/meilisearch
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/publish-docker-tag.yml

@@ -0,0 +1,20 @@
+---
+on:
+  push:
+    tags:
+      - '*'
+
+name: Publish tagged image to Docker Hub
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - name: Publish to Registry
+        uses: elgohr/Publish-Docker-Github-Action@master
+        with:
+          name: getmeili/meilisearch
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          tag_names: true

.github/workflows/sdks-tests.yml

@@ -1,226 +0,0 @@
-# If any test fails, the engine team should ensure the "breaking" changes are expected and contact the integration team
-name: SDKs tests
-
-on:
-  workflow_dispatch:
-    inputs:
-      docker_image:
-        description: 'The Meilisearch Docker image used'
-        required: false
-        default: nightly
-  schedule:
-    - cron: "0 6 * * MON" # Every Monday at 6:00AM
-
-env:
-  MEILI_MASTER_KEY: 'masterKey'
-  MEILI_NO_ANALYTICS: 'true'
-
-jobs:
-  define-docker-image:
-    runs-on: ubuntu-latest
-    outputs:
-      docker-image: ${{ steps.define-image.outputs.docker-image }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Define the Docker image we need to use
-        id: define-image
-        run: |
-          event=${{ github.event_name }}
-          echo "docker-image=nightly" >> $GITHUB_OUTPUT
-          if [[ $event == 'workflow_dispatch' ]]; then
-            echo "docker-image=${{ github.event.inputs.docker_image }}" >> $GITHUB_OUTPUT
-          fi
-
-  meilisearch-js-tests:
-    needs: define-docker-image
-    name: JS SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-js
-      - name: Setup node
-        uses: actions/setup-node@v3
-        with:
-          cache: 'yarn'
-      - name: Install dependencies
-        run: yarn --dev
-      - name: Run tests
-        run: yarn test
-      - name: Build project
-        run: yarn build
-      - name: Run ESM env
-        run: yarn test:env:esm
-      - name: Run Node.js env
-        run: yarn test:env:nodejs
-      - name: Run node typescript env
-        run: yarn test:env:node-ts
-      - name: Run Browser env
-        run: yarn test:env:browser
-
-  instant-meilisearch-tests:
-    needs: define-docker-image
-    name: instant-meilisearch tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/instant-meilisearch
-      - name: Setup node
-        uses: actions/setup-node@v3
-        with:
-          cache: yarn
-      - name: Install dependencies
-        run: yarn install
-      - name: Run tests
-        run: yarn test
-      - name: Build all the playgrounds and the packages
-        run: yarn build
-
-  meilisearch-php-tests:
-    needs: define-docker-image
-    name: PHP SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-php
-      - name: Install PHP
-        uses: shivammathur/setup-php@v2
-        with:
-          coverage: none
-      - name: Validate composer.json and composer.lock
-        run: composer validate
-      - name: Install dependencies
-        run: |
-          composer remove --dev friendsofphp/php-cs-fixer --no-update --no-interaction
-          composer update --prefer-dist --no-progress
-      - name: Run test suite - default HTTP client (Guzzle 7)
-        run: |
-          sh scripts/tests.sh
-          composer remove --dev guzzlehttp/guzzle http-interop/http-factory-guzzle
-
-  meilisearch-python-tests:
-    needs: define-docker-image
-    name: Python SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-python
-      - name: Set up Python
-        uses: actions/setup-python@v4
-      - name: Install pipenv
-        uses: dschep/install-pipenv-action@v1
-      - name: Install dependencies
-        run: pipenv install --dev --python=${{ matrix.python-version }}
-      - name: Test with pytest
-        run: pipenv run pytest
-
-  meilisearch-go-tests:
-    needs: define-docker-image
-    name: Go SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-go
-      - name: Get dependencies
-        run: |
-          go get -v -t -d ./...
-          if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-          fi
-      - name: Run integration tests
-        run: go test -v ./...
-
-  meilisearch-ruby-tests:
-    needs: define-docker-image
-    name: Ruby SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-ruby
-      - name: Set up Ruby 3
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3
-      - name: Install ruby dependencies
-        run: bundle install --with test
-      - name: Run test suite
-        run: bundle exec rspec
-
-  meilisearch-rust-tests:
-    needs: define-docker-image
-    name: Rust SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-rust
-      - name: Build
-        run: cargo build --verbose
-      - name: Run tests
-        run: cargo test --verbose

.github/workflows/test-suite.yml

@@ -1,194 +0,0 @@
-name: Test suite
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Everyday at 5:00am
-    - cron: '0 5 * * *'
-  pull_request:
-  push:
-    # trying and staging branches are for Bors config
-    branches:
-      - trying
-      - staging
-
-env:
-  CARGO_TERM_COLOR: always
-  RUST_BACKTRACE: 1
-  RUSTFLAGS: "-D warnings"
-
-jobs:
-  test-linux:
-    name: Tests on ubuntu-18.04
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update && apt-get install -y curl
-          apt-get install build-essential -y
-      - name: Setup test with Rust stable
-        if: github.event_name != 'schedule'
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-      - name: Setup test with Rust nightly
-        if: github.event_name == 'schedule'
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.0
-      - name: Run cargo check without any default features
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --locked --release --no-default-features --all
-      - name: Run cargo test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked --release --all
-
-  test-others:
-    name: Tests on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-12, windows-2022]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.0
-      - name: Run cargo check without any default features
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --locked --release --no-default-features --all
-      - name: Run cargo test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked --release --all
-
-  test-all-features:
-    name: Tests all features on cron schedule only
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
-    if: github.event_name == 'schedule'
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update
-          apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-      - name: Run cargo build with all features
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --workspace --locked --release --all-features
-      - name: Run cargo test with all features
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace --locked --release --all-features
-
-  test-disabled-tokenization:
-    name: Test disabled tokenization
-    runs-on: ubuntu-latest
-    container:
-      image: ubuntu:18.04
-    if: github.event_name == 'schedule'
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update
-          apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-      - name: Run cargo tree without default features and check lindera is not present
-        run: |
-          cargo tree -f '{p} {f}' -e normal --no-default-features | grep lindera -vqz
-      - name: Run cargo tree with default features and check lindera is pressent
-        run: |
-          cargo tree -f '{p} {f}' -e normal | grep lindera -qz
-
-  # We run tests in debug also, to make sure that the debug_assertions are hit
-  test-debug:
-    name: Run tests in debug
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update && apt-get install -y curl
-          apt-get install build-essential -y
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.0
-      - name: Run tests in debug
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked --all
-
-  clippy:
-    name: Run Clippy
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: 1.69.0
-          override: true
-          components: clippy
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.0
-      - name: Run cargo clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all-targets -- --deny warnings
-
-  fmt:
-    name: Run Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly
-          override: true
-          components: rustfmt
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.5.0
-      - name: Run cargo fmt
-        # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
-        # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate
-        # we are going to create an empty file where rustfmt expects it.
-        run: |
-          echo -ne "\n" > benchmarks/benches/datasets_paths.rs
-          cargo fmt --all -- --check

.github/workflows/test.yml

@@ -0,0 +1,21 @@
+---
+on: [pull_request]
+
+name: Cargo test
+
+jobs:
+  check:
+    name: MeiliSearch
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - name: Run cargo test
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --locked

.github/workflows/update-cargo-toml-version.yml

@@ -1,47 +0,0 @@
-name: Update Meilisearch version in Cargo.toml
-
-on:
-  workflow_dispatch:
-    inputs:
-      new_version:
-        description: 'The new version (vX.Y.Z)'
-        required: true
-
-env:
-  NEW_VERSION: ${{ github.event.inputs.new_version }}
-  NEW_BRANCH: update-version-${{ github.event.inputs.new_version }}
-  GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-
-jobs:
-  update-version-cargo-toml:
-    name: Update version in Cargo.toml
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-      - name: Install sd
-        run: cargo install sd
-      - name: Update Cargo.toml file
-        run: |
-          raw_new_version=$(echo $NEW_VERSION | cut -d 'v' -f 2)
-          new_string="version = \"$raw_new_version\""
-          sd '^version = "\d+.\d+.\w+"$' "$new_string" Cargo.toml
-      - name: Build Meilisearch to update Cargo.lock
-        run: cargo build
-      - name: Commit and push the changes to the ${{ env.NEW_BRANCH }} branch
-        uses: EndBug/add-and-commit@v9
-        with:
-          message: "Update version for the next release (${{ env.NEW_VERSION }}) in Cargo.toml"
-          new_branch: ${{ env.NEW_BRANCH }}
-      - name: Create the PR pointing to ${{ github.ref_name }}
-        run: |
-          gh pr create \
-            --title "Update version for the next release ($NEW_VERSION) in Cargo.toml" \
-            --body '⚠️ This PR is automatically generated. Check the new version is the expected one and Cargo.lock has been updated before merging.' \
-            --label 'skip changelog' \
-            --milestone $NEW_VERSION \
-            --base $GITHUB_REF_NAME

.gitignore

@@ -1,20 +1,7 @@
-.idea/
-.vscode/
 /target
+meilisearch-core/target
 **/*.csv
 **/*.json_lines
 **/*.rs.bk
 /*.mdb
 /query-history.txt
-/data.ms
-/snapshots
-/dumps
-
-# Snapshots
-## ... large
-*.full.snap
-##  ... unreviewed
-*.snap.new
-
-# Fuzzcheck data for the facet indexing fuzz test
-milli/fuzz/update::facet::incremental::fuzz::fuzz/

.rustfmt.toml

@@ -1,5 +0,0 @@
-unstable_features = true
-
-use_small_heuristics = "max"
-imports_granularity = "Module"
-group_imports = "StdExternalCrate"

CODE_OF_CONDUCT.md

@@ -1,76 +0,0 @@
-# Contributor Covenant Code of Conduct
-
-## Our Pledge
-
-In the interest of fostering an open and welcoming environment, we as
-contributors and maintainers pledge to making participation in our project and
-our community a harassment-free experience for everyone, regardless of age, body
-size, disability, ethnicity, sex characteristics, gender identity and expression,
-level of experience, education, socio-economic status, nationality, personal
-appearance, race, religion, or sexual identity and orientation.
-
-## Our Standards
-
-Examples of behavior that contributes to creating a positive environment
-include:
-
-* Using welcoming and inclusive language
-* Being respectful of differing viewpoints and experiences
-* Gracefully accepting constructive criticism
-* Focusing on what is best for the community
-* Showing empathy towards other community members
-
-Examples of unacceptable behavior by participants include:
-
-* The use of sexualized language or imagery and unwelcome sexual attention or
- advances
-* Trolling, insulting/derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or electronic
- address, without explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
- professional setting
-
-## Our Responsibilities
-
-Project maintainers are responsible for clarifying the standards of acceptable
-behavior and are expected to take appropriate and fair corrective action in
-response to any instances of unacceptable behavior.
-
-Project maintainers have the right and responsibility to remove, edit, or
-reject comments, commits, code, wiki edits, issues, and other contributions
-that are not aligned to this Code of Conduct, or to ban temporarily or
-permanently any contributor for other behaviors that they deem inappropriate,
-threatening, offensive, or harmful.
-
-## Scope
-
-This Code of Conduct applies both within project spaces and in public spaces
-when an individual is representing the project or its community. Examples of
-representing a project or community include using an official project e-mail
-address, posting via an official social media account, or acting as an appointed
-representative at an online or offline event. Representation of a project may be
-further defined and clarified by project maintainers.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported by contacting the project team at bonjour@meilisearch.com. All
-complaints will be reviewed and investigated and will result in a response that
-is deemed necessary and appropriate to the circumstances. The project team is
-obligated to maintain confidentiality with regard to the reporter of an incident.
-Further details of specific enforcement policies may be posted separately.
-
-Project maintainers who do not follow or enforce the Code of Conduct in good
-faith may face temporary or permanent repercussions as determined by other
-members of the project's leadership.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
-available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see
-https://www.contributor-covenant.org/faq

CONTRIBUTING.md

@@ -1,139 +0,0 @@
-# Contributing
-
-First, thank you for contributing to Meilisearch! The goal of this document is to provide everything you need to start contributing to Meilisearch.
-
-Remember that there are many ways to contribute other than writing code: writing [tutorials or blog posts](https://github.com/meilisearch/awesome-meilisearch), improving [the documentation](https://github.com/meilisearch/documentation), submitting [bug reports](https://github.com/meilisearch/meilisearch/issues/new?assignees=&labels=&template=bug_report.md&title=) and [feature requests](https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal)...
-
-The code in this repository is only concerned with managing multiple indexes, handling the update store, and exposing an HTTP API. Search and indexation are the domain of our core engine, [`milli`](https://github.com/meilisearch/milli), while tokenization is handled by [our `charabia` library](https://github.com/meilisearch/charabia/).
-
-If Meilisearch does not offer optimized support for your language, please consider contributing to `charabia` by following the [CONTRIBUTING.md file](https://github.com/meilisearch/charabia/blob/main/CONTRIBUTING.md) and integrating your intended normalizer/segmenter.
-
-## Table of Contents
-
-- [Assumptions](#assumptions)
-- [How to Contribute](#how-to-contribute)
-- [Development Workflow](#development-workflow)
-- [Git Guidelines](#git-guidelines)
-- [Release Process (for internal team only)](#release-process-for-internal-team-only)
-
-## Assumptions
-
-1. **You're familiar with [GitHub](https://github.com) and the [Pull Requests (PR)](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests) workflow.**
-2. **You've read the Meilisearch [documentation](https://www.meilisearch.com/docs).**
-3. **You know about the [Meilisearch community on Discord](https://discord.meilisearch.com).
-   Please use this for help.**
-
-## How to Contribute
-
-1. Ensure your change has an issue! Find an
-   [existing issue](https://github.com/meilisearch/meilisearch/issues/) or [open a new issue](https://github.com/meilisearch/meilisearch/issues/new).
-   * This is where you can get a feel if the change will be accepted or not.
-2. Once approved, [fork the Meilisearch repository](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) in your own GitHub account.
-3. [Create a new Git branch](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-and-deleting-branches-within-your-repository)
-4. Review the [Development Workflow](#development-workflow) section that describes the steps to maintain the repository.
-5. Make your changes on your branch.
-6. [Submit the branch as a Pull Request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) pointing to the `main` branch of the Meilisearch repository. A maintainer should comment and/or review your Pull Request within a few days. Although depending on the circumstances, it may take longer.
-
-## Development Workflow
-
-### Setup and run Meilisearch
-
-```bash
-cargo run --release
-```
-
-We recommend using the `--release` flag to test the full performance of Meilisearch.
-
-### Test
-
-```bash
-cargo test
-```
-
-This command will be triggered to each PR as a requirement for merging it.
-
-#### Snapshot-based tests
-
-We are using [insta](https://insta.rs) to perform snapshot-based testing.
-We recommend using the insta tooling (such as `cargo-insta`) to update the snapshots if they change following a PR.
-
-New tests should use insta where possible rather than manual `assert` statements.
-
-Furthermore, we provide some macros on top of insta, notably a way to use snapshot hashes instead of inline snapshots, saving a lot of space in the repository.
-
-To effectively debug snapshot-based hashes, we recommend you export the `MEILI_TEST_FULL_SNAPS` environment variable so that snapshot are fully created locally:
-
-```
-export MEILI_TEST_FULL_SNAPS=true # add this to your .bashrc, .zshrc, ...
-```
-
-#### Test troubleshooting
-
-If you get a "Too many open files" error you might want to increase the open file limit using this command:
-
-```bash
-ulimit -Sn 3000
-```
-
-## Git Guidelines
-
-### Git Branches
-
-All changes must be made in a branch and submitted as PR.
-
-We do not enforce any branch naming style, but please use something descriptive of your changes.
-
-### Git Commits
-
-As minimal requirements, your commit message should:
-- be capitalized
-- not finish by a dot or any other punctuation character (!,?)
-- start with a verb so that we can read your commit message this way: "This commit will ...", where "..." is the commit message.
-  e.g.: "Fix the home page button" or "Add more tests for create_index method"
-
-We don't follow any other convention, but if you want to use one, we recommend [the Chris Beams one](https://chris.beams.io/posts/git-commit/).
-
-### GitHub Pull Requests
-
-Some notes on GitHub PRs:
-
-- All PRs must be reviewed and approved by at least one maintainer.
-- The PR title should be accurate and descriptive of the changes.
-- [Convert your PR as a draft](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/changing-the-stage-of-a-pull-request) if your changes are a work in progress: no one will review it until you pass your PR as ready for review.<br>
-  The draft PRs are recommended when you want to show that you are working on something and make your work visible.
-- The branch related to the PR must be **up-to-date with `main`** before merging. Fortunately, this project uses [Bors](https://github.com/bors-ng/bors-ng) to automatically enforce this requirement without the PR author having to rebase manually.
-
-## Release Process (for internal team only)
-
-Meilisearch tools follow the [Semantic Versioning Convention](https://semver.org/).
-
-### Automation to rebase and Merge the PRs
-
-This project integrates a bot that helps us manage pull requests merging.<br>
-_[Read more about this](https://github.com/meilisearch/integration-guides/blob/main/resources/bors.md)._
-
-### How to Publish a new Release
-
-The full Meilisearch release process is described in [this guide](https://github.com/meilisearch/engine-team/blob/main/resources/meilisearch-release.md). Please follow it carefully before doing any release.
-
-### How to publish a prototype
-
-Depending on the developed feature, you might need to provide a prototyped version of Meilisearch to make it easier to test by the users.
-
-This happens in two steps:
-- [Release the prototype](https://github.com/meilisearch/engine-team/blob/main/resources/prototypes.md#how-to-publish-a-prototype)
-- [Communicate about it](https://github.com/meilisearch/engine-team/blob/main/resources/prototypes.md#communication)
-
-### Release assets
-
-For each release, the following assets are created:
-- Binaries for different platforms (Linux, MacOS, Windows and ARM architectures) are attached to the GitHub release
-- Binaries are pushed to HomeBrew and APT (not published for RC)
-- Docker tags are created/updated:
-  - `vX.Y.Z`
-  - `vX.Y` (not published for RC)
-  - `latest` (not published for RC)
-
-<hr>
-
-Thank you again for reading this through, we can not wait to begin to work with you if you made your way through this contributing guide ❤️

Cargo.toml

@@ -1,59 +1,11 @@
 [workspace]
-resolver = "2"
 members = [
-    "meilisearch",
+    "meilisearch-core",
+    "meilisearch-http",
+    "meilisearch-schema",
+    "meilisearch-tokenizer",
     "meilisearch-types",
-    "meilisearch-auth",
-    "meili-snap",
-    "index-scheduler",
-    "dump",
-    "file-store",
-    "permissive-json-pointer",
-    "milli",
-    "filter-parser",
-    "flatten-serde-json",
-    "json-depth-checker",
-    "benchmarks",
-    "fuzzers",
 ]
 
-[workspace.package]
-version = "1.3.0"
-authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
-description = "Meilisearch HTTP server"
-homepage = "https://meilisearch.com"
-readme = "README.md"
-edition = "2021"
-license = "MIT"
-
 [profile.release]
-codegen-units = 1
-
-[profile.dev.package.flate2]
-opt-level = 3
-
-[profile.dev.package.grenad]
-opt-level = 3
-[profile.dev.package.roaring]
-opt-level = 3
-
-[profile.dev.package.lindera-ipadic-builder]
-opt-level = 3
-[profile.dev.package.encoding]
-opt-level = 3
-[profile.dev.package.yada]
-opt-level = 3
-
-[profile.release.package.lindera-ipadic-builder]
-opt-level = 3
-[profile.release.package.encoding]
-opt-level = 3
-[profile.release.package.yada]
-opt-level = 3
-
-[profile.bench.package.lindera-ipadic-builder]
-opt-level = 3
-[profile.bench.package.encoding]
-opt-level = 3
-[profile.bench.package.yada]
-opt-level = 3
+debug = true

Cross.toml

@@ -1,7 +0,0 @@
-[build.env]
-passthrough = [
-    "RUST_BACKTRACE",
-    "CARGO_TERM_COLOR",
-    "RUSTFLAGS",
-    "JEMALLOC_SYS_WITH_LG_PAGE"
-]

Dockerfile

@@ -1,47 +1,27 @@
 # Compile
-FROM    rust:alpine3.16 AS compiler
+FROM    alpine:3.10 AS compiler
 
-RUN     apk add -q --update-cache --no-cache build-base openssl-dev
+RUN     apk update --quiet
+RUN     apk add curl
+RUN     apk add build-base
+
+RUN     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 
 WORKDIR /meilisearch
 
-ARG     COMMIT_SHA
-ARG     COMMIT_DATE
-ARG     GIT_TAG
-ENV     VERGEN_GIT_SHA=${COMMIT_SHA} VERGEN_GIT_COMMIT_TIMESTAMP=${COMMIT_DATE} VERGEN_GIT_SEMVER_LIGHTWEIGHT=${GIT_TAG}
+COPY    . .
+
 ENV     RUSTFLAGS="-C target-feature=-crt-static"
 
-COPY    . .
-RUN     set -eux; \
-        apkArch="$(apk --print-arch)"; \
-        if [ "$apkArch" = "aarch64" ]; then \
-            export JEMALLOC_SYS_WITH_LG_PAGE=16; \
-        fi && \
-        cargo build --release
+RUN     $HOME/.cargo/bin/cargo build --release
 
 # Run
-FROM    alpine:3.16
+FROM    alpine:3.10
 
-ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
-ENV     MEILI_SERVER_PROVIDER docker
+RUN     apk update --quiet
+RUN     apk add libgcc
 
-RUN     apk update --quiet \
-        && apk add -q --no-cache libgcc tini curl
+COPY    --from=compiler /meilisearch/target/release/meilisearch .
 
-# add meilisearch to the `/bin` so you can run it from anywhere and it's easy
-# to find.
-COPY    --from=compiler /meilisearch/target/release/meilisearch /bin/meilisearch
-# To stay compatible with the older version of the container (pre v0.27.0) we're
-# going to symlink the meilisearch binary in the path to `/meilisearch`
-RUN     ln -s /bin/meilisearch /meilisearch
-
-# This directory should hold all the data related to meilisearch so we're going
-# to move our PWD in there.
-# We don't want to put the meilisearch binary
-WORKDIR /meili_data
-
-
-EXPOSE  7700/tcp
-
-ENTRYPOINT ["tini", "--"]
-CMD     /bin/meilisearch
+ENV     MEILI_HTTP_ADDR 0.0.0.0:8080
+CMD     ./meilisearch

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019-2022 Meili SAS
+Copyright (c) 2019-2020 Meili SAS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

PROFILING.md

@@ -1,19 +0,0 @@
-# Profiling Meilisearch
-
-Search engine technologies are complex pieces of software that require thorough profiling tools. We chose to use [Puffin](https://github.com/EmbarkStudios/puffin), which the Rust gaming industry uses extensively. You can export and import the profiling reports using the top bar's _File_ menu options.
-
-![An example profiling with Puffin viewer](assets/profiling-example.png)
-
-## Profiling the Indexing Process
-
-When you enable the `profile-with-puffin` feature of Meilisearch, a Puffin HTTP server will run on Meilisearch and listen on the default _0.0.0.0:8585_ address. This server will record a "frame" whenever it executes the `IndexScheduler::tick` method.
-
-Once your Meilisearch is running and awaits new indexation operations, you must [install and run the `puffin_viewer` tool](https://github.com/EmbarkStudios/puffin/tree/main/puffin_viewer) to see the profiling results. I advise you to run the viewer with the `RUST_LOG=puffin_http::client=debug` environment variable to see the client trying to connect to your server.
-
-Another piece of advice on the Puffin viewer UI interface is to consider the _Merge children with same ID_ option. It can hide the exact actual timings at which events were sent. Please turn it off when you see strange gaps on the Flamegraph. It can help.
-
-## Profiling the Search Process
-
-We still need to take the time to profile the search side of the engine with Puffin. It would require time to profile the filtering phase, query parsing, creation, and execution. We could even profile the Actix HTTP server.
-
-The only issue we see is the framing system. Puffin requires a global frame-based profiling phase, which collides with Meilisearch's ability to accept and answer multiple requests on different threads simultaneously.

README.md

@@ -1,118 +1,154 @@
-<p align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-light-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
-  </a>
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-dark-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
-  </a>
-</p>
+# MeiliSearch
 
-<h4 align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Website</a> |
-  <a href="https://roadmap.meilisearch.com/tabs/1-under-consideration">Roadmap</a> |
-  <a href="https://www.meilisearch.com/pricing?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Meilisearch Cloud</a> |
-  <a href="https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Blog</a> |
-  <a href="https://www.meilisearch.com/docs?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Documentation</a> |
-  <a href="https://www.meilisearch.com/docs/faq?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">FAQ</a> |
-  <a href="https://discord.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Discord</a>
-</h4>
+[![Build Status](https://github.com/meilisearch/MeiliSearch/workflows/Cargo%20test/badge.svg)](https://github.com/meilisearch/MeiliSearch/actions)
+[![dependency status](https://deps.rs/repo/github/meilisearch/MeiliSearch/status.svg)](https://deps.rs/repo/github/meilisearch/MeiliSearch)
+[![License](https://img.shields.io/badge/license-MIT-informational)](https://github.com/meilisearch/MeiliSearch/blob/master/LICENSE)
 
-<p align="center">
-  <a href="https://deps.rs/repo/github/meilisearch/meilisearch"><img src="https://deps.rs/repo/github/meilisearch/meilisearch/status.svg" alt="Dependency status"></a>
-  <a href="https://github.com/meilisearch/meilisearch/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
-  <a href="https://ms-bors.herokuapp.com/repositories/52"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
-</p>
+⚡ Ultra relevant and instant full-text search API 🔍
 
-<p align="center">⚡ A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow 🔍</p>
+MeiliSearch is a powerful, fast, open-source, easy to use, and deploy search engine. The search and indexation are fully customizable and handles features like typo-tolerance, filters, and synonyms.
 
-Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow.
+## What MeiliSearch has to offer
+* Search as-you-type experience (answers < 50ms)
+* Full-text search
+* Typo tolerant (understands typos and spelling mistakes)
+* Supports Kanji
+* Supports Synonym
+* Easy to install, deploy, and maintain
+* Whole documents returned
+* Highly customizable
+* RESTfull API 
 
-<p align="center" name="demo">
-  <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-light-mode-only" target="_blank">
-    <img src="assets/demo-light.gif#gh-light-mode-only" alt="A bright colored application for finding movies screening near the user">
-  </a>
-  <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-dark-mode-only" target="_blank">
-    <img src="assets/demo-dark.gif#gh-dark-mode-only" alt="A dark colored application for finding movies screening near the user">
-  </a>
-</p>
+For more [details about those features, go to our documentation](https://docs.meilisearch.com/introduction/features.html).
 
-🔥 [**Try it!**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-link) 🔥
+[![crates.io demo gif](misc/crates-io-demo.gif)](https://crates.meilisearch.com)
 
-## ✨ Features
+> Meili helps the Rust community find crates on [crates.meilisearch.com](https://crates.meilisearch.com)
 
-- **Search-as-you-type:** find search results in less than 50 milliseconds
-- **[Typo tolerance](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#typo-tolerance):** get relevant matches even when queries contain typos and misspellings
-- **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code
-- **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** sort results based on price, date, or pretty much anything else your users need
-- **[Synonym support](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#synonyms):** configure synonyms to include more relevant content in your search results
-- **[Geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** filter and sort documents based on geographic data
-- **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
-- **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** control which users can access what data with API keys that allow fine-grained permissions handling
-- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** personalize search results for any number of application tenants
-- **Highly Customizable:** customize Meilisearch to your specific needs or use our out-of-the-box and hassle-free presets
-- **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** integrate Meilisearch in your technical stack with our plugins and SDKs
-- **Easy to install, deploy, and maintain**
+### In-depth features
 
-## 📖 Documentation
+- Provides [6 default ranking criteria](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/criterion/mod.rs#L107-L113) used to [bucket sort](https://en.wikipedia.org/wiki/Bucket_sort) documents
+- Accepts [custom criteria](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/criterion/mod.rs#L24-L33) and can apply them in any custom order
+- Support [ranged queries](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/query_builder.rs#L283), useful for paginating results
+- Can [distinct](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/query_builder.rs#L265-L270) and [filter](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/query_builder.rs#L246-L259) returned documents based on context defined rules
+- Searches for [concatenated](https://github.com/meilisearch/MeiliSearch/pull/164) and [splitted query words](https://github.com/meilisearch/MeiliSearch/pull/232) to improve the search quality.
+- Can store complete documents or only [user schema specified fields](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-schema/src/lib.rs#L265-L279)
+- The [default tokenizer](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-tokenizer/src/lib.rs) can index latin and kanji based languages
+- Returns [the matching text areas](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/lib.rs#L66-L88), useful to highlight matched words in results
+- Accepts query time search config like the [searchable attributes](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/query_builder.rs#L272-L275)
+- Supports [runtime incremental indexing](https://github.com/meilisearch/MeiliSearch/blob/dc5c42821e1340e96cb90a3da472264624a26326/meilisearch-core/src/store/mod.rs#L143-L173)
 
-You can consult Meilisearch's documentation at [https://www.meilisearch.com/docs](https://www.meilisearch.com/docs/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=docs).
 
-## 🚀 Getting started
+## Quick Start
 
-For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
+You can deploy your instant, relevant, and typo-tolerant MeiliSearch search engine by yourself too.
+Something similar to the demo above can be achieved by following these little three steps first.
+You still need to create your front-end to make it pretty, though.
 
-You may also want to check out [Meilisearch 101](https://www.meilisearch.com/docs/learn/getting_started/filtering_and_sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) for an introduction to some of Meilisearch's most popular features.
+### Deploy the Server
 
-## ⚡ Supercharge your Meilisearch experience
+If you have not yet installed Rust and its package manager `cargo`, go to [the installation page](https://www.rust-lang.org/tools/install).<br/>
+You can deploy the server on your machine; it listens to HTTP requests on the 8080 port by default.
 
-Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/pricing?utm_campaign=oss&utm_source=engine&utm_medium=meilisearch). Get started with a 14-day free trial! No credit card required.
+```bash
+cargo run --release
+```
 
-## 🧰 SDKs & integration tools
+For more logs during the execution, run:
+```bash
+RUST_LOG=info cargo run --release
+```
 
-Install one of our SDKs in your project for seamless integration between Meilisearch and your favorite language or framework!
+### Create an Index and Upload Some Documents
 
-Take a look at the complete [Meilisearch integration list](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-link).
+MeiliSearch can serve multiple indexes, with different kinds of documents,
+therefore, it is required to create the index before sending documents to it.
 
-[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-logos)
+```bash
+curl -i -X POST 'http://127.0.0.1:8080/indexes' --data '{ "name": "Movies", "uid": "movies" }'
+```
 
-## ⚙️ Advanced usage
+Now that the server knows about our brand new index, we can send it data.
+We provided you a small dataset that is available in the `datasets/` directory.
 
-Experienced users will want to keep our [API Reference](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) close at hand.
+```bash
+curl -i -X POST 'http://127.0.0.1:8080/indexes/movies/documents' \
+  --header 'content-type: application/json' \
+  --data @datasets/movies/movies.json
+```
 
-We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [API keys](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), and [tenant tokens](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
+### Search for Documents
 
-Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
+The search engine is now aware of our documents and can serve those via our HTTP server again.
+The [`jq` command-line tool](https://stedolan.github.io/jq/) can significantly help you read the server responses.
 
-## 📊 Telemetry
+```bash
+curl 'http://127.0.0.1:8080/indexes/movies/search?q=botman'
+```
 
-Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
+```json
+{
+  "hits": [
+    {
+      "id": "29751",
+      "title": "Batman Unmasked: The Psychology of the Dark Knight",
+      "poster": "https://image.tmdb.org/t/p/w1280/jjHu128XLARc2k4cJrblAvZe0HE.jpg",
+      "overview": "Delve into the world of Batman and the vigilante justice tha",
+      "release_date": "2008-07-15"
+    },
+    {
+      "id": "471474",
+      "title": "Batman: Gotham by Gaslight",
+      "poster": "https://image.tmdb.org/t/p/w1280/7souLi5zqQCnpZVghaXv0Wowi0y.jpg",
+      "overview": "ve Victorian Age Gotham City, Batman begins his war on crime",
+      "release_date": "2018-01-12"
+    }
+  ],
+  "offset": 0,
+  "limit": 2,
+  "processingTimeMs": 1,
+  "query": "botman"
+}
+```
 
-To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
+## Performances
 
-If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) of our documentation.
+With a dataset composed of _100 353_ documents with _352_ attributes each and _3_ of them indexed.
+So more than _300 000_ fields indexed for _35 million_ stored we can handle more than _2.8k req/sec_ with an average response time of _9 ms_ on an Intel i7-7700 (8) @ 4.2GHz.
 
-## 📫 Get in touch!
+Requests are made using [wrk](https://github.com/wg/wrk) and scripted to simulate real users' queries.
 
-Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
+```
+Running 10s test @ http://localhost:2230
+  2 threads and 25 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     9.52ms    7.61ms  99.25ms   84.58%
+    Req/Sec     1.41k   119.11     1.78k    64.50%
+  28080 requests in 10.01s, 7.42MB read
+Requests/sec:   2806.46
+Transfer/sec:    759.17KB
+```
 
-🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
+We also indexed a dataset containing something like _12 millions_ cities names in _24 minutes_ on a machine with _8 cores_, _64 GB of RAM_, and a _300 GB NMVe_ SSD.<br/>
+The resulting database was _16 GB_ and search results were between _30 ms_ and _4 seconds_ for short prefix queries.
 
-💌 Want to make a suggestion or give feedback? Here are some of the channels where you can reach us:
+### Notes
 
-- For feature requests, please visit our [product repository](https://github.com/meilisearch/product/discussions)
-- Found a bug? Open an [issue](https://github.com/meilisearch/meilisearch/issues)!
-- Want to be part of our Discord community? [Join us!](https://discord.gg/meilisearch)
+With Rust 1.32 the allocator has been [changed to use the system allocator](https://blog.rust-lang.org/2019/01/17/Rust-1.32.0.html#jemalloc-is-removed-by-default).
+We have seen much better performances when [using jemalloc as the global allocator](https://github.com/alexcrichton/jemallocator#documentation).
 
-Thank you for your support!
+## How it works
 
-## 👩‍💻 Contributing
+MeiliSearch uses [LMDB](https://en.wikipedia.org/wiki/Lightning_Memory-Mapped_Database) as the internal key-value store. The key-value store allows us to handle updates and queries with small memory and CPU overheads. The whole ranking system is [data oriented](https://github.com/meilisearch/MeiliSearch/issues/82) and provides excellent great performances.
 
-Meilisearch is, and will always be, open-source! If you want to contribute to the project, please take a look at [our contribution guidelines](CONTRIBUTING.md).
+You can [read the deep dive](deep-dive.md) if you want more information on the engine; it describes the whole process of generating updates and handling queries. Also, you can take a look at the [typos and ranking rules](typos-ranking-rules.md) if you want to know the default rules used to sort the documents.
 
-## 📦 Versioning
+## Contributing
 
-Meilisearch releases and their associated binaries are available [in this GitHub page](https://github.com/meilisearch/meilisearch/releases).
+We will be glad if you submit issues and pull requests. You can help to grow this project and start contributing by checking [issues tagged "good-first-issue"](https://github.com/meilisearch/MeiliSearch/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22). It is a good start!
 
-The binaries are versioned following [SemVer conventions](https://semver.org/). To know more, read our [versioning policy](https://github.com/meilisearch/engine-team/blob/main/resources/versioning-policy.md).
+### Analytic Events
 
-Differently from the binaries, crates in this repository are not currently available on [crates.io](https://crates.io/) and do not follow [SemVer conventions](https://semver.org).
+We send events to our Amplitude instance to be aware of the number of people who use MeiliSearch.<br/>
+We only send the platform on which the server runs once by day. No other information is sent.<br/>
+If you do not want us to send events, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.

SECURITY.md

@@ -1,33 +0,0 @@
-# Security
-
-Meilisearch takes the security of our software products and services seriously.
-
-If you believe you have found a security vulnerability in any Meilisearch-owned repository, please report it to us as described below.
-
-## Supported versions
-
-As long as we are pre-v1.0, only the latest version of Meilisearch will be supported with security updates.
-
-## Reporting security issues
-
-⚠️ Please do not report security vulnerabilities through public GitHub issues. ⚠️
-
-Instead, please kindly email us at security@meilisearch.com
-
-Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
-
-- Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
-- Full paths of source file(s) related to the manifestation of the issue
-- The location of the affected source code (tag/branch/commit or direct URL)
-- Any special configuration required to reproduce the issue
-- Step-by-step instructions to reproduce the issue
-- Proof-of-concept or exploit code (if possible)
-- Impact of the issue, including how an attacker might exploit the issue
-
-This information will help us triage your report more quickly.
-
-You will receive a response from us within 72 hours. If the issue is confirmed, we will release a patch as soon as possible depending on complexity.
-
-## Preferred languages
-
-We prefer all communications to be in English.

assets/do-btn-blue.svg

@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<svg width="200px" height="42px" viewBox="0 0 200 42" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
-    <!-- Generator: Sketch 52.5 (67469) - http://www.bohemiancoding.com/sketch -->
-    <title>do-btn-blue</title>
-    <desc>Created with Sketch.</desc>
-    <g id="Page-1" stroke="none" stroke-width="1" fill="none" fill-rule="evenodd">
-        <g id="Partner-welcome-kit-Copy-3" transform="translate(-651.000000, -762.000000)">
-            <g id="do-btn-blue" transform="translate(651.000000, 763.000000)">
-                <rect id="Rectangle-Copy" fill="#0069FF" x="0" y="0" width="200" height="40" rx="6"></rect>
-                <path d="M45,0 L45,40" id="Line-2" stroke="#FFFFFF" stroke-linecap="square"></path>
-                <g id="DO_Logo_horizontal_blue-Copy" transform="translate(13.000000, 11.000000)" fill="#FFFFFF">
-                    <path d="M10.0098493,20 L10.0098493,16.1262429 C14.12457,16.1262429 17.2897398,12.0548452 15.7269372,7.74627862 C15.1334679,6.14538921 13.8674,4.86072487 12.2650328,4.28756693 C7.952489,2.72620566 3.87733294,5.88845634 3.87733294,9.99938223 C3.87733294,9.99938223 3.87733294,9.99938223 3.87733294,9.99938223 L0,9.99938223 C0,3.45747613 6.3303395,-1.64165309 13.1948014,0.492866119 C16.2017127,1.42177726 18.57559,3.81322933 19.5053586,6.79760341 C21.6418482,13.6754986 16.5577943,20 10.0098493,20 Z" id="XMLID_49_"></path>
-                    <polygon id="XMLID_47_" points="9.56521739 15.6521739 6.08695652 15.6521739 6.08695652 12.173913 6.08695652 12.173913 9.56521739 12.173913 9.56521739 12.173913"></polygon>
-                    <polygon id="XMLID_46_" points="6.08695652 19.1304348 3.47826087 19.1304348 3.47826087 19.1304348 3.47826087 16.5217391 6.08695652 16.5217391"></polygon>
-                    <polygon id="XMLID_45_" points="3.47826087 16.5217391 0.869565217 16.5217391 0.869565217 16.5217391 0.869565217 13.9130435 0.869565217 13.9130435 3.47826087 13.9130435 3.47826087 13.9130435"></polygon>
-                </g>
-                <text id="Create-a-Droplet-Copy" font-family="Sailec-Medium, Sailec" font-size="16" font-weight="400" fill="#FFFFFF">
-                    <tspan x="58" y="26">Create a Droplet</tspan>
-                </text>
-            </g>
-        </g>
-    </g>
-</svg>

assets/logo.svg

@@ -1,19 +0,0 @@
-<svg width="300" height="300" viewBox="0 0 300 300" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M0 237L55.426 96.7678C63.2367 77.0063 82.499 64 103.955 64H137.371L81.9447 204.232C74.1341 223.993 54.8717 237 33.4156 237H0Z" fill="url(#paint0_linear_1_898)"/>
-<path d="M81.3123 237L136.738 96.7682C144.549 77.0067 163.811 64.0004 185.267 64.0004H218.683L163.257 204.232C155.446 223.994 136.184 237 114.728 237H81.3123Z" fill="url(#paint1_linear_1_898)"/>
-<path d="M162.629 237L218.055 96.7682C225.866 77.0067 245.128 64.0004 266.584 64.0004H300L244.574 204.232C236.763 223.994 217.501 237 196.045 237H162.629Z" fill="url(#paint2_linear_1_898)"/>
-<defs>
-<linearGradient id="paint0_linear_1_898" x1="300.001" y1="50.7858" x2="1.63474" y2="221.244" gradientUnits="userSpaceOnUse">
-<stop stop-color="#FF5CAA"/>
-<stop offset="1" stop-color="#FF4E62"/>
-</linearGradient>
-<linearGradient id="paint1_linear_1_898" x1="300.001" y1="50.7858" x2="1.63474" y2="221.244" gradientUnits="userSpaceOnUse">
-<stop stop-color="#FF5CAA"/>
-<stop offset="1" stop-color="#FF4E62"/>
-</linearGradient>
-<linearGradient id="paint2_linear_1_898" x1="300.001" y1="50.7858" x2="1.63474" y2="221.244" gradientUnits="userSpaceOnUse">
-<stop stop-color="#FF5CAA"/>
-<stop offset="1" stop-color="#FF4E62"/>
-</linearGradient>
-</defs>
-</svg>

assets/meilisearch-logo-dark.svg

@@ -1,30 +0,0 @@
-<svg width="495" height="74" viewBox="0 0 495 74" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M181.842 42.5349C181.842 37.6137 184.201 34.715 188.718 34.715C192.965 34.715 194.381 37.7486 194.381 41.6585V62.6238H203.953V40.5799C203.953 32.3556 199.639 26.4907 191.145 26.4907C186.089 26.4907 182.516 28.0412 179.415 31.4792C177.393 28.3782 173.955 26.4907 169.168 26.4907C164.112 26.4907 160.607 28.5805 158.989 31.614V27.2996H150.158V62.6238H159.731V42.3326C159.731 37.6137 162.157 34.715 166.607 34.715C170.854 34.715 172.269 37.7486 172.269 41.6585V62.6238H181.842V42.5349Z" fill="white"/>
-<path d="M243.245 47.7256C243.245 47.7256 243.379 46.4448 243.379 44.8943C243.379 34.4454 236.301 26.4907 225.852 26.4907C215.403 26.4907 208.123 34.4454 208.123 44.8943C208.123 55.7477 215.471 63.4327 225.92 63.4327C234.077 63.4327 240.548 58.5116 242.638 51.3659H232.998C231.852 53.9276 229.088 55.2084 226.189 55.2084C221.403 55.2084 218.302 52.5793 217.628 47.7256H243.245ZM225.785 34.1757C230.234 34.1757 233.133 36.8722 233.807 40.8495H217.763C218.572 36.8048 221.403 34.1757 225.785 34.1757Z" fill="white"/>
-<path d="M244.791 35.524H249.038V62.6238H258.61V27.2996H244.791V35.524ZM253.824 22.7156C257.195 22.7156 259.622 20.3561 259.622 16.9855C259.622 13.6149 257.195 11.188 253.824 11.188C250.454 11.188 248.027 13.6149 248.027 16.9855C248.027 20.3561 250.454 22.7156 253.824 22.7156Z" fill="white"/>
-<path d="M278.432 54.3995C278.163 54.3995 277.758 54.4669 277.152 54.4669C274.994 54.4669 274.725 53.4557 274.725 51.9726V12.0644H265.152V52.6467C265.152 59.6576 267.849 62.7586 275.466 62.7586C276.747 62.7586 277.96 62.6238 278.432 62.5564V54.3995Z" fill="white"/>
-<path d="M279.521 35.524H283.768V62.6238H293.341V27.2996H279.521V35.524ZM288.555 22.7156C291.925 22.7156 294.352 20.3561 294.352 16.9855C294.352 13.6149 291.925 11.188 288.555 11.188C285.184 11.188 282.757 13.6149 282.757 16.9855C282.757 20.3561 285.184 22.7156 288.555 22.7156Z" fill="white"/>
-<path d="M312.557 62.9937C321.86 62.9937 326.242 58.0726 326.242 52.8819C326.242 38.4556 305.007 46.4777 305.007 36.9725C305.007 33.8716 307.636 31.2425 312.962 31.2425C318.422 31.2425 320.984 34.2086 321.388 37.9163H326.175C325.77 33.2648 322.602 27.0629 313.097 27.0629C304.94 27.0629 300.356 31.9166 300.356 37.1748C300.356 51.264 321.591 43.1745 321.591 53.0167C321.591 56.4547 318.355 58.8142 312.557 58.8142C306.625 58.8142 303.659 55.848 303.322 51.4662H298.468C298.873 57.4659 302.648 62.9937 312.557 62.9937Z" fill="white"/>
-<path d="M364.256 46.4103C364.256 46.4103 364.324 45.3317 364.324 44.5901C364.324 34.8827 358.054 27.0629 347.808 27.0629C337.494 27.0629 330.955 35.4894 330.955 44.9946C330.955 54.6346 337.022 62.9937 347.875 62.9937C356.032 62.9937 361.695 58.0052 363.717 51.4662H358.729C357.245 55.6458 353.201 58.6794 347.943 58.6794C340.729 58.6794 336.213 53.3538 335.741 46.4103H364.256ZM347.808 31.3773C354.549 31.3773 358.931 35.8939 359.538 42.5004H335.876C336.685 36.1636 341.134 31.3773 347.808 31.3773Z" fill="white"/>
-<path d="M394.037 45.871V49.1068C394.037 54.9717 389.79 59.0164 381.634 59.0164C376.578 59.0164 373.814 56.9266 373.814 52.41C373.814 50.118 374.892 48.3652 376.578 47.4215C378.33 46.4777 380.69 45.871 394.037 45.871ZM381.094 62.9937C387.027 62.9937 391.813 61.1062 394.24 57.1963V62.1848H398.824V39.7364C398.824 32.1188 394.442 27.0629 384.532 27.0629C375.027 27.0629 370.848 31.8492 369.971 37.9837H374.623C375.566 33.13 379.274 31.1751 384.33 31.1751C390.802 31.1751 394.037 33.8716 394.037 39.669V41.8936C383.184 41.8936 378.667 42.0959 375.297 43.4441C371.387 44.9946 369.095 48.4327 369.095 52.5448C369.095 58.5445 372.937 62.9937 381.094 62.9937Z" fill="white"/>
-<path d="M424.991 27.6022C424.991 27.6022 424.182 27.5348 423.845 27.5348C417.509 27.5348 414.138 30.838 412.857 33.1974V27.8718H408.273V62.1848H413.059V42.7026C413.059 35.5569 417.441 32.0514 423.306 32.0514C424.182 32.0514 424.991 32.1188 424.991 32.1188V27.6022Z" fill="white"/>
-<path d="M425.809 45.062C425.809 54.4324 432.28 62.9937 442.729 62.9937C452.032 62.9937 457.425 56.7918 458.773 49.9831H453.92C452.504 55.3087 448.594 58.6794 442.729 58.6794C435.516 58.6794 430.662 52.9493 430.662 45.062C430.662 37.1073 435.516 31.3773 442.729 31.3773C448.594 31.3773 452.504 34.7479 453.92 40.0735H458.773C457.425 33.2648 452.032 27.0629 442.729 27.0629C432.28 27.0629 425.809 35.6243 425.809 45.062Z" fill="white"/>
-<path d="M470.041 11.6254H465.255V62.1848H470.041V41.8936C470.041 34.8827 474.558 31.2425 480.355 31.2425C486.49 31.2425 489.389 35.0176 489.389 41.2195V62.1848H494.175V40.2757C494.175 32.6581 489.658 27.0629 481.164 27.0629C474.76 27.0629 471.255 30.5683 470.041 32.6581V11.6254Z" fill="white"/>
-<path d="M0.825012 73.993L24.0688 14.5224C27.3443 6.14179 35.4223 0.625977 44.4203 0.625977H58.4336L35.1899 60.0966C31.9143 68.4772 23.8363 73.993 14.8384 73.993H0.825012Z" fill="url(#paint0_linear_0_3)"/>
-<path d="M34.9246 73.9932L58.1684 14.5226C61.444 6.14197 69.5219 0.626152 78.5199 0.626152H92.5333L69.2895 60.0968C66.014 68.4774 57.936 73.9932 48.938 73.9932H34.9246Z" fill="url(#paint1_linear_0_3)"/>
-<path d="M69.0262 73.9932L92.27 14.5226C95.5456 6.14197 103.624 0.626152 112.622 0.626152H126.635L103.391 60.0968C100.116 68.4774 92.0376 73.9932 83.0396 73.9932H69.0262Z" fill="url(#paint2_linear_0_3)"/>
-<defs>
-<linearGradient id="paint0_linear_0_3" x1="126.635" y1="-4.97799" x2="0.825008" y2="66.0978" gradientUnits="userSpaceOnUse">
-<stop stop-color="#FF5CAA"/>
-<stop offset="1" stop-color="#FF4E62"/>
-</linearGradient>
-<linearGradient id="paint1_linear_0_3" x1="126.635" y1="-4.97799" x2="0.825008" y2="66.0978" gradientUnits="userSpaceOnUse">
-<stop stop-color="#FF5CAA"/>
-<stop offset="1" stop-color="#FF4E62"/>
-</linearGradient>
-<linearGradient id="paint2_linear_0_3" x1="126.635" y1="-4.97799" x2="0.825008" y2="66.0978" gradientUnits="userSpaceOnUse">
-<stop stop-color="#FF5CAA"/>
-<stop offset="1" stop-color="#FF4E62"/>
-</linearGradient>
-</defs>
-</svg>

assets/meilisearch-logo-light.svg

@@ -1,30 +0,0 @@
-<svg width="495" height="74" viewBox="0 0 495 74" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path d="M181.84 42.5347C181.84 37.6136 184.199 34.7149 188.716 34.7149C192.963 34.7149 194.378 37.7484 194.378 41.6584V62.6237H203.951V40.5798C203.951 32.3554 199.637 26.4906 191.143 26.4906C186.087 26.4906 182.514 28.041 179.413 31.4791C177.39 28.3781 173.952 26.4906 169.166 26.4906C164.11 26.4906 160.605 28.5804 158.987 31.6139V27.2995H150.156V62.6237H159.728V42.3325C159.728 37.6136 162.155 34.7149 166.604 34.7149C170.851 34.7149 172.267 37.7484 172.267 41.6584V62.6237H181.84V42.5347Z" fill="#21004B"/>
-<path d="M243.242 47.7255C243.242 47.7255 243.377 46.4447 243.377 44.8942C243.377 34.4452 236.299 26.4906 225.85 26.4906C215.401 26.4906 208.12 34.4452 208.12 44.8942C208.12 55.7476 215.468 63.4326 225.917 63.4326C234.074 63.4326 240.546 58.5115 242.636 51.3658H232.996C231.85 53.9274 229.086 55.2083 226.187 55.2083C221.401 55.2083 218.3 52.5792 217.626 47.7255H243.242ZM225.783 34.1756C230.232 34.1756 233.131 36.8721 233.805 40.8494H217.76C218.569 36.8047 221.401 34.1756 225.783 34.1756Z" fill="#21004B"/>
-<path d="M244.789 35.5238H249.036V62.6237H258.608V27.2995H244.789V35.5238ZM253.822 22.7155C257.193 22.7155 259.619 20.356 259.619 16.9854C259.619 13.6148 257.193 11.1879 253.822 11.1879C250.451 11.1879 248.024 13.6148 248.024 16.9854C248.024 20.356 250.451 22.7155 253.822 22.7155Z" fill="#21004B"/>
-<path d="M278.43 54.3993C278.16 54.3993 277.756 54.4667 277.149 54.4667C274.992 54.4667 274.722 53.4556 274.722 51.9725V12.0643H265.15V52.6466C265.15 59.6575 267.846 62.7585 275.464 62.7585C276.745 62.7585 277.958 62.6237 278.43 62.5562V54.3993Z" fill="#21004B"/>
-<path d="M279.519 35.5238H283.766V62.6237H293.339V27.2995H279.519V35.5238ZM288.553 22.7155C291.923 22.7155 294.35 20.356 294.35 16.9854C294.35 13.6148 291.923 11.1879 288.553 11.1879C285.182 11.1879 282.755 13.6148 282.755 16.9854C282.755 20.356 285.182 22.7155 288.553 22.7155Z" fill="#21004B"/>
-<path d="M312.557 62.9939C321.86 62.9939 326.242 58.0728 326.242 52.882C326.242 38.4557 305.007 46.4778 305.007 36.9726C305.007 33.8717 307.636 31.2426 312.962 31.2426C318.422 31.2426 320.984 34.2087 321.388 37.9164H326.175C325.77 33.265 322.602 27.063 313.097 27.063C304.94 27.063 300.356 31.9167 300.356 37.1749C300.356 51.2641 321.591 43.1746 321.591 53.0168C321.591 56.4548 318.355 58.8143 312.557 58.8143C306.625 58.8143 303.659 55.8481 303.322 51.4663H298.468C298.872 57.466 302.648 62.9939 312.557 62.9939Z" fill="#21004B"/>
-<path d="M364.256 46.4104C364.256 46.4104 364.324 45.3318 364.324 44.5903C364.324 34.8829 358.054 27.063 347.808 27.063C337.494 27.063 330.955 35.4896 330.955 44.9947C330.955 54.6347 337.022 62.9939 347.875 62.9939C356.032 62.9939 361.695 58.0053 363.717 51.4663H358.728C357.245 55.6459 353.201 58.6795 347.942 58.6795C340.729 58.6795 336.213 53.3539 335.741 46.4104H364.256ZM347.808 31.3774C354.549 31.3774 358.931 35.894 359.537 42.5005H335.876C336.685 36.1637 341.134 31.3774 347.808 31.3774Z" fill="#21004B"/>
-<path d="M394.037 45.8711V49.1069C394.037 54.9718 389.79 59.0165 381.633 59.0165C376.578 59.0165 373.814 56.9267 373.814 52.4101C373.814 50.1181 374.892 48.3654 376.578 47.4216C378.33 46.4778 380.69 45.8711 394.037 45.8711ZM381.094 62.9939C387.026 62.9939 391.813 61.1063 394.24 57.1964V62.1849H398.824V39.7366C398.824 32.1189 394.442 27.063 384.532 27.063C375.027 27.063 370.847 31.8493 369.971 37.9838H374.623C375.566 33.1301 379.274 31.1752 384.33 31.1752C390.802 31.1752 394.037 33.8717 394.037 39.6691V41.8938C383.184 41.8938 378.667 42.096 375.297 43.4442C371.387 44.9947 369.095 48.4328 369.095 52.5449C369.095 58.5446 372.937 62.9939 381.094 62.9939Z" fill="#21004B"/>
-<path d="M424.991 27.6023C424.991 27.6023 424.182 27.5349 423.845 27.5349C417.508 27.5349 414.138 30.8381 412.857 33.1975V27.872H408.273V62.1849H413.059V42.7027C413.059 35.557 417.441 32.0515 423.306 32.0515C424.182 32.0515 424.991 32.1189 424.991 32.1189V27.6023Z" fill="#21004B"/>
-<path d="M425.809 45.0621C425.809 54.4325 432.28 62.9939 442.729 62.9939C452.032 62.9939 457.425 56.7919 458.773 49.9832H453.92C452.504 55.3088 448.594 58.6795 442.729 58.6795C435.516 58.6795 430.662 52.9494 430.662 45.0621C430.662 37.1075 435.516 31.3774 442.729 31.3774C448.594 31.3774 452.504 34.748 453.92 40.0736H458.773C457.425 33.265 452.032 27.063 442.729 27.063C432.28 27.063 425.809 35.6244 425.809 45.0621Z" fill="#21004B"/>
-<path d="M470.041 11.6255H465.255V62.1849H470.041V41.8938C470.041 34.8829 474.558 31.2426 480.355 31.2426C486.49 31.2426 489.389 35.0177 489.389 41.2196V62.1849H494.175V40.2759C494.175 32.6582 489.658 27.063 481.164 27.063C474.76 27.063 471.255 30.5685 470.041 32.6582V11.6255Z" fill="#21004B"/>
-<path d="M0.824951 73.993L24.0688 14.5224C27.3443 6.14179 35.4223 0.625977 44.4202 0.625977H58.4336L35.1898 60.0966C31.9143 68.4772 23.8363 73.993 14.8383 73.993H0.824951Z" fill="url(#paint0_linear_0_15)"/>
-<path d="M34.9246 73.9932L58.1684 14.5226C61.4439 6.14197 69.5219 0.626152 78.5199 0.626152H92.5332L69.2894 60.0968C66.0139 68.4774 57.9359 73.9932 48.9379 73.9932H34.9246Z" fill="url(#paint1_linear_0_15)"/>
-<path d="M69.0262 73.9932L92.27 14.5226C95.5455 6.14197 103.623 0.626152 112.621 0.626152H126.635L103.391 60.0968C100.115 68.4774 92.0375 73.9932 83.0395 73.9932H69.0262Z" fill="url(#paint2_linear_0_15)"/>
-<defs>
-<linearGradient id="paint0_linear_0_15" x1="126.635" y1="-4.97799" x2="0.824952" y2="66.0978" gradientUnits="userSpaceOnUse">
-<stop stop-color="#FF5CAA"/>
-<stop offset="1" stop-color="#FF4E62"/>
-</linearGradient>
-<linearGradient id="paint1_linear_0_15" x1="126.635" y1="-4.97799" x2="0.824952" y2="66.0978" gradientUnits="userSpaceOnUse">
-<stop stop-color="#FF5CAA"/>
-<stop offset="1" stop-color="#FF4E62"/>
-</linearGradient>
-<linearGradient id="paint2_linear_0_15" x1="126.635" y1="-4.97799" x2="0.824952" y2="66.0978" gradientUnits="userSpaceOnUse">
-<stop stop-color="#FF5CAA"/>
-<stop offset="1" stop-color="#FF4E62"/>
-</linearGradient>
-</defs>
-</svg>

assets/milli-logo.svg

@@ -1,6 +0,0 @@
-<svg width="277" height="236" viewBox="0 0 277 236" fill="none" xmlns="http://www.w3.org/2000/svg">
-<path fill-rule="evenodd" clip-rule="evenodd" d="M213.085 190L242.907 86H276.196L246.375 190H213.085Z" fill="#494949"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M0 190L29.8215 86H63.1111L33.2896 190H0Z" fill="#494949"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M124.986 0L57.5772 235.083L60.7752 236H90.6038L158.276 0H124.986Z" fill="#494949"/>
-<path fill-rule="evenodd" clip-rule="evenodd" d="M195.273 0L127.601 236H160.891L228.563 0H195.273Z" fill="#494949"/>
-</svg>

assets/prometheus-basic-scraper.yml

@@ -1,19 +0,0 @@
-global:
-  scrape_interval:     15s # By default, scrape targets every 15 seconds.
-
-  # Attach these labels to any time series or alerts when communicating with
-  # external systems (federation, remote storage, Alertmanager).
-  external_labels:
-    monitor: 'codelab-monitor'
-
-# A scrape configuration containing exactly one endpoint to scrape:
-# Here it's Prometheus itself.
-scrape_configs:
-  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
-  - job_name: 'meilisearch'
-
-    # Override the global default and scrape targets from this job every 5 seconds.
-    scrape_interval: 5s
-
-    static_configs:
-      - targets: ['localhost:7700']

benchmarks/.gitignore

@@ -1 +0,0 @@
-benches/datasets_paths.rs

benchmarks/Cargo.toml

@@ -1,50 +0,0 @@
-[package]
-name = "benchmarks"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
-[dependencies]
-anyhow = "1.0.70"
-csv = "1.2.1"
-milli = { path = "../milli" }
-mimalloc = { version = "0.1.36", default-features = false }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
-
-[dev-dependencies]
-criterion = { version = "0.4.0", features = ["html_reports"] }
-rand = "0.8.5"
-rand_chacha = "0.3.1"
-roaring = "0.10.1"
-
-[build-dependencies]
-anyhow = "1.0.70"
-bytes = "1.4.0"
-convert_case = "0.6.0"
-flate2 = "1.0.25"
-reqwest = { version = "0.11.16", features = ["blocking", "rustls-tls"], default-features = false }
-
-[features]
-default = ["milli/all-tokenizations"]
-
-[[bench]]
-name = "search_songs"
-harness = false
-
-[[bench]]
-name = "search_wiki"
-harness = false
-
-[[bench]]
-name = "search_geo"
-harness = false
-
-[[bench]]
-name = "indexing"
-harness = false

benchmarks/README.md

@@ -1,138 +0,0 @@
-Benchmarks
-==========
-
-## TOC
-
-- [Run the benchmarks](#run-the-benchmarks)
-- [Comparison between benchmarks](#comparison-between-benchmarks)
-- [Datasets](#datasets)
-
-## Run the benchmarks
-
-### On our private server
-
-The Meili team has self-hosted his own GitHub runner to run benchmarks on our dedicated bare metal server.
-
-To trigger the benchmark workflow:
-- Go to the `Actions` tab of this repository.
-- Select the `Benchmarks` workflow on the left.
-- Click on `Run workflow` in the blue banner.
-- Select the branch on which you want to run the benchmarks and select the dataset you want (default: `songs`).
-- Finally, click on `Run workflow`.
-
-This GitHub workflow will run the benchmarks and push the `critcmp` report to a DigitalOcean Space (= S3).
-
-The name of the uploaded file is displayed in the workflow.
-
-_[More about critcmp](https://github.com/BurntSushi/critcmp)._
-
-💡 To compare the just-uploaded benchmark with another one, check out the [next section](#comparison-between-benchmarks).
-
-### On your machine
-
-To run all the benchmarks (~5h):
-
-```bash
-cargo bench
-```
-
-To run only the `search_songs` (~1h), `search_wiki` (~3h), `search_geo` (~20m) or `indexing` (~2h) benchmark:
-
-```bash
-cargo bench --bench <dataset name>
-```
-
-By default, the benchmarks will be downloaded and uncompressed automatically in the target directory.<br>
-If you don't want to download the datasets every time you update something on the code, you can specify a custom directory with the environment variable `MILLI_BENCH_DATASETS_PATH`:
-
-```bash
-mkdir ~/datasets
-MILLI_BENCH_DATASETS_PATH=~/datasets cargo bench --bench search_songs # the four datasets are downloaded
-touch build.rs
-MILLI_BENCH_DATASETS_PATH=~/datasets cargo bench --bench songs # the code is compiled again but the datasets are not downloaded
-```
-
-## Comparison between benchmarks
-
-The benchmark reports we push are generated with `critcmp`. Thus, we use `critcmp` to show the result of a benchmark, or compare results between multiple benchmarks.
-
-We provide a script to download and display the comparison report.
-
-Requirements:
-- `grep`
-- `curl`
-- [`critcmp`](https://github.com/BurntSushi/critcmp)
-
-List the available file in the DO Space:
-
-```bash
-./benchmarks/script/list.sh
-```
-```bash
-songs_main_09a4321.json
-songs_geosearch_24ec456.json
-search_songs_main_cb45a10b.json
-```
-
-Run the comparison script:
-
-```bash
-# we get the result of ONE benchmark, this give you an idea of how much time an operation took
-./benchmarks/scripts/compare.sh son songs_geosearch_24ec456.json
-# we compare two benchmarks
-./benchmarks/scripts/compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json
-# we compare three benchmarks
-./benchmarks/scripts/compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json search_songs_main_cb45a10b.json
-```
-
-## Datasets
-
-The benchmarks uses the following datasets:
-- `smol-songs`
-- `smol-wiki`
-- `movies`
-- `smol-all-countries`
-
-### Songs
-
-`smol-songs` is a subset of the [`songs.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/songs.csv.gz).
-
-It was generated with this command:
-
-```bash
-xsv sample --seed 42 1000000 songs.csv -o smol-songs.csv
-```
-
-_[Download the generated `smol-songs` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-songs.csv.gz)._
-
-### Wiki
-
-`smol-wiki` is a subset of the [`wikipedia-articles.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/wiki-articles.csv.gz).
-
-It was generated with the following command:
-
-```bash
-xsv sample --seed 42 500000 wiki-articles.csv -o smol-wiki-articles.csv
-```
-
-_[Download the `smol-wiki` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-wiki-articles.csv.gz)._
-
-### Movies
-
-`movies` is a really small dataset we uses as our example in the [getting started](https://www.meilisearch.com/docs/learn/getting_started/quick_start)
-
-_[Download the `movies` dataset](https://www.meilisearch.com/movies.json)._
-
-
-### All Countries
-
-`smol-all-countries` is a subset of the [`all-countries.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/all-countries.csv.gz)
-It has been converted to jsonlines and then edited so it matches our format for the `_geo` field.
-
-It was generated with the following command:
-```bash
-bat all-countries.csv.gz | gunzip | xsv sample --seed 42 1000000 | csv2json-lite | sd '"latitude":"(.*?)","longitude":"(.*?)"' '"_geo": { "lat": $1, "lng": $2 }' | sd '\[|\]|,$' '' | gzip > smol-all-countries.jsonl.gz
-```
-
-_[Download the `smol-all-countries` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-all-countries.jsonl.gz)._
-

benchmarks/benches/search_geo.rs

@@ -1,122 +0,0 @@
-mod datasets_paths;
-mod utils;
-
-use criterion::{criterion_group, criterion_main};
-use milli::update::Settings;
-use utils::Conf;
-
-#[global_allocator]
-static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
-
-fn base_conf(builder: &mut Settings) {
-    let displayed_fields =
-        ["geonameid", "name", "asciiname", "alternatenames", "_geo", "population"]
-            .iter()
-            .map(|s| s.to_string())
-            .collect();
-    builder.set_displayed_fields(displayed_fields);
-
-    let searchable_fields =
-        ["name", "alternatenames", "elevation"].iter().map(|s| s.to_string()).collect();
-    builder.set_searchable_fields(searchable_fields);
-
-    let filterable_fields =
-        ["_geo", "population", "elevation"].iter().map(|s| s.to_string()).collect();
-    builder.set_filterable_fields(filterable_fields);
-
-    let sortable_fields =
-        ["_geo", "population", "elevation"].iter().map(|s| s.to_string()).collect();
-    builder.set_sortable_fields(sortable_fields);
-}
-
-#[rustfmt::skip]
-const BASE_CONF: Conf = Conf {
-    dataset: datasets_paths::SMOL_ALL_COUNTRIES,
-    dataset_format: "jsonl",
-    queries: &[
-        "",
-    ],
-    configure: base_conf,
-    primary_key: Some("geonameid"),
-    ..Conf::BASE
-};
-
-fn bench_geo(c: &mut criterion::Criterion) {
-    #[rustfmt::skip]
-    let confs = &[
-        // A basic placeholder with no geo
-        utils::Conf {
-            group_name: "placeholder with no geo",
-            ..BASE_CONF
-        },
-        // Medium aglomeration: probably the most common usecase
-        utils::Conf {
-            group_name: "asc sort from Lille",
-            sort: Some(vec!["_geoPoint(50.62999333378238, 3.086269263384099):asc"]),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "desc sort from Lille",
-            sort: Some(vec!["_geoPoint(50.62999333378238, 3.086269263384099):desc"]),
-            ..BASE_CONF
-        },
-        // Big agglomeration: a lot of documents close to our point
-        utils::Conf {
-            group_name: "asc sort from Tokyo",
-            sort: Some(vec!["_geoPoint(35.749512532692144, 139.61664952543356):asc"]),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "desc sort from Tokyo",
-            sort: Some(vec!["_geoPoint(35.749512532692144, 139.61664952543356):desc"]),
-            ..BASE_CONF
-        },
-        // The furthest point from any civilization
-        utils::Conf {
-            group_name: "asc sort from Point Nemo",
-            sort: Some(vec!["_geoPoint(-48.87561645055408, -123.39275749319793):asc"]),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "desc sort from Point Nemo",
-            sort: Some(vec!["_geoPoint(-48.87561645055408, -123.39275749319793):desc"]),
-            ..BASE_CONF
-        },
-        // Filters
-        utils::Conf {
-            group_name: "filter of 100km from Lille",
-            filter: Some("_geoRadius(50.62999333378238, 3.086269263384099, 100000)"),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "filter of 1km from Lille",
-            filter: Some("_geoRadius(50.62999333378238, 3.086269263384099, 1000)"),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "filter of 100km from Tokyo",
-            filter: Some("_geoRadius(35.749512532692144, 139.61664952543356, 100000)"),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "filter of 1km from Tokyo",
-            filter: Some("_geoRadius(35.749512532692144, 139.61664952543356, 1000)"),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "filter of 100km from Point Nemo",
-            filter: Some("_geoRadius(-48.87561645055408, -123.39275749319793, 100000)"),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "filter of 1km from Point Nemo",
-            filter: Some("_geoRadius(-48.87561645055408, -123.39275749319793, 1000)"),
-            ..BASE_CONF
-        },
-    ];
-
-    utils::run_benches(c, confs);
-}
-
-criterion_group!(benches, bench_geo);
-criterion_main!(benches);

benchmarks/benches/search_songs.rs

@@ -1,196 +0,0 @@
-mod datasets_paths;
-mod utils;
-
-use criterion::{criterion_group, criterion_main};
-use milli::update::Settings;
-use utils::Conf;
-
-#[global_allocator]
-static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
-
-fn base_conf(builder: &mut Settings) {
-    let displayed_fields =
-        ["id", "title", "album", "artist", "genre", "country", "released", "duration"]
-            .iter()
-            .map(|s| s.to_string())
-            .collect();
-    builder.set_displayed_fields(displayed_fields);
-
-    let searchable_fields = ["title", "album", "artist"].iter().map(|s| s.to_string()).collect();
-    builder.set_searchable_fields(searchable_fields);
-
-    let faceted_fields = ["released-timestamp", "duration-float", "genre", "country", "artist"]
-        .iter()
-        .map(|s| s.to_string())
-        .collect();
-    builder.set_filterable_fields(faceted_fields);
-}
-
-#[rustfmt::skip]
-const BASE_CONF: Conf = Conf {
-    dataset: datasets_paths::SMOL_SONGS,
-    queries: &[
-        "john ",             // 9097
-        "david ",            // 4794
-        "charles ",          // 1957
-        "david bowie ",      // 1200
-        "michael jackson ",  // 600
-        "thelonious monk ",  // 303
-        "charles mingus ",   // 142
-        "marcus miller ",    // 60
-        "tamo ",             // 13
-        "Notstandskomitee ", // 4
-    ],
-    configure: base_conf,
-    primary_key: Some("id"),
-    ..Conf::BASE
-};
-
-fn bench_songs(c: &mut criterion::Criterion) {
-    let default_criterion: Vec<String> =
-        milli::default_criteria().iter().map(|criteria| criteria.to_string()).collect();
-    let default_criterion = default_criterion.iter().map(|s| s.as_str());
-    let asc_default: Vec<&str> =
-        std::iter::once("released-timestamp:asc").chain(default_criterion.clone()).collect();
-    let desc_default: Vec<&str> =
-        std::iter::once("released-timestamp:desc").chain(default_criterion.clone()).collect();
-
-    let basic_with_quote: Vec<String> = BASE_CONF
-        .queries
-        .iter()
-        .map(|s| {
-            s.trim().split(' ').map(|s| format!(r#""{}""#, s)).collect::<Vec<String>>().join(" ")
-        })
-        .collect();
-    let basic_with_quote: &[&str] =
-        &basic_with_quote.iter().map(|s| s.as_str()).collect::<Vec<&str>>();
-
-    #[rustfmt::skip]
-    let confs = &[
-        /* first we bench each criterion alone */
-        utils::Conf {
-            group_name: "proximity",
-            queries: &[
-                "black saint sinner lady ",
-                "les dangeureuses 1960 ",
-                "The Disneyland Sing-Along Chorus ",
-                "Under Great Northern Lights ",
-                "7000 Danses Un Jour Dans Notre Vie ",
-            ],
-            criterion: Some(&["proximity"]),
-            optional_words: false,
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "typo",
-            queries: &[
-                "mongus ",
-                "thelonius monk ",
-                "Disnaylande ",
-                "the white striper ",
-                "indochie ",
-                "indochien ",
-                "klub des loopers ",
-                "fear of the duck ",
-                "michel depech ",
-                "stromal ",
-                "dire straights ",
-                "Arethla Franklin ",
-            ],
-            criterion: Some(&["typo"]),
-            optional_words: false,
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "words",
-            queries: &[
-                "the black saint and the sinner lady and the good doggo ", // four words to pop
-                "les liaisons dangeureuses 1793 ",                         // one word to pop
-                "The Disneyland Children's Sing-Alone song ",              // two words to pop
-                "seven nation mummy ",                                     // one word to pop
-                "7000 Danses / Le Baiser / je me trompe de mots ",         // four words to pop
-                "Bring Your Daughter To The Slaughter but now this is not part of the title ", // nine words to pop
-                "whathavenotnsuchforth and a good amount of words to pop to match the first one ", // 13
-            ],
-            criterion: Some(&["words"]),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "asc",
-            criterion: Some(&["released-timestamp:desc"]),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "desc",
-            criterion: Some(&["released-timestamp:desc"]),
-            ..BASE_CONF
-        },
-
-        /* then we bench the asc and desc criterion on top of the default criterion */
-        utils::Conf {
-            group_name: "asc + default",
-            criterion: Some(&asc_default[..]),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "desc + default",
-            criterion: Some(&desc_default[..]),
-            ..BASE_CONF
-        },
-
-        /* we bench the filters with the default request */
-        utils::Conf {
-            group_name: "basic filter: <=",
-            filter: Some("released-timestamp <= 946728000"), // year 2000
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "basic filter: TO",
-            filter: Some("released-timestamp 946728000 TO 1262347200"), // year 2000 to 2010
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "big filter",
-            filter: Some("released-timestamp != 1262347200 AND (NOT (released-timestamp = 946728000)) AND (duration-float = 1 OR (duration-float 1.1 TO 1.5 AND released-timestamp > 315576000))"),
-            ..BASE_CONF
-        },
-
-        /* the we bench some global / normal search with all the default criterion in the default
-         * order */
-        utils::Conf {
-            group_name: "basic placeholder",
-            queries: &[""],
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "basic without quote",
-            queries: &BASE_CONF
-                .queries
-                .iter()
-                .map(|s| s.trim()) // we remove the space at the end of each request
-                .collect::<Vec<&str>>(),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "basic with quote",
-            queries: basic_with_quote,
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "prefix search",
-            queries: &[
-                "s", // 500k+ results
-                "a", //
-                "b", //
-                "i", //
-                "x", // only 7k results
-            ],
-            ..BASE_CONF
-        },
-    ];
-
-    utils::run_benches(c, confs);
-}
-
-criterion_group!(benches, bench_songs);
-criterion_main!(benches);

benchmarks/benches/search_wiki.rs

@@ -1,129 +0,0 @@
-mod datasets_paths;
-mod utils;
-
-use criterion::{criterion_group, criterion_main};
-use milli::update::Settings;
-use utils::Conf;
-
-#[global_allocator]
-static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
-
-fn base_conf(builder: &mut Settings) {
-    let displayed_fields = ["title", "body", "url"].iter().map(|s| s.to_string()).collect();
-    builder.set_displayed_fields(displayed_fields);
-
-    let searchable_fields = ["title", "body"].iter().map(|s| s.to_string()).collect();
-    builder.set_searchable_fields(searchable_fields);
-}
-
-#[rustfmt::skip]
-const BASE_CONF: Conf = Conf {
-    dataset: datasets_paths::SMOL_WIKI_ARTICLES,
-    queries: &[
-        "mingus ",        // 46 candidates
-        "miles davis ",   // 159
-        "rock and roll ", // 1007
-        "machine ",       // 3448
-        "spain ",         // 7002
-        "japan ",         // 10.593
-        "france ",        // 17.616
-        "film ",          // 24.959
-    ],
-    configure: base_conf,
-    ..Conf::BASE
-};
-
-fn bench_songs(c: &mut criterion::Criterion) {
-    let basic_with_quote: Vec<String> = BASE_CONF
-        .queries
-        .iter()
-        .map(|s| {
-            s.trim().split(' ').map(|s| format!(r#""{}""#, s)).collect::<Vec<String>>().join(" ")
-        })
-        .collect();
-    let basic_with_quote: &[&str] =
-        &basic_with_quote.iter().map(|s| s.as_str()).collect::<Vec<&str>>();
-
-    #[rustfmt::skip]
-    let confs = &[
-        /* first we bench each criterion alone */
-        utils::Conf {
-            group_name: "proximity",
-            queries: &[
-                "herald sings ",
-                "april paris ",
-                "tea two ",
-                "diesel engine ",
-            ],
-            criterion: Some(&["proximity"]),
-            optional_words: false,
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "typo",
-            queries: &[
-                "migrosoft ",
-                "linax ",
-                "Disnaylande ",
-                "phytogropher ",
-                "nympalidea ",
-                "aritmetric ",
-                "the fronce ",
-                "sisan ",
-            ],
-            criterion: Some(&["typo"]),
-            optional_words: false,
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "words",
-            queries: &[
-                "the black saint and the sinner lady and the good doggo ", // four words to pop, 27 results
-                "Kameya Tokujirō mingus monk ",                            // two words to pop, 55
-                "Ulrich Hensel meilisearch milli ",                        // two words to pop, 306
-                "Idaho Bellevue pizza ",                                   // one word to pop, 800
-                "Abraham machin ",                                         // one word to pop, 1141
-            ],
-            criterion: Some(&["words"]),
-            ..BASE_CONF
-        },
-        /* the we bench some global / normal search with all the default criterion in the default
-         * order */
-        utils::Conf {
-            group_name: "basic placeholder",
-            queries: &[""],
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "basic without quote",
-            queries: &BASE_CONF
-                .queries
-                .iter()
-                .map(|s| s.trim()) // we remove the space at the end of each request
-                .collect::<Vec<&str>>(),
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "basic with quote",
-            queries: basic_with_quote,
-            ..BASE_CONF
-        },
-        utils::Conf {
-            group_name: "prefix search",
-            queries: &[
-                "t", // 453k results
-                "c", // 405k
-                "g", // 318k
-                "j", // 227k
-                "q", // 71k
-                "x", // 17k
-            ],
-            ..BASE_CONF
-        },
-    ];
-
-    utils::run_benches(c, confs);
-}
-
-criterion_group!(benches, bench_songs);
-criterion_main!(benches);

benchmarks/benches/utils.rs

@@ -1,256 +0,0 @@
-#![allow(dead_code)]
-
-use std::fs::{create_dir_all, remove_dir_all, File};
-use std::io::{self, BufRead, BufReader, Cursor, Read, Seek};
-use std::num::ParseFloatError;
-use std::path::Path;
-use std::str::FromStr;
-
-use criterion::BenchmarkId;
-use milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
-use milli::heed::EnvOpenOptions;
-use milli::update::{
-    IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Settings,
-};
-use milli::{Criterion, Filter, Index, Object, TermsMatchingStrategy};
-use serde_json::Value;
-
-pub struct Conf<'a> {
-    /// where we are going to create our database.mmdb directory
-    /// each benchmark will first try to delete it and then recreate it
-    pub database_name: &'a str,
-    /// the dataset to be used, it must be an uncompressed csv
-    pub dataset: &'a str,
-    /// The format of the dataset
-    pub dataset_format: &'a str,
-    pub group_name: &'a str,
-    pub queries: &'a [&'a str],
-    /// here you can change which criterion are used and in which order.
-    /// - if you specify something all the base configuration will be thrown out
-    /// - if you don't specify anything (None) the default configuration will be kept
-    pub criterion: Option<&'a [&'a str]>,
-    /// the last chance to configure your database as you want
-    pub configure: fn(&mut Settings),
-    pub filter: Option<&'a str>,
-    pub sort: Option<Vec<&'a str>>,
-    /// enable or disable the optional words on the query
-    pub optional_words: bool,
-    /// primary key, if there is None we'll auto-generate docids for every documents
-    pub primary_key: Option<&'a str>,
-}
-
-impl Conf<'_> {
-    pub const BASE: Self = Conf {
-        database_name: "benches.mmdb",
-        dataset_format: "csv",
-        dataset: "",
-        group_name: "",
-        queries: &[],
-        criterion: None,
-        configure: |_| (),
-        filter: None,
-        sort: None,
-        optional_words: true,
-        primary_key: None,
-    };
-}
-
-pub fn base_setup(conf: &Conf) -> Index {
-    match remove_dir_all(conf.database_name) {
-        Ok(_) => (),
-        Err(e) if e.kind() == std::io::ErrorKind::NotFound => (),
-        Err(e) => panic!("{}", e),
-    }
-    create_dir_all(conf.database_name).unwrap();
-
-    let mut options = EnvOpenOptions::new();
-    options.map_size(100 * 1024 * 1024 * 1024); // 100 GB
-    options.max_readers(10);
-    let index = Index::new(options, conf.database_name).unwrap();
-
-    let config = IndexerConfig::default();
-    let mut wtxn = index.write_txn().unwrap();
-    let mut builder = Settings::new(&mut wtxn, &index, &config);
-
-    if let Some(primary_key) = conf.primary_key {
-        builder.set_primary_key(primary_key.to_string());
-    }
-
-    if let Some(criterion) = conf.criterion {
-        builder.reset_filterable_fields();
-        builder.reset_criteria();
-        builder.reset_stop_words();
-
-        let criterion = criterion.iter().map(|s| Criterion::from_str(s).unwrap()).collect();
-        builder.set_criteria(criterion);
-    }
-
-    (conf.configure)(&mut builder);
-
-    builder.execute(|_| (), || false).unwrap();
-    wtxn.commit().unwrap();
-
-    let config = IndexerConfig::default();
-    let mut wtxn = index.write_txn().unwrap();
-    let indexing_config = IndexDocumentsConfig {
-        autogenerate_docids: conf.primary_key.is_none(),
-        update_method: IndexDocumentsMethod::ReplaceDocuments,
-        ..Default::default()
-    };
-    let builder =
-        IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| (), || false).unwrap();
-    let documents = documents_from(conf.dataset, conf.dataset_format);
-    let (builder, user_error) = builder.add_documents(documents).unwrap();
-    user_error.unwrap();
-    builder.execute().unwrap();
-    wtxn.commit().unwrap();
-
-    index
-}
-
-pub fn run_benches(c: &mut criterion::Criterion, confs: &[Conf]) {
-    for conf in confs {
-        let index = base_setup(conf);
-
-        let file_name = Path::new(conf.dataset).file_name().and_then(|f| f.to_str()).unwrap();
-        let name = format!("{}: {}", file_name, conf.group_name);
-        let mut group = c.benchmark_group(&name);
-
-        for &query in conf.queries {
-            group.bench_with_input(BenchmarkId::from_parameter(query), &query, |b, &query| {
-                b.iter(|| {
-                    let rtxn = index.read_txn().unwrap();
-                    let mut search = index.search(&rtxn);
-                    search.query(query).terms_matching_strategy(TermsMatchingStrategy::default());
-                    if let Some(filter) = conf.filter {
-                        let filter = Filter::from_str(filter).unwrap().unwrap();
-                        search.filter(filter);
-                    }
-                    if let Some(sort) = &conf.sort {
-                        let sort = sort.iter().map(|sort| sort.parse().unwrap()).collect();
-                        search.sort_criteria(sort);
-                    }
-                    let _ids = search.execute().unwrap();
-                });
-            });
-        }
-        group.finish();
-
-        index.prepare_for_closing().wait();
-    }
-}
-
-pub fn documents_from(filename: &str, filetype: &str) -> DocumentsBatchReader<impl BufRead + Seek> {
-    let reader = File::open(filename)
-        .unwrap_or_else(|_| panic!("could not find the dataset in: {}", filename));
-    let reader = BufReader::new(reader);
-    let documents = match filetype {
-        "csv" => documents_from_csv(reader).unwrap(),
-        "json" => documents_from_json(reader).unwrap(),
-        "jsonl" => documents_from_jsonl(reader).unwrap(),
-        otherwise => panic!("invalid update format {:?}", otherwise),
-    };
-    DocumentsBatchReader::from_reader(Cursor::new(documents)).unwrap()
-}
-
-fn documents_from_jsonl(reader: impl BufRead) -> anyhow::Result<Vec<u8>> {
-    let mut documents = DocumentsBatchBuilder::new(Vec::new());
-
-    for result in serde_json::Deserializer::from_reader(reader).into_iter::<Object>() {
-        let object = result?;
-        documents.append_json_object(&object)?;
-    }
-
-    documents.into_inner().map_err(Into::into)
-}
-
-fn documents_from_json(reader: impl BufRead) -> anyhow::Result<Vec<u8>> {
-    let mut documents = DocumentsBatchBuilder::new(Vec::new());
-
-    documents.append_json_array(reader)?;
-
-    documents.into_inner().map_err(Into::into)
-}
-
-fn documents_from_csv(reader: impl BufRead) -> anyhow::Result<Vec<u8>> {
-    let csv = csv::Reader::from_reader(reader);
-
-    let mut documents = DocumentsBatchBuilder::new(Vec::new());
-    documents.append_csv(csv)?;
-
-    documents.into_inner().map_err(Into::into)
-}
-
-enum AllowedType {
-    String,
-    Number,
-}
-
-fn parse_csv_header(header: &str) -> (String, AllowedType) {
-    // if there are several separators we only split on the last one.
-    match header.rsplit_once(':') {
-        Some((field_name, field_type)) => match field_type {
-            "string" => (field_name.to_string(), AllowedType::String),
-            "number" => (field_name.to_string(), AllowedType::Number),
-            // we may return an error in this case.
-            _otherwise => (header.to_string(), AllowedType::String),
-        },
-        None => (header.to_string(), AllowedType::String),
-    }
-}
-
-struct CSVDocumentDeserializer<R>
-where
-    R: Read,
-{
-    documents: csv::StringRecordsIntoIter<R>,
-    headers: Vec<(String, AllowedType)>,
-}
-
-impl<R: Read> CSVDocumentDeserializer<R> {
-    fn from_reader(reader: R) -> io::Result<Self> {
-        let mut records = csv::Reader::from_reader(reader);
-
-        let headers = records.headers()?.into_iter().map(parse_csv_header).collect();
-
-        Ok(Self { documents: records.into_records(), headers })
-    }
-}
-
-impl<R: Read> Iterator for CSVDocumentDeserializer<R> {
-    type Item = anyhow::Result<Object>;
-
-    fn next(&mut self) -> Option<Self::Item> {
-        let csv_document = self.documents.next()?;
-
-        match csv_document {
-            Ok(csv_document) => {
-                let mut document = Object::new();
-
-                for ((field_name, field_type), value) in
-                    self.headers.iter().zip(csv_document.into_iter())
-                {
-                    let parsed_value: Result<Value, ParseFloatError> = match field_type {
-                        AllowedType::Number => {
-                            value.parse::<f64>().map(Value::from).map_err(Into::into)
-                        }
-                        AllowedType::String => Ok(Value::String(value.to_string())),
-                    };
-
-                    match parsed_value {
-                        Ok(value) => drop(document.insert(field_name.to_string(), value)),
-                        Err(_e) => {
-                            return Some(Err(anyhow::anyhow!(
-                                "Value '{}' is not a valid number",
-                                value
-                            )))
-                        }
-                    }
-                }
-
-                Some(Ok(document))
-            }
-            Err(e) => Some(Err(anyhow::anyhow!("Error parsing csv document: {}", e))),
-        }
-    }
-}

benchmarks/build.rs

@@ -1,115 +0,0 @@
-use std::fs::File;
-use std::io::{Cursor, Read, Seek, Write};
-use std::path::{Path, PathBuf};
-use std::{env, fs};
-
-use bytes::Bytes;
-use convert_case::{Case, Casing};
-use flate2::read::GzDecoder;
-use reqwest::IntoUrl;
-
-const BASE_URL: &str = "https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets";
-
-const DATASET_SONGS: (&str, &str) = ("smol-songs", "csv");
-const DATASET_SONGS_1_2: (&str, &str) = ("smol-songs-1_2", "csv");
-const DATASET_SONGS_3_4: (&str, &str) = ("smol-songs-3_4", "csv");
-const DATASET_SONGS_4_4: (&str, &str) = ("smol-songs-4_4", "csv");
-const DATASET_WIKI: (&str, &str) = ("smol-wiki-articles", "csv");
-const DATASET_WIKI_1_2: (&str, &str) = ("smol-wiki-articles-1_2", "csv");
-const DATASET_WIKI_3_4: (&str, &str) = ("smol-wiki-articles-3_4", "csv");
-const DATASET_WIKI_4_4: (&str, &str) = ("smol-wiki-articles-4_4", "csv");
-const DATASET_MOVIES: (&str, &str) = ("movies", "json");
-const DATASET_MOVIES_1_2: (&str, &str) = ("movies-1_2", "json");
-const DATASET_MOVIES_3_4: (&str, &str) = ("movies-3_4", "json");
-const DATASET_MOVIES_4_4: (&str, &str) = ("movies-4_4", "json");
-const DATASET_NESTED_MOVIES: (&str, &str) = ("nested_movies", "json");
-const DATASET_GEO: (&str, &str) = ("smol-all-countries", "jsonl");
-
-const ALL_DATASETS: &[(&str, &str)] = &[
-    DATASET_SONGS,
-    DATASET_SONGS_1_2,
-    DATASET_SONGS_3_4,
-    DATASET_SONGS_4_4,
-    DATASET_WIKI,
-    DATASET_WIKI_1_2,
-    DATASET_WIKI_3_4,
-    DATASET_WIKI_4_4,
-    DATASET_MOVIES,
-    DATASET_MOVIES_1_2,
-    DATASET_MOVIES_3_4,
-    DATASET_MOVIES_4_4,
-    DATASET_NESTED_MOVIES,
-    DATASET_GEO,
-];
-
-/// The name of the environment variable used to select the path
-/// of the directory containing the datasets
-const BASE_DATASETS_PATH_KEY: &str = "MILLI_BENCH_DATASETS_PATH";
-
-fn main() -> anyhow::Result<()> {
-    let out_dir = PathBuf::from(env::var(BASE_DATASETS_PATH_KEY).unwrap_or(env::var("OUT_DIR")?));
-
-    let benches_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).join("benches");
-    let mut manifest_paths_file = File::create(benches_dir.join("datasets_paths.rs"))?;
-    write!(
-        manifest_paths_file,
-        r#"//! This file is generated by the build script.
-//! Do not modify by hand, use the build.rs file.
-#![allow(dead_code)]
-"#
-    )?;
-    writeln!(manifest_paths_file)?;
-
-    for (dataset, extension) in ALL_DATASETS {
-        let out_path = out_dir.join(dataset);
-        let out_file = out_path.with_extension(extension);
-
-        writeln!(
-            &mut manifest_paths_file,
-            r#"pub const {}: &str = {:?};"#,
-            dataset.to_case(Case::ScreamingSnake),
-            out_file.display(),
-        )?;
-
-        if out_file.exists() {
-            eprintln!(
-                "The dataset {} already exists on the file system and will not be downloaded again",
-                out_path.display(),
-            );
-            continue;
-        }
-        let url = format!("{}/{}.{}.gz", BASE_URL, dataset, extension);
-        eprintln!("downloading: {}", url);
-        let bytes = retry(|| download_dataset(url.clone()), 10)?;
-        eprintln!("{} downloaded successfully", url);
-        eprintln!("uncompressing in {}", out_file.display());
-        uncompress_in_file(bytes, &out_file)?;
-    }
-
-    Ok(())
-}
-
-fn retry<Ok, Err>(fun: impl Fn() -> Result<Ok, Err>, times: usize) -> Result<Ok, Err> {
-    for _ in 0..times {
-        if let ok @ Ok(_) = fun() {
-            return ok;
-        }
-    }
-    fun()
-}
-
-fn download_dataset<U: IntoUrl>(url: U) -> anyhow::Result<Cursor<Bytes>> {
-    let bytes =
-        reqwest::blocking::Client::builder().timeout(None).build()?.get(url).send()?.bytes()?;
-    Ok(Cursor::new(bytes))
-}
-
-fn uncompress_in_file<R: Read + Seek, P: AsRef<Path>>(bytes: R, path: P) -> anyhow::Result<()> {
-    let path = path.as_ref();
-    let mut gz = GzDecoder::new(bytes);
-    let mut dataset = Vec::new();
-    gz.read_to_end(&mut dataset)?;
-
-    fs::write(path, dataset)?;
-    Ok(())
-}

benchmarks/scripts/compare.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-
-# Requirements:
-# - critcmp. See: https://github.com/BurntSushi/critcmp
-# - curl
-
-# Usage
-# $ bash compare.sh json_file1 json_file1
-# ex: bash compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json
-
-# Checking that critcmp is installed
-command -v critcmp > /dev/null 2>&1
-if [[ "$?" -ne 0 ]]; then
-    echo 'You must install critcmp to make this script work.'
-    echo 'See: https://github.com/BurntSushi/critcmp'
-    echo '  $ cargo install critcmp'
-    exit 1
-fi
-
-s3_url='https://milli-benchmarks.fra1.digitaloceanspaces.com/critcmp_results'
-
-for file in $@
-do
-    file_s3_url="$s3_url/$file"
-    file_local_path="/tmp/$file"
-
-    if [[ ! -f $file_local_path ]]; then
-        curl $file_s3_url --output $file_local_path --silent
-        if [[ "$?" -ne 0 ]]; then
-            echo 'curl command failed.'
-            exit 1
-        fi
-    fi
-done
-
-path_list=$(echo " $@" | sed 's/ / \/tmp\//g')
-
-critcmp $path_list

benchmarks/scripts/list.sh

@@ -1,14 +0,0 @@
-#!/usr/bin/env bash
-
-# Requirements:
-# - curl
-# - grep
-
-res=$(curl -s https://milli-benchmarks.fra1.digitaloceanspaces.com | grep -o '<Key>[^<]\+' | cut -c 5- | grep critcmp_results/ | cut -c 18-)
-
-for pattern in "$@"
-do
-	res=$(echo "$res" | grep $pattern)
-done
-
-echo "$res"

benchmarks/src/lib.rs

@@ -1,5 +0,0 @@
-//! This library is only used to isolate the benchmarks
-//! from the original milli library.
-//!
-//! It does not include interesting functions for milli library
-//! users only for milli contributors.

bors.toml

@@ -1,11 +0,0 @@
-status = [
-    'Tests on ubuntu-18.04',
-    'Tests on macos-12',
-    'Tests on windows-2022',
-    'Run Clippy',
-    'Run Rustfmt',
-    'Run tests in debug',
-]
-pr_status = ['Milestone Check']
-# 3 hours timeout
-timeout-sec = 10800

config.toml

@@ -1,131 +0,0 @@
-# This file shows the default configuration of Meilisearch.
-# All variables are defined here: https://www.meilisearch.com/docs/learn/configuration/instance_options#environment-variables
-
-# Designates the location where database files will be created and retrieved.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#database-path
-db_path = "./data.ms"
-
-# Configures the instance's environment. Value must be either `production` or `development`.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#environment
-env = "development"
-
-# The address on which the HTTP server will listen.
-http_addr = "localhost:7700"
-
-# Sets the instance's master key, automatically protecting all routes except GET /health.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#master-key
-# master_key = "YOUR_MASTER_KEY_VALUE"
-
-# Deactivates Meilisearch's built-in telemetry when provided.
-# Meilisearch automatically collects data from all instances that do not opt out using this flag.
-# All gathered data is used solely for the purpose of improving Meilisearch, and can be deleted at any time.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#disable-analytics
-# no_analytics = true
-
-# Sets the maximum size of accepted payloads.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#payload-limit-size
-http_payload_size_limit = "100 MB"
-
-# Defines how much detail should be present in Meilisearch's logs.
-# Meilisearch currently supports six log levels, listed in order of increasing verbosity:  `OFF`, `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#log-level
-log_level = "INFO"
-
-# Sets the maximum amount of RAM Meilisearch can use when indexing.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#max-indexing-memory
-# max_indexing_memory = "2 GiB"
-
-# Sets the maximum number of threads Meilisearch can use during indexing.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#max-indexing-threads
-# max_indexing_threads = 4
-
-#############
-### DUMPS ###
-#############
-
-# Sets the directory where Meilisearch will create dump files.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#dump-directory
-dump_dir = "dumps/"
-
-# Imports the dump file located at the specified path. Path must point to a .dump file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#import-dump
-# import_dump = "./path/to/my/file.dump"
-
-# Prevents Meilisearch from throwing an error when `import_dump` does not point to a valid dump file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-missing-dump
-ignore_missing_dump = false
-
-# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_dump`.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-dump-if-db-exists
-ignore_dump_if_db_exists = false
-
-
-#################
-### SNAPSHOTS ###
-#################
-
-# Enables scheduled snapshots when true, disable when false (the default).
-# If the value is given as an integer, then enables the scheduled snapshot with the passed value as the interval
-# between each snapshot, in seconds.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#schedule-snapshot-creation
-schedule_snapshot = false
-
-# Sets the directory where Meilisearch will store snapshots.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#snapshot-destination
-snapshot_dir = "snapshots/"
-
-# Launches Meilisearch after importing a previously-generated snapshot at the given filepath.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#import-snapshot
-# import_snapshot = "./path/to/my/snapshot"
-
-# Prevents a Meilisearch instance from throwing an error when `import_snapshot` does not point to a valid snapshot file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-missing-snapshot
-ignore_missing_snapshot = false
-
-# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_snapshot`.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-snapshot-if-db-exists
-ignore_snapshot_if_db_exists = false
-
-
-###########
-### SSL ###
-###########
-
-# Enables client authentication in the specified path.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-authentication-path
-# ssl_auth_path = "./path/to/root"
-
-# Sets the server's SSL certificates.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-certificates-path
-# ssl_cert_path = "./path/to/certfile"
-
-# Sets the server's SSL key files.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-key-path
-# ssl_key_path = "./path/to/private-key"
-
-# Sets the server's OCSP file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-ocsp-path
-# ssl_ocsp_path = "./path/to/ocsp-file"
-
-# Makes SSL authentication mandatory.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-require-auth
-ssl_require_auth = false
-
-# Activates SSL session resumption.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-resumption
-ssl_resumption = false
-
-# Activates SSL tickets.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-tickets
-ssl_tickets = false
-
-#############################
-### Experimental features ###
-#############################
-
-# Experimental metrics feature. For more information, see: <https://github.com/meilisearch/meilisearch/discussions/3518>
-# Enables the Prometheus metrics on the `GET /metrics` endpoint.
-experimental_enable_metrics = false
-
-# Experimental RAM reduction during indexing, do not use in production, see: <https://github.com/meilisearch/product/discussions/652>
-experimental_reduce_indexing_memory_usage = false

datasets/movies/README.md

@@ -0,0 +1 @@
+_datas in movies.csv are from https://www.themoviedb.org/_

datasets/movies/schema.toml

@@ -0,0 +1,21 @@
+# This schema has been generated ...
+# The order in which the attributes are declared is important,
+# it specify the attribute xxx...
+identifier = "id"
+
+[attributes.id]
+displayed = true
+
+[attributes.title]
+displayed = true
+indexed = true
+
+[attributes.overview]
+displayed = true
+indexed = true
+
+[attributes.release_date]
+displayed = true
+
+[attributes.poster]
+displayed = true

deep-dive.md

@@ -0,0 +1,95 @@
+# A deep dive in MeiliSearch
+
+On the 15 of May 2019.
+
+MeiliSearch is a full text search engine based on a final state transducer named [fst](https://github.com/BurntSushi/fst) and a key-value store named [sled](https://github.com/spacejam/sled). The goal of a search engine is to store data and to respond to queries as accurate and fast as possible. To achieve this it must save the matching words in an [inverted index](https://en.wikipedia.org/wiki/Inverted_index).
+
+<!-- MarkdownTOC autolink="true" -->
+
+- [Where is the data stored?](#where-is-the-data-stored)
+- [What does the key-value store contains?](#what-does-the-key-value-store-contains)
+    - [The inverted word index](#the-inverted-word-index)
+        - [A final state transducer](#a-final-state-transducer)
+        - [Document indexes](#document-indexes)
+    - [The schema](#the-schema)
+    - [Document attributes](#document-attributes)
+- [How is a request processed?](#how-is-a-request-processed)
+    - [Query lexemes](#query-lexemes)
+    - [Automatons and query index](#automatons-and-query-index)
+    - [Sort by criteria](#sort-by-criteria)
+
+<!-- /MarkdownTOC -->
+
+## Where is the data stored?
+
+MeiliSearch is entirely backed by a key-value store like any good database (i.e. Postgres, MySQL). This brings a great flexibility in the way documents can be stored and updates handled along time.
+
+[sled will brings some](https://github.com/spacejam/sled/tree/434533332a3f485e6d2e467023be0a0b55d3a1af#plans) of the [A.C.I.D. properties](https://en.wikipedia.org/wiki/ACID_(computer_science)) to help us be sure the saved data is consistent.
+
+
+
+## What does the key-value store contains?
+
+It contain the inverted word index, the schema and the documents fields.
+
+### The inverted word index
+
+[The inverted word index](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/words_index.rs) is a sled Tree dedicated to store and give access to all documents that contains a specific word. The information stored under the word is simply a big ordered array of where in the document the word has been found. In other word, a big list of [`DocIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/lib.rs#L35-L51).
+
+#### A final state transducer
+
+_...also abbreviated fst_
+
+This is the first entry point of the engine, you can read more about how it work with the beautiful blog post of @BurntSushi, [Index 1,600,000,000 Keys with Automata and Rust](https://blog.burntsushi.net/transducers/).
+
+To make it short it is a powerful way to store all the words that are present in the indexed documents. You construct it by giving it all the words you want to index. When you want to search in it you can provide any automaton you want, in MeiliSearch [a custom levenshtein automaton](https://github.com/tantivy-search/levenshtein-automata/) is used.
+
+#### Document indexes
+
+The `fst` will only return the words that match with the search automaton but the goal of the search engine is to retrieve all matches in all the documents when a query is made. You want it to return some sort of position in an attribute in a document, an information about where the given word matched.
+
+To make it possible we retrieve all of the `DocIndex` corresponding to all the matching words in the fst, we use the [`WordsIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/words_index.rs#L11-L21) Tree to get the `DocIndexes` corresponding the words.
+
+### The schema
+
+The schema is a data structure that represents which documents attributes should be stored and which should be indexed. It is stored under a the [`MainIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/main_index.rs#L12) Tree and given to MeiliSearch only at the creation of an index.
+
+Each document attribute is associated to a unique 16 bit number named [`SchemaAttr`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/schema.rs#L186).
+
+In the future, this schema type could be given along with updates, the database could be able to handled a new schema and reindex the database according to the new one.
+
+### Document attributes
+
+When the engine handle a query the result that the requester want is a document, not only the [`Matches`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/lib.rs#L62-L88) associated to it, fields of the original document must be returned too.
+
+So MeiliSearch again uses the power of the underlying key-value store and save the documents attributes marked as _STORE_ in the schema. The dedicated Tree for this information is the [`DocumentsIndex`](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/documents_index.rs#L11).
+
+When a document field is saved in the key-value store its value is binary encoded using [message pack](https://github.com/3Hren/msgpack-rust), so a document must be serializable using serde.
+
+
+
+## How is a request processed?
+
+Now that we have our inverted index we are able to return results based on a query. In the MeiliSearch universe a query is a simple string containing words.
+
+### Query lexemes
+
+The first step to be able to call the underlying structures is to split the query in words, for that we use a [custom tokenizer](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-tokenizer/src/lib.rs#L82-L84). Note that a tokenizer is specialized for a human language, this is the hard part.
+
+### Automatons and query index
+
+So to query the fst we need an automaton, in MeiliSearch we use a [levenshtein automaton](https://en.wikipedia.org/wiki/Levenshtein_automaton), this automaton is constructed using a string and a maximum distance. According to the [Algolia's blog post](https://blog.algolia.com/inside-the-algolia-engine-part-3-query-processing/#algolia%e2%80%99s-way-of-searching-for-alternatives) we [created the DFAs](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/automaton.rs#L59-L78) with different settings.
+
+Thanks to the power of the fst library [it is possible to union multiple automatons](https://docs.rs/fst/0.3.2/fst/map/struct.OpBuilder.html#method.union) on the same fst set. The `Stream` is able to return all the matching words. We use these words to find the whole list of `DocIndexes` associated.
+
+With all these informations it is possible [to reconstruct a list of all the `DocIndexes` associated](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/query_builder.rs#L103-L130) with the words queried.
+
+### Sort by criteria
+
+Now that we are able to get a big list of [DocIndexes](https://github.com/Kerollmops/MeiliSearch/blob/550dc1e99224e386516877450320f694947332d4/src/lib.rs#L21-L36) it is not enough to sort them by criteria, we need more informations like the levenshtein distance or the fact that a query word match exactly the word stored in the fst. So [we stuff it a little bit](https://github.com/Kerollmops/MeiliSearch/blob/550dc1e99224e386516877450320f694947332d4/src/rank/query_builder.rs#L86-L93), and aggregate all these [Matches](https://github.com/Kerollmops/MeiliSearch/blob/550dc1e99224e386516877450320f694947332d4/src/lib.rs#L47-L74) for each document. This way it will be easy to sort a simple vector of document using a bunch of functions.
+
+With this big list of documents and associated matches [we are able to sort only the part of the slice that we want](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/query_builder.rs#L160-L188) using bucket sorting. [Each criterion](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-core/src/criterion/mod.rs#L95-L101) is evaluated on each subslice without copy, thanks to [GroupByMut](https://docs.rs/slice-group-by/0.2.4/slice_group_by/) which, I hope [will soon be merged](https://github.com/rust-lang/rfcs/pull/2477).
+
+Note that it is possible to customize the criteria used by using the `QueryBuilder::with_criteria` constructor, this way you can implement some custom ranking based on the document attributes using the appropriate structure and the [`document` method](https://github.com/meilisearch/MeiliSearch/blob/3db823de002243004612e36a19b4578d800dab97/meilisearch-data/src/database/index.rs#L86).
+
+At this point, MeiliSearch work is over 🎉

download-latest.sh

@@ -1,167 +0,0 @@
-#!/bin/sh
-
-# This script can optionally use a GitHub token to increase your request limit (for example, if using this script in a CI).
-# To use a GitHub token, pass it through the GITHUB_PAT environment variable.
-
-# GLOBALS
-
-# Colors
-RED='\033[31m'
-GREEN='\033[32m'
-DEFAULT='\033[0m'
-
-# Project name
-PNAME='meilisearch'
-
-# GitHub API address
-GITHUB_API='https://api.github.com/repos/meilisearch/meilisearch/releases'
-# GitHub Release address
-GITHUB_REL='https://github.com/meilisearch/meilisearch/releases/download/'
-
-# FUNCTIONS
-
-# Gets the version of the latest stable version of Meilisearch by setting the $latest variable.
-# Returns 0 in case of success, 1 otherwise.
-get_latest() {
-    # temp_file is needed because the grep would start before the download is over
-    temp_file=$(mktemp -q /tmp/$PNAME.XXXXXXXXX)
-    latest_release="$GITHUB_API/latest"
-
-    if [ $? -ne 0 ]; then
-        echo "$0: Can't create temp file."
-        fetch_release_failure_usage
-        exit 1
-    fi
-
-    if [ -z "$GITHUB_PAT" ]; then
-        curl -s "$latest_release" > "$temp_file" || return 1
-    else
-        curl -H "Authorization: token $GITHUB_PAT" -s "$latest_release" > "$temp_file" || return 1
-    fi
-
-    latest="$(cat "$temp_file" | grep '"tag_name":' | cut -d ':' -f2 | tr -d '"' | tr -d ',' | tr -d ' ')"
-
-    rm -f "$temp_file"
-    return 0
-}
-
-# Gets the OS by setting the $os variable.
-# Returns 0 in case of success, 1 otherwise.
-get_os() {
-    os_name=$(uname -s)
-    case "$os_name" in
-    'Darwin')
-        os='macos'
-        ;;
-    'Linux')
-        os='linux'
-        ;;
-     'MINGW'*)
-        os='windows'
-        ;;
-    *)
-        return 1
-    esac
-    return 0
-}
-
-# Gets the architecture by setting the $archi variable.
-# Returns 0 in case of success, 1 otherwise.
-get_archi() {
-    architecture=$(uname -m)
-    case "$architecture" in
-    'x86_64' | 'amd64' )
-        archi='amd64'
-        ;;
-    'arm64')
-        # macOS M1/M2
-        if [ $os = 'macos' ]; then
-            archi='apple-silicon'
-        else
-            archi='aarch64'
-        fi
-        ;;
-    'aarch64')
-        archi='aarch64'
-        ;;
-    *)
-        return 1
-    esac
-    return 0
-}
-
-success_usage() {
-    printf "$GREEN%s\n$DEFAULT" "Meilisearch $latest binary successfully downloaded as '$binary_name' file."
-    echo ''
-    echo 'Run it:'
-    echo "    $ ./$PNAME"
-    echo 'Usage:'
-    echo "    $ ./$PNAME --help"
-}
-
-not_available_failure_usage() {
-    printf "$RED%s\n$DEFAULT" 'ERROR: Meilisearch binary is not available for your OS distribution or your architecture yet.'
-    echo ''
-    echo 'However, you can easily compile the binary from the source files.'
-    echo 'Follow the steps at the page ("Source" tab): https://www.meilisearch.com/docs/learn/getting_started/installation'
-}
-
-fetch_release_failure_usage() {
-    echo ''
-    printf "$RED%s\n$DEFAULT" 'ERROR: Impossible to get the latest stable version of Meilisearch.'
-    echo 'Please let us know about this issue: https://github.com/meilisearch/meilisearch/issues/new/choose'
-    echo ''
-    echo 'In the meantime, you can manually download the appropriate binary from the GitHub release assets here: https://github.com/meilisearch/meilisearch/releases/latest'
-}
-
-fill_release_variables() {
-    # Fill $latest variable.
-    if ! get_latest; then
-        fetch_release_failure_usage
-        exit 1
-    fi
-    if [ "$latest" = '' ]; then
-        fetch_release_failure_usage
-        exit 1
-     fi
-     # Fill $os variable.
-     if ! get_os; then
-        not_available_failure_usage
-        exit 1
-     fi
-     # Fill $archi variable.
-     if ! get_archi; then
-        not_available_failure_usage
-        exit 1
-     fi
-}
-
-download_binary() {
-    fill_release_variables
-    echo "Downloading Meilisearch binary $latest for $os, architecture $archi..."
-    case "$os" in
-        'windows')
-            release_file="$PNAME-$os-$archi.exe"
-            binary_name="$PNAME.exe"
-            ;;
-        *)
-            release_file="$PNAME-$os-$archi"
-            binary_name="$PNAME"
-    esac
-    # Fetch the Meilisearch binary.
-    curl --fail -OL "$GITHUB_REL/$latest/$release_file"
-    if [ $? -ne 0 ]; then
-        fetch_release_failure_usage
-        exit 1
-    fi
-    mv "$release_file" "$binary_name"
-    chmod 744 "$binary_name"
-    success_usage
-}
-
-# MAIN
-
-main() {
-    download_binary
-}
-main

dump/Cargo.toml

@@ -1,35 +0,0 @@
-[package]
-name = "dump"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-edition.workspace = true
-homepage.workspace = true
-readme.workspace = true
-license.workspace = true
-
-[dependencies]
-anyhow = "1.0.70"
-flate2 = "1.0.25"
-http = "0.2.9"
-log = "0.4.17"
-meilisearch-auth = { path = "../meilisearch-auth" }
-meilisearch-types = { path = "../meilisearch-types" }
-once_cell = "1.17.1"
-regex = "1.7.3"
-roaring = { version = "0.10.1", features = ["serde"] }
-serde = { version = "1.0.160", features = ["derive"] }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
-tar = "0.4.38"
-tempfile = "3.5.0"
-thiserror = "1.0.40"
-time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-uuid = { version = "1.3.1", features = ["serde", "v4"] }
-
-[dev-dependencies]
-big_s = "1.0.2"
-maplit = "1.0.2"
-meili-snap = { path = "../meili-snap" }
-meilisearch-types = { path = "../meilisearch-types" }

dump/README.md

@@ -1,17 +0,0 @@
-```
-dump
-├── indexes
-│   ├── cattos
-│   │   ├── documents.jsonl
-│   │   └── settings.json
-│   └── doggos
-│       ├── documents.jsonl
-│       └── settings.json
-├── instance-uid.uuid
-├── keys.jsonl
-├── metadata.json
-└── tasks
-    ├── update_files
-    │   └── [task_id].jsonl
-    └── queue.jsonl
-```

dump/src/error.rs

@@ -1,34 +0,0 @@
-use meilisearch_types::error::{Code, ErrorCode};
-use thiserror::Error;
-
-#[derive(Debug, Error)]
-pub enum Error {
-    #[error("Bad index name.")]
-    BadIndexName,
-    #[error("Malformed task.")]
-    MalformedTask,
-
-    #[error(transparent)]
-    Io(#[from] std::io::Error),
-    #[error(transparent)]
-    Serde(#[from] serde_json::Error),
-    #[error(transparent)]
-    Uuid(#[from] uuid::Error),
-}
-
-impl ErrorCode for Error {
-    fn error_code(&self) -> Code {
-        match self {
-            Error::Io(e) => e.error_code(),
-
-            // These errors either happen when creating a dump and don't need any error code,
-            // or come from an internal bad deserialization.
-            Error::Serde(_) => Code::Internal,
-            Error::Uuid(_) => Code::Internal,
-
-            // all these errors should never be raised when creating a dump, thus no error code should be associated.
-            Error::BadIndexName => Code::Internal,
-            Error::MalformedTask => Code::Internal,
-        }
-    }
-}

dump/src/lib.rs

@@ -1,479 +0,0 @@
-#![allow(clippy::type_complexity)]
-#![allow(clippy::wrong_self_convention)]
-
-use meilisearch_types::error::ResponseError;
-use meilisearch_types::keys::Key;
-use meilisearch_types::milli::update::IndexDocumentsMethod;
-use meilisearch_types::settings::Unchecked;
-use meilisearch_types::tasks::{Details, IndexSwap, KindWithContent, Status, Task, TaskId};
-use meilisearch_types::InstanceUid;
-use roaring::RoaringBitmap;
-use serde::{Deserialize, Serialize};
-use time::OffsetDateTime;
-
-mod error;
-mod reader;
-mod writer;
-
-pub use error::Error;
-pub use reader::{DumpReader, UpdateFile};
-pub use writer::DumpWriter;
-
-const CURRENT_DUMP_VERSION: Version = Version::V6;
-
-type Result<T> = std::result::Result<T, Error>;
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct Metadata {
-    pub dump_version: Version,
-    pub db_version: String,
-    #[serde(with = "time::serde::rfc3339")]
-    pub dump_date: OffsetDateTime,
-}
-
-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct IndexMetadata {
-    pub uid: String,
-    pub primary_key: Option<String>,
-    #[serde(with = "time::serde::rfc3339")]
-    pub created_at: OffsetDateTime,
-    #[serde(with = "time::serde::rfc3339")]
-    pub updated_at: OffsetDateTime,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
-pub enum Version {
-    V1,
-    V2,
-    V3,
-    V4,
-    V5,
-    V6,
-}
-
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct TaskDump {
-    pub uid: TaskId,
-    #[serde(default)]
-    pub index_uid: Option<String>,
-    pub status: Status,
-    #[serde(rename = "type")]
-    pub kind: KindDump,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub canceled_by: Option<TaskId>,
-
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub details: Option<Details>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub error: Option<ResponseError>,
-
-    #[serde(with = "time::serde::rfc3339")]
-    pub enqueued_at: OffsetDateTime,
-    #[serde(
-        with = "time::serde::rfc3339::option",
-        skip_serializing_if = "Option::is_none",
-        default
-    )]
-    pub started_at: Option<OffsetDateTime>,
-    #[serde(
-        with = "time::serde::rfc3339::option",
-        skip_serializing_if = "Option::is_none",
-        default
-    )]
-    pub finished_at: Option<OffsetDateTime>,
-}
-
-// A `Kind` specific version made for the dump. If modified you may break the dump.
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub enum KindDump {
-    DocumentImport {
-        primary_key: Option<String>,
-        method: IndexDocumentsMethod,
-        documents_count: u64,
-        allow_index_creation: bool,
-    },
-    DocumentDeletion {
-        documents_ids: Vec<String>,
-    },
-    DocumentClear,
-    DocumentDeletionByFilter {
-        filter: serde_json::Value,
-    },
-    Settings {
-        settings: Box<meilisearch_types::settings::Settings<Unchecked>>,
-        is_deletion: bool,
-        allow_index_creation: bool,
-    },
-    IndexDeletion,
-    IndexCreation {
-        primary_key: Option<String>,
-    },
-    IndexUpdate {
-        primary_key: Option<String>,
-    },
-    IndexSwap {
-        swaps: Vec<IndexSwap>,
-    },
-    TaskCancelation {
-        query: String,
-        tasks: RoaringBitmap,
-    },
-    TasksDeletion {
-        query: String,
-        tasks: RoaringBitmap,
-    },
-    DumpCreation {
-        keys: Vec<Key>,
-        instance_uid: Option<InstanceUid>,
-    },
-    SnapshotCreation,
-}
-
-impl From<Task> for TaskDump {
-    fn from(task: Task) -> Self {
-        TaskDump {
-            uid: task.uid,
-            index_uid: task.index_uid().map(|uid| uid.to_string()),
-            status: task.status,
-            kind: task.kind.into(),
-            canceled_by: task.canceled_by,
-            details: task.details,
-            error: task.error,
-            enqueued_at: task.enqueued_at,
-            started_at: task.started_at,
-            finished_at: task.finished_at,
-        }
-    }
-}
-
-impl From<KindWithContent> for KindDump {
-    fn from(kind: KindWithContent) -> Self {
-        match kind {
-            KindWithContent::DocumentAdditionOrUpdate {
-                primary_key,
-                method,
-                documents_count,
-                allow_index_creation,
-                ..
-            } => KindDump::DocumentImport {
-                primary_key,
-                method,
-                documents_count,
-                allow_index_creation,
-            },
-            KindWithContent::DocumentDeletion { documents_ids, .. } => {
-                KindDump::DocumentDeletion { documents_ids }
-            }
-            KindWithContent::DocumentDeletionByFilter { filter_expr, .. } => {
-                KindDump::DocumentDeletionByFilter { filter: filter_expr }
-            }
-            KindWithContent::DocumentClear { .. } => KindDump::DocumentClear,
-            KindWithContent::SettingsUpdate {
-                new_settings,
-                is_deletion,
-                allow_index_creation,
-                ..
-            } => KindDump::Settings { settings: new_settings, is_deletion, allow_index_creation },
-            KindWithContent::IndexDeletion { .. } => KindDump::IndexDeletion,
-            KindWithContent::IndexCreation { primary_key, .. } => {
-                KindDump::IndexCreation { primary_key }
-            }
-            KindWithContent::IndexUpdate { primary_key, .. } => {
-                KindDump::IndexUpdate { primary_key }
-            }
-            KindWithContent::IndexSwap { swaps } => KindDump::IndexSwap { swaps },
-            KindWithContent::TaskCancelation { query, tasks } => {
-                KindDump::TaskCancelation { query, tasks }
-            }
-            KindWithContent::TaskDeletion { query, tasks } => {
-                KindDump::TasksDeletion { query, tasks }
-            }
-            KindWithContent::DumpCreation { keys, instance_uid } => {
-                KindDump::DumpCreation { keys, instance_uid }
-            }
-            KindWithContent::SnapshotCreation => KindDump::SnapshotCreation,
-        }
-    }
-}
-
-#[cfg(test)]
-pub(crate) mod test {
-    use std::fs::File;
-    use std::io::Seek;
-    use std::str::FromStr;
-
-    use big_s::S;
-    use maplit::{btreemap, btreeset};
-    use meilisearch_types::facet_values_sort::FacetValuesSort;
-    use meilisearch_types::index_uid_pattern::IndexUidPattern;
-    use meilisearch_types::keys::{Action, Key};
-    use meilisearch_types::milli;
-    use meilisearch_types::milli::update::Setting;
-    use meilisearch_types::settings::{Checked, FacetingSettings, Settings};
-    use meilisearch_types::tasks::{Details, Status};
-    use serde_json::{json, Map, Value};
-    use time::macros::datetime;
-    use uuid::Uuid;
-
-    use crate::reader::Document;
-    use crate::{DumpReader, DumpWriter, IndexMetadata, KindDump, TaskDump, Version};
-
-    pub fn create_test_instance_uid() -> Uuid {
-        Uuid::parse_str("9e15e977-f2ae-4761-943f-1eaf75fd736d").unwrap()
-    }
-
-    pub fn create_test_index_metadata() -> IndexMetadata {
-        IndexMetadata {
-            uid: S("doggo"),
-            primary_key: None,
-            created_at: datetime!(2022-11-20 12:00 UTC),
-            updated_at: datetime!(2022-11-21 00:00 UTC),
-        }
-    }
-
-    pub fn create_test_documents() -> Vec<Map<String, Value>> {
-        vec![
-            json!({ "id": 1, "race": "golden retriever", "name": "paul", "age": 4 })
-                .as_object()
-                .unwrap()
-                .clone(),
-            json!({ "id": 2, "race": "bernese mountain", "name": "tamo", "age": 6 })
-                .as_object()
-                .unwrap()
-                .clone(),
-            json!({ "id": 3, "race": "great pyrenees", "name": "patou", "age": 5 })
-                .as_object()
-                .unwrap()
-                .clone(),
-        ]
-    }
-
-    pub fn create_test_settings() -> Settings<Checked> {
-        let settings = Settings {
-            displayed_attributes: Setting::Set(vec![S("race"), S("name")]),
-            searchable_attributes: Setting::Set(vec![S("name"), S("race")]),
-            filterable_attributes: Setting::Set(btreeset! { S("race"), S("age") }),
-            sortable_attributes: Setting::Set(btreeset! { S("age") }),
-            ranking_rules: Setting::NotSet,
-            stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
-            synonyms: Setting::NotSet,
-            distinct_attribute: Setting::NotSet,
-            typo_tolerance: Setting::NotSet,
-            faceting: Setting::Set(FacetingSettings {
-                max_values_per_facet: Setting::Set(111),
-                sort_facet_values_by: Setting::Set(
-                    btreemap! { S("age") => FacetValuesSort::Count },
-                ),
-            }),
-            pagination: Setting::NotSet,
-            _kind: std::marker::PhantomData,
-        };
-        settings.check()
-    }
-
-    pub fn create_test_tasks() -> Vec<(TaskDump, Option<Vec<Document>>)> {
-        vec![
-            (
-                TaskDump {
-                    uid: 0,
-                    index_uid: Some(S("doggo")),
-                    status: Status::Succeeded,
-                    kind: KindDump::DocumentImport {
-                        method: milli::update::IndexDocumentsMethod::UpdateDocuments,
-                        allow_index_creation: true,
-                        primary_key: Some(S("bone")),
-                        documents_count: 12,
-                    },
-                    canceled_by: None,
-                    details: Some(Details::DocumentAdditionOrUpdate {
-                        received_documents: 12,
-                        indexed_documents: Some(10),
-                    }),
-                    error: None,
-                    enqueued_at: datetime!(2022-11-11 0:00 UTC),
-                    started_at: Some(datetime!(2022-11-20 0:00 UTC)),
-                    finished_at: Some(datetime!(2022-11-21 0:00 UTC)),
-                },
-                None,
-            ),
-            (
-                TaskDump {
-                    uid: 1,
-                    index_uid: Some(S("doggo")),
-                    status: Status::Enqueued,
-                    kind: KindDump::DocumentImport {
-                        method: milli::update::IndexDocumentsMethod::UpdateDocuments,
-                        allow_index_creation: true,
-                        primary_key: None,
-                        documents_count: 2,
-                    },
-                    canceled_by: None,
-                    details: Some(Details::DocumentAdditionOrUpdate {
-                        received_documents: 2,
-                        indexed_documents: None,
-                    }),
-                    error: None,
-                    enqueued_at: datetime!(2022-11-11 0:00 UTC),
-                    started_at: None,
-                    finished_at: None,
-                },
-                Some(vec![
-                    json!({ "id": 4, "race": "leonberg" }).as_object().unwrap().clone(),
-                    json!({ "id": 5, "race": "patou" }).as_object().unwrap().clone(),
-                ]),
-            ),
-            (
-                TaskDump {
-                    uid: 5,
-                    index_uid: Some(S("catto")),
-                    status: Status::Enqueued,
-                    kind: KindDump::IndexDeletion,
-                    canceled_by: None,
-                    details: None,
-                    error: None,
-                    enqueued_at: datetime!(2022-11-15 0:00 UTC),
-                    started_at: None,
-                    finished_at: None,
-                },
-                None,
-            ),
-        ]
-    }
-
-    pub fn create_test_api_keys() -> Vec<Key> {
-        vec![
-            Key {
-                description: Some(S("The main key to manage all the doggos")),
-                name: Some(S("doggos_key")),
-                uid: Uuid::from_str("9f8a34da-b6b2-42f0-939b-dbd4c3448655").unwrap(),
-                actions: vec![Action::DocumentsAll],
-                indexes: vec![IndexUidPattern::from_str("doggos").unwrap()],
-                expires_at: Some(datetime!(4130-03-14 12:21 UTC)),
-                created_at: datetime!(1960-11-15 0:00 UTC),
-                updated_at: datetime!(2022-11-10 0:00 UTC),
-            },
-            Key {
-                description: Some(S("The master key for everything and even the doggos")),
-                name: Some(S("master_key")),
-                uid: Uuid::from_str("4622f717-1c00-47bb-a494-39d76a49b591").unwrap(),
-                actions: vec![Action::All],
-                indexes: vec![IndexUidPattern::all()],
-                expires_at: None,
-                created_at: datetime!(0000-01-01 00:01 UTC),
-                updated_at: datetime!(1964-05-04 17:25 UTC),
-            },
-            Key {
-                description: Some(S("The useless key to for nothing nor the doggos")),
-                name: Some(S("useless_key")),
-                uid: Uuid::from_str("fb80b58b-0a34-412f-8ba7-1ce868f8ac5c").unwrap(),
-                actions: vec![],
-                indexes: vec![],
-                expires_at: None,
-                created_at: datetime!(400-02-29 0:00 UTC),
-                updated_at: datetime!(1024-02-29 0:00 UTC),
-            },
-        ]
-    }
-
-    pub fn create_test_dump() -> File {
-        let instance_uid = create_test_instance_uid();
-        let dump = DumpWriter::new(Some(instance_uid)).unwrap();
-
-        // ========== Adding an index
-        let documents = create_test_documents();
-        let settings = create_test_settings();
-
-        let mut index = dump.create_index("doggos", &create_test_index_metadata()).unwrap();
-        for document in &documents {
-            index.push_document(document).unwrap();
-        }
-        index.flush().unwrap();
-        index.settings(&settings).unwrap();
-
-        // ========== pushing the task queue
-        let tasks = create_test_tasks();
-
-        let mut task_queue = dump.create_tasks_queue().unwrap();
-        for (task, update_file) in &tasks {
-            let mut update = task_queue.push_task(task).unwrap();
-            if let Some(update_file) = update_file {
-                for u in update_file {
-                    update.push_document(u).unwrap();
-                }
-            }
-        }
-        task_queue.flush().unwrap();
-
-        // ========== pushing the api keys
-        let api_keys = create_test_api_keys();
-
-        let mut keys = dump.create_keys().unwrap();
-        for key in &api_keys {
-            keys.push_key(key).unwrap();
-        }
-        keys.flush().unwrap();
-
-        // ========== TODO: create features here
-
-        // create the dump
-        let mut file = tempfile::tempfile().unwrap();
-        dump.persist_to(&mut file).unwrap();
-        file.rewind().unwrap();
-
-        file
-    }
-
-    #[test]
-    fn test_creating_and_read_dump() {
-        let mut file = create_test_dump();
-        let mut dump = DumpReader::open(&mut file).unwrap();
-
-        // ==== checking the top level infos
-        assert_eq!(dump.version(), Version::V6);
-        assert!(dump.date().is_some());
-        assert_eq!(dump.instance_uid().unwrap().unwrap(), create_test_instance_uid());
-
-        // ==== checking the index
-        let mut indexes = dump.indexes().unwrap();
-        let mut index = indexes.next().unwrap().unwrap();
-        assert!(indexes.next().is_none()); // there was only one index in the dump
-
-        for (document, expected) in index.documents().unwrap().zip(create_test_documents()) {
-            assert_eq!(document.unwrap(), expected);
-        }
-        assert_eq!(index.settings().unwrap(), create_test_settings());
-        assert_eq!(index.metadata(), &create_test_index_metadata());
-
-        drop(index);
-        drop(indexes);
-
-        // ==== checking the task queue
-        for (task, expected) in dump.tasks().unwrap().zip(create_test_tasks()) {
-            let (task, content_file) = task.unwrap();
-            assert_eq!(task, expected.0);
-
-            if let Some(expected_update) = expected.1 {
-                assert!(
-                    content_file.is_some(),
-                    "A content file was expected for the task {}.",
-                    expected.0.uid
-                );
-                let updates = content_file.unwrap().collect::<Result<Vec<_>, _>>().unwrap();
-                assert_eq!(updates, expected_update);
-            }
-        }
-
-        // ==== checking the keys
-        for (key, expected) in dump.keys().unwrap().zip(create_test_api_keys()) {
-            assert_eq!(key.unwrap(), expected);
-        }
-    }
-}

dump/src/reader/compat/mod.rs

@@ -1,5 +0,0 @@
-pub mod v1_to_v2;
-pub mod v2_to_v3;
-pub mod v3_to_v4;
-pub mod v4_to_v5;
-pub mod v5_to_v6;

dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-3.snap

@@ -1,38 +0,0 @@
----
-source: dump/src/reader/compat/v1_to_v2.rs
-expression: products.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "sortableAttributes": [],
-  "rankingRules": [
-    "typo",
-    "words",
-    "proximity",
-    "attribute",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {
-    "android": [
-      "phone",
-      "smartphone"
-    ],
-    "iphone": [
-      "phone",
-      "smartphone"
-    ],
-    "phone": [
-      "android",
-      "iphone",
-      "smartphone"
-    ]
-  },
-  "distinctAttribute": null
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-6.snap

@@ -1,31 +0,0 @@
----
-source: dump/src/reader/compat/v1_to_v2.rs
-expression: movies.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [
-    "genres",
-    "id"
-  ],
-  "sortableAttributes": [
-    "genres",
-    "id"
-  ],
-  "rankingRules": [
-    "typo",
-    "words",
-    "proximity",
-    "attribute",
-    "exactness",
-    "release_date:asc"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-9.snap

@@ -1,24 +0,0 @@
----
-source: dump/src/reader/compat/v1_to_v2.rs
-expression: spells.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "sortableAttributes": [],
-  "rankingRules": [
-    "typo",
-    "words",
-    "proximity",
-    "attribute",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-11.snap

@@ -1,23 +0,0 @@
----
-source: dump/src/reader/compat/v2_to_v3.rs
-expression: movies2.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "rankingRules": [
-    "words",
-    "typo",
-    "proximity",
-    "attribute",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-14.snap

@@ -1,23 +0,0 @@
----
-source: dump/src/reader/compat/v2_to_v3.rs
-expression: spells.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "rankingRules": [
-    "words",
-    "typo",
-    "proximity",
-    "attribute",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-5.snap

@@ -1,37 +0,0 @@
----
-source: dump/src/reader/compat/v2_to_v3.rs
-expression: products.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "rankingRules": [
-    "words",
-    "typo",
-    "proximity",
-    "attribute",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {
-    "android": [
-      "phone",
-      "smartphone"
-    ],
-    "iphone": [
-      "phone",
-      "smartphone"
-    ],
-    "phone": [
-      "android",
-      "iphone",
-      "smartphone"
-    ]
-  },
-  "distinctAttribute": null
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-8.snap

@@ -1,24 +0,0 @@
----
-source: dump/src/reader/compat/v2_to_v3.rs
-expression: movies.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "rankingRules": [
-    "words",
-    "typo",
-    "proximity",
-    "attribute",
-    "exactness",
-    "release_date:asc"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-12.snap

@@ -1,25 +0,0 @@
----
-source: dump/src/reader/compat/v3_to_v4.rs
-expression: movies2.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "sortableAttributes": [],
-  "rankingRules": [
-    "words",
-    "typo",
-    "proximity",
-    "attribute",
-    "sort",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-15.snap

@@ -1,25 +0,0 @@
----
-source: dump/src/reader/compat/v3_to_v4.rs
-expression: spells.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "sortableAttributes": [],
-  "rankingRules": [
-    "words",
-    "typo",
-    "proximity",
-    "attribute",
-    "sort",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-6.snap

@@ -1,39 +0,0 @@
----
-source: dump/src/reader/compat/v3_to_v4.rs
-expression: products.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "sortableAttributes": [],
-  "rankingRules": [
-    "words",
-    "typo",
-    "proximity",
-    "attribute",
-    "sort",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {
-    "android": [
-      "phone",
-      "smartphone"
-    ],
-    "iphone": [
-      "phone",
-      "smartphone"
-    ],
-    "phone": [
-      "android",
-      "iphone",
-      "smartphone"
-    ]
-  },
-  "distinctAttribute": null
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-9.snap

@@ -1,31 +0,0 @@
----
-source: dump/src/reader/compat/v3_to_v4.rs
-expression: movies.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [
-    "genres",
-    "id"
-  ],
-  "sortableAttributes": [
-    "release_date"
-  ],
-  "rankingRules": [
-    "words",
-    "typo",
-    "proximity",
-    "attribute",
-    "sort",
-    "exactness",
-    "release_date:asc"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-12.snap

@@ -1,56 +0,0 @@
----
-source: dump/src/reader/compat/v4_to_v5.rs
-expression: spells.settings().unwrap()
----
-{
-  "displayedAttributes": "Reset",
-  "searchableAttributes": "Reset",
-  "filterableAttributes": {
-    "Set": []
-  },
-  "sortableAttributes": {
-    "Set": []
-  },
-  "rankingRules": {
-    "Set": [
-      "words",
-      "typo",
-      "proximity",
-      "attribute",
-      "sort",
-      "exactness"
-    ]
-  },
-  "stopWords": {
-    "Set": []
-  },
-  "synonyms": {
-    "Set": {}
-  },
-  "distinctAttribute": "Reset",
-  "typoTolerance": {
-    "Set": {
-      "enabled": {
-        "Set": true
-      },
-      "minWordSizeForTypos": {
-        "Set": {
-          "oneTypo": {
-            "Set": 5
-          },
-          "twoTypos": {
-            "Set": 9
-          }
-        }
-      },
-      "disableOnWords": {
-        "Set": []
-      },
-      "disableOnAttributes": {
-        "Set": []
-      }
-    }
-  },
-  "faceting": "NotSet",
-  "pagination": "NotSet"
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-6.snap

@@ -1,70 +0,0 @@
----
-source: dump/src/reader/compat/v4_to_v5.rs
-expression: products.settings().unwrap()
----
-{
-  "displayedAttributes": "Reset",
-  "searchableAttributes": "Reset",
-  "filterableAttributes": {
-    "Set": []
-  },
-  "sortableAttributes": {
-    "Set": []
-  },
-  "rankingRules": {
-    "Set": [
-      "words",
-      "typo",
-      "proximity",
-      "attribute",
-      "sort",
-      "exactness"
-    ]
-  },
-  "stopWords": {
-    "Set": []
-  },
-  "synonyms": {
-    "Set": {
-      "android": [
-        "phone",
-        "smartphone"
-      ],
-      "iphone": [
-        "phone",
-        "smartphone"
-      ],
-      "phone": [
-        "android",
-        "iphone",
-        "smartphone"
-      ]
-    }
-  },
-  "distinctAttribute": "Reset",
-  "typoTolerance": {
-    "Set": {
-      "enabled": {
-        "Set": true
-      },
-      "minWordSizeForTypos": {
-        "Set": {
-          "oneTypo": {
-            "Set": 5
-          },
-          "twoTypos": {
-            "Set": 9
-          }
-        }
-      },
-      "disableOnWords": {
-        "Set": []
-      },
-      "disableOnAttributes": {
-        "Set": []
-      }
-    }
-  },
-  "faceting": "NotSet",
-  "pagination": "NotSet"
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-9.snap

@@ -1,62 +0,0 @@
----
-source: dump/src/reader/compat/v4_to_v5.rs
-expression: movies.settings().unwrap()
----
-{
-  "displayedAttributes": "Reset",
-  "searchableAttributes": "Reset",
-  "filterableAttributes": {
-    "Set": [
-      "genres",
-      "id"
-    ]
-  },
-  "sortableAttributes": {
-    "Set": [
-      "release_date"
-    ]
-  },
-  "rankingRules": {
-    "Set": [
-      "words",
-      "typo",
-      "proximity",
-      "attribute",
-      "sort",
-      "exactness",
-      "release_date:asc"
-    ]
-  },
-  "stopWords": {
-    "Set": []
-  },
-  "synonyms": {
-    "Set": {}
-  },
-  "distinctAttribute": "Reset",
-  "typoTolerance": {
-    "Set": {
-      "enabled": {
-        "Set": true
-      },
-      "minWordSizeForTypos": {
-        "Set": {
-          "oneTypo": {
-            "Set": 5
-          },
-          "twoTypos": {
-            "Set": 9
-          }
-        }
-      },
-      "disableOnWords": {
-        "Set": []
-      },
-      "disableOnAttributes": {
-        "Set": []
-      }
-    }
-  },
-  "faceting": "NotSet",
-  "pagination": "NotSet"
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-12.snap

@@ -1,40 +0,0 @@
----
-source: dump/src/reader/compat/v5_to_v6.rs
-expression: spells.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "sortableAttributes": [],
-  "rankingRules": [
-    "words",
-    "typo",
-    "proximity",
-    "attribute",
-    "sort",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null,
-  "typoTolerance": {
-    "enabled": true,
-    "minWordSizeForTypos": {
-      "oneTypo": 5,
-      "twoTypos": 9
-    },
-    "disableOnWords": [],
-    "disableOnAttributes": []
-  },
-  "faceting": {
-    "maxValuesPerFacet": 100
-  },
-  "pagination": {
-    "maxTotalHits": 1000
-  }
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-6.snap

@@ -1,54 +0,0 @@
----
-source: dump/src/reader/compat/v5_to_v6.rs
-expression: products.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "sortableAttributes": [],
-  "rankingRules": [
-    "words",
-    "typo",
-    "proximity",
-    "attribute",
-    "sort",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {
-    "android": [
-      "phone",
-      "smartphone"
-    ],
-    "iphone": [
-      "phone",
-      "smartphone"
-    ],
-    "phone": [
-      "android",
-      "iphone",
-      "smartphone"
-    ]
-  },
-  "distinctAttribute": null,
-  "typoTolerance": {
-    "enabled": true,
-    "minWordSizeForTypos": {
-      "oneTypo": 5,
-      "twoTypos": 9
-    },
-    "disableOnWords": [],
-    "disableOnAttributes": []
-  },
-  "faceting": {
-    "maxValuesPerFacet": 100
-  },
-  "pagination": {
-    "maxTotalHits": 1000
-  }
-}

dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-9.snap

@@ -1,46 +0,0 @@
----
-source: dump/src/reader/compat/v5_to_v6.rs
-expression: movies.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [
-    "genres",
-    "id"
-  ],
-  "sortableAttributes": [
-    "release_date"
-  ],
-  "rankingRules": [
-    "words",
-    "typo",
-    "proximity",
-    "attribute",
-    "sort",
-    "exactness",
-    "release_date:asc"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null,
-  "typoTolerance": {
-    "enabled": true,
-    "minWordSizeForTypos": {
-      "oneTypo": 5,
-      "twoTypos": 9
-    },
-    "disableOnWords": [],
-    "disableOnAttributes": []
-  },
-  "faceting": {
-    "maxValuesPerFacet": 100
-  },
-  "pagination": {
-    "maxTotalHits": 1000
-  }
-}

dump/src/reader/compat/v1_to_v2.rs

@@ -1,410 +0,0 @@
-use std::str::FromStr;
-
-use super::v2_to_v3::CompatV2ToV3;
-use crate::reader::{v1, v2, Document};
-use crate::Result;
-
-pub struct CompatV1ToV2 {
-    pub from: v1::V1Reader,
-}
-
-impl CompatV1ToV2 {
-    pub fn new(v1: v1::V1Reader) -> Self {
-        Self { from: v1 }
-    }
-
-    pub fn to_v3(self) -> CompatV2ToV3 {
-        CompatV2ToV3::Compat(self)
-    }
-
-    pub fn version(&self) -> crate::Version {
-        self.from.version()
-    }
-
-    pub fn date(&self) -> Option<time::OffsetDateTime> {
-        self.from.date()
-    }
-
-    pub fn index_uuid(&self) -> Vec<v2::meta::IndexUuid> {
-        self.from
-            .index_uuid()
-            .into_iter()
-            .enumerate()
-            // we use the index of the index 😬 as UUID for the index, so that we can link the v2::Task to their index
-            .map(|(index, index_uuid)| v2::meta::IndexUuid {
-                uid: index_uuid.uid,
-                uuid: uuid::Uuid::from_u128(index as u128),
-            })
-            .collect()
-    }
-
-    pub fn indexes(&self) -> Result<impl Iterator<Item = Result<CompatIndexV1ToV2>> + '_> {
-        Ok(self.from.indexes()?.map(|index_reader| Ok(CompatIndexV1ToV2 { from: index_reader? })))
-    }
-
-    pub fn tasks(
-        &mut self,
-    ) -> Box<dyn Iterator<Item = Result<(v2::Task, Option<v2::UpdateFile>)>> + '_> {
-        // Convert an error here to an iterator yielding the error
-        let indexes = match self.from.indexes() {
-            Ok(indexes) => indexes,
-            Err(err) => return Box::new(std::iter::once(Err(err))),
-        };
-        let it = indexes.enumerate().flat_map(
-            move |(index, index_reader)| -> Box<dyn Iterator<Item = _>> {
-                let index_reader = match index_reader {
-                    Ok(index_reader) => index_reader,
-                    Err(err) => return Box::new(std::iter::once(Err(err))),
-                };
-                Box::new(
-                    index_reader
-                        .tasks()
-                        // Filter out the UpdateStatus::Customs variant that is not supported in v2
-                        // and enqueued tasks, that don't contain the necessary update file in v1
-                        .filter_map(move |task| -> Option<_> {
-                            let task = match task {
-                                Ok(task) => task,
-                                Err(err) => return Some(Err(err)),
-                            };
-                            Some(Ok((
-                                v2::Task {
-                                    uuid: uuid::Uuid::from_u128(index as u128),
-                                    update: Option::from(task)?,
-                                },
-                                None,
-                            )))
-                        }),
-                )
-            },
-        );
-        Box::new(it)
-    }
-}
-
-pub struct CompatIndexV1ToV2 {
-    pub from: v1::V1IndexReader,
-}
-
-impl CompatIndexV1ToV2 {
-    pub fn metadata(&self) -> &crate::IndexMetadata {
-        self.from.metadata()
-    }
-
-    pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Document>> + '_>> {
-        self.from.documents().map(|it| Box::new(it) as Box<dyn Iterator<Item = _>>)
-    }
-
-    pub fn settings(&mut self) -> Result<v2::settings::Settings<v2::settings::Checked>> {
-        Ok(v2::settings::Settings::<v2::settings::Unchecked>::from(self.from.settings()?).check())
-    }
-}
-
-impl From<v1::settings::Settings> for v2::Settings<v2::Unchecked> {
-    fn from(source: v1::settings::Settings) -> Self {
-        Self {
-            displayed_attributes: option_to_setting(source.displayed_attributes)
-                .map(|displayed| displayed.into_iter().collect()),
-            searchable_attributes: option_to_setting(source.searchable_attributes),
-            filterable_attributes: option_to_setting(source.attributes_for_faceting.clone())
-                .map(|filterable| filterable.into_iter().collect()),
-            sortable_attributes: option_to_setting(source.attributes_for_faceting)
-                .map(|sortable| sortable.into_iter().collect()),
-            ranking_rules: option_to_setting(source.ranking_rules).map(|ranking_rules| {
-                ranking_rules
-                    .into_iter()
-                    .filter_map(|ranking_rule| {
-                        match v1::settings::RankingRule::from_str(&ranking_rule) {
-                            Ok(ranking_rule) => {
-                                let criterion: Option<v2::settings::Criterion> =
-                                    ranking_rule.into();
-                                criterion.as_ref().map(ToString::to_string)
-                            }
-                            Err(()) => {
-                                log::warn!(
-                                    "Could not import the following ranking rule: `{}`.",
-                                    ranking_rule
-                                );
-                                None
-                            }
-                        }
-                    })
-                    .collect()
-            }),
-            stop_words: option_to_setting(source.stop_words),
-            synonyms: option_to_setting(source.synonyms),
-            distinct_attribute: option_to_setting(source.distinct_attribute),
-            _kind: std::marker::PhantomData,
-        }
-    }
-}
-
-fn option_to_setting<T>(opt: Option<Option<T>>) -> v2::Setting<T> {
-    match opt {
-        Some(Some(t)) => v2::Setting::Set(t),
-        None => v2::Setting::NotSet,
-        Some(None) => v2::Setting::Reset,
-    }
-}
-
-impl From<v1::update::UpdateStatus> for Option<v2::updates::UpdateStatus> {
-    fn from(source: v1::update::UpdateStatus) -> Self {
-        use v1::update::UpdateStatus as UpdateStatusV1;
-        use v2::updates::UpdateStatus as UpdateStatusV2;
-        Some(match source {
-            UpdateStatusV1::Enqueued { content } => {
-                log::warn!(
-                    "Cannot import task {} (importing enqueued tasks from v1 dumps is unsupported)",
-                    content.update_id
-                );
-                log::warn!("Task will be skipped in the queue of imported tasks.");
-
-                return None;
-            }
-            UpdateStatusV1::Failed { content } => UpdateStatusV2::Failed(v2::updates::Failed {
-                from: v2::updates::Processing {
-                    from: v2::updates::Enqueued {
-                        update_id: content.update_id,
-                        meta: Option::from(content.update_type)?,
-                        enqueued_at: content.enqueued_at,
-                        content: None,
-                    },
-                    started_processing_at: content.processed_at
-                        - std::time::Duration::from_secs_f64(content.duration),
-                },
-                error: v2::ResponseError {
-                    // error code is ignored by serialization, and so always default in deserialized v2 dumps
-                    // that's a good thing, because we don't have them in v1 dump 😅
-                    code: http::StatusCode::default(),
-                    message: content.error.unwrap_or_default(),
-                    // error codes are unchanged between v1 and v2
-                    error_code: content.error_code.unwrap_or_default(),
-                    // error types are unchanged between v1 and v2
-                    error_type: content.error_type.unwrap_or_default(),
-                    // error links are unchanged between v1 and v2
-                    error_link: content.error_link.unwrap_or_default(),
-                },
-                failed_at: content.processed_at,
-            }),
-            UpdateStatusV1::Processed { content } => {
-                UpdateStatusV2::Processed(v2::updates::Processed {
-                    success: match &content.update_type {
-                        v1::update::UpdateType::ClearAll => {
-                            v2::updates::UpdateResult::DocumentDeletion { deleted: u64::MAX }
-                        }
-                        v1::update::UpdateType::Customs => v2::updates::UpdateResult::Other,
-                        v1::update::UpdateType::DocumentsAddition { number } => {
-                            v2::updates::UpdateResult::DocumentsAddition(
-                                v2::updates::DocumentAdditionResult { nb_documents: *number },
-                            )
-                        }
-                        v1::update::UpdateType::DocumentsPartial { number } => {
-                            v2::updates::UpdateResult::DocumentsAddition(
-                                v2::updates::DocumentAdditionResult { nb_documents: *number },
-                            )
-                        }
-                        v1::update::UpdateType::DocumentsDeletion { number } => {
-                            v2::updates::UpdateResult::DocumentDeletion { deleted: *number as u64 }
-                        }
-                        v1::update::UpdateType::Settings { .. } => v2::updates::UpdateResult::Other,
-                    },
-                    processed_at: content.processed_at,
-                    from: v2::updates::Processing {
-                        from: v2::updates::Enqueued {
-                            update_id: content.update_id,
-                            meta: Option::from(content.update_type)?,
-                            enqueued_at: content.enqueued_at,
-                            content: None,
-                        },
-                        started_processing_at: content.processed_at
-                            - std::time::Duration::from_secs_f64(content.duration),
-                    },
-                })
-            }
-        })
-    }
-}
-
-impl From<v1::update::UpdateType> for Option<v2::updates::UpdateMeta> {
-    fn from(source: v1::update::UpdateType) -> Self {
-        Some(match source {
-            v1::update::UpdateType::ClearAll => v2::updates::UpdateMeta::ClearDocuments,
-            v1::update::UpdateType::Customs => {
-                log::warn!("Ignoring task with type 'Customs' that is no longer supported");
-                return None;
-            }
-            v1::update::UpdateType::DocumentsAddition { .. } => {
-                v2::updates::UpdateMeta::DocumentsAddition {
-                    method: v2::updates::IndexDocumentsMethod::ReplaceDocuments,
-                    format: v2::updates::UpdateFormat::Json,
-                    primary_key: None,
-                }
-            }
-            v1::update::UpdateType::DocumentsPartial { .. } => {
-                v2::updates::UpdateMeta::DocumentsAddition {
-                    method: v2::updates::IndexDocumentsMethod::UpdateDocuments,
-                    format: v2::updates::UpdateFormat::Json,
-                    primary_key: None,
-                }
-            }
-            v1::update::UpdateType::DocumentsDeletion { .. } => {
-                v2::updates::UpdateMeta::DeleteDocuments { ids: vec![] }
-            }
-            v1::update::UpdateType::Settings { settings } => {
-                v2::updates::UpdateMeta::Settings((*settings).into())
-            }
-        })
-    }
-}
-
-impl From<v1::settings::SettingsUpdate> for v2::Settings<v2::Unchecked> {
-    fn from(source: v1::settings::SettingsUpdate) -> Self {
-        let ranking_rules = v2::Setting::from(source.ranking_rules);
-
-        // go from the concrete types of v1 (RankingRule) to the concrete type of v2 (Criterion),
-        // and then back to string as this is what the settings manipulate
-        let ranking_rules = ranking_rules.map(|ranking_rules| {
-            ranking_rules
-                .into_iter()
-                // filter out the WordsPosition ranking rule that exists in v1 but not v2
-                .filter_map(Option::<v2::settings::Criterion>::from)
-                .map(|criterion| criterion.to_string())
-                .collect()
-        });
-
-        Self {
-            displayed_attributes: v2::Setting::from(source.displayed_attributes)
-                .map(|displayed_attributes| displayed_attributes.into_iter().collect()),
-            searchable_attributes: source.searchable_attributes.into(),
-            filterable_attributes: v2::Setting::from(source.attributes_for_faceting.clone())
-                .map(|attributes_for_faceting| attributes_for_faceting.into_iter().collect()),
-            sortable_attributes: v2::Setting::from(source.attributes_for_faceting)
-                .map(|attributes_for_faceting| attributes_for_faceting.into_iter().collect()),
-            ranking_rules,
-            stop_words: source.stop_words.into(),
-            synonyms: source.synonyms.into(),
-            distinct_attribute: source.distinct_attribute.into(),
-            _kind: std::marker::PhantomData,
-        }
-    }
-}
-
-impl From<v1::settings::RankingRule> for Option<v2::settings::Criterion> {
-    fn from(source: v1::settings::RankingRule) -> Self {
-        match source {
-            v1::settings::RankingRule::Typo => Some(v2::settings::Criterion::Typo),
-            v1::settings::RankingRule::Words => Some(v2::settings::Criterion::Words),
-            v1::settings::RankingRule::Proximity => Some(v2::settings::Criterion::Proximity),
-            v1::settings::RankingRule::Attribute => Some(v2::settings::Criterion::Attribute),
-            v1::settings::RankingRule::WordsPosition => {
-                log::warn!("Removing the 'WordsPosition' ranking rule that is no longer supported, please check the resulting ranking rules of your indexes");
-                None
-            }
-            v1::settings::RankingRule::Exactness => Some(v2::settings::Criterion::Exactness),
-            v1::settings::RankingRule::Asc(field_name) => {
-                Some(v2::settings::Criterion::Asc(field_name))
-            }
-            v1::settings::RankingRule::Desc(field_name) => {
-                Some(v2::settings::Criterion::Desc(field_name))
-            }
-        }
-    }
-}
-
-impl<T> From<v1::settings::UpdateState<T>> for v2::Setting<T> {
-    fn from(source: v1::settings::UpdateState<T>) -> Self {
-        match source {
-            v1::settings::UpdateState::Update(new_value) => v2::Setting::Set(new_value),
-            v1::settings::UpdateState::Clear => v2::Setting::Reset,
-            v1::settings::UpdateState::Nothing => v2::Setting::NotSet,
-        }
-    }
-}
-
-#[cfg(test)]
-pub(crate) mod test {
-    use std::fs::File;
-    use std::io::BufReader;
-
-    use flate2::bufread::GzDecoder;
-    use meili_snap::insta;
-    use tempfile::TempDir;
-
-    use super::*;
-
-    #[test]
-    fn compat_v1_v2() {
-        let dump = File::open("tests/assets/v1.dump").unwrap();
-        let dir = TempDir::new().unwrap();
-        let mut dump = BufReader::new(dump);
-        let gz = GzDecoder::new(&mut dump);
-        let mut archive = tar::Archive::new(gz);
-        archive.unpack(dir.path()).unwrap();
-
-        let mut dump = v1::V1Reader::open(dir).unwrap().to_v2();
-
-        // top level infos
-        assert_eq!(dump.date(), None);
-
-        // tasks
-        let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"2298010973ee98cf4670787314176a3a");
-        assert_eq!(update_files.len(), 9);
-        assert!(update_files[..].iter().all(|u| u.is_none())); // no update file in dumps v1
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut products = indexes.pop().unwrap();
-        let mut movies = indexes.pop().unwrap();
-        let mut spells = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
-        {
-          "uid": "products",
-          "primaryKey": "sku",
-          "createdAt": "2022-10-02T13:23:39.976870431Z",
-          "updatedAt": "2022-10-02T13:27:54.353262482Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(products.settings().unwrap());
-        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
-
-        // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
-        {
-          "uid": "movies",
-          "primaryKey": "id",
-          "createdAt": "2022-10-02T13:15:29.477512777Z",
-          "updatedAt": "2022-10-02T13:21:12.671204856Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies.settings().unwrap());
-        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5");
-
-        // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
-        {
-          "uid": "dnd_spells",
-          "primaryKey": "index",
-          "createdAt": "2022-10-02T13:38:26.358882984Z",
-          "updatedAt": "2022-10-02T13:38:26.385609433Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(spells.settings().unwrap());
-        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"aa24c0cfc733d66c396237ad44263bed");
-    }
-}

dump/src/reader/compat/v2_to_v3.rs

@@ -1,513 +0,0 @@
-use std::convert::TryInto;
-use std::str::FromStr;
-
-use time::OffsetDateTime;
-use uuid::Uuid;
-
-use super::v1_to_v2::{CompatIndexV1ToV2, CompatV1ToV2};
-use super::v3_to_v4::CompatV3ToV4;
-use crate::reader::{v2, v3, Document};
-use crate::Result;
-
-pub enum CompatV2ToV3 {
-    V2(v2::V2Reader),
-    Compat(CompatV1ToV2),
-}
-
-impl CompatV2ToV3 {
-    pub fn new(v2: v2::V2Reader) -> CompatV2ToV3 {
-        CompatV2ToV3::V2(v2)
-    }
-
-    pub fn index_uuid(&self) -> Vec<v3::meta::IndexUuid> {
-        let v2_uuids = match self {
-            CompatV2ToV3::V2(from) => from.index_uuid(),
-            CompatV2ToV3::Compat(compat) => compat.index_uuid(),
-        };
-        v2_uuids
-            .into_iter()
-            .map(|index| v3::meta::IndexUuid { uid: index.uid, uuid: index.uuid })
-            .collect()
-    }
-
-    pub fn to_v4(self) -> CompatV3ToV4 {
-        CompatV3ToV4::Compat(self)
-    }
-
-    pub fn version(&self) -> crate::Version {
-        match self {
-            CompatV2ToV3::V2(from) => from.version(),
-            CompatV2ToV3::Compat(compat) => compat.version(),
-        }
-    }
-
-    pub fn date(&self) -> Option<time::OffsetDateTime> {
-        match self {
-            CompatV2ToV3::V2(from) => from.date(),
-            CompatV2ToV3::Compat(compat) => compat.date(),
-        }
-    }
-
-    pub fn instance_uid(&self) -> Result<Option<uuid::Uuid>> {
-        Ok(None)
-    }
-
-    pub fn indexes(&self) -> Result<impl Iterator<Item = Result<CompatIndexV2ToV3>> + '_> {
-        Ok(match self {
-            CompatV2ToV3::V2(from) => Box::new(from.indexes()?.map(|index_reader| -> Result<_> {
-                let compat = CompatIndexV2ToV3::new(index_reader?);
-                Ok(compat)
-            }))
-                as Box<dyn Iterator<Item = Result<CompatIndexV2ToV3>> + '_>,
-            CompatV2ToV3::Compat(compat) => Box::new(compat.indexes()?.map(|index_reader| {
-                let compat = CompatIndexV2ToV3::Compat(Box::new(index_reader?));
-                Ok(compat)
-            }))
-                as Box<dyn Iterator<Item = Result<CompatIndexV2ToV3>> + '_>,
-        })
-    }
-
-    pub fn tasks(
-        &mut self,
-    ) -> Box<
-        dyn Iterator<Item = Result<(v3::Task, Option<Box<dyn Iterator<Item = Result<Document>>>>)>>
-            + '_,
-    > {
-        let tasks = match self {
-            CompatV2ToV3::V2(from) => from.tasks(),
-            CompatV2ToV3::Compat(compat) => compat.tasks(),
-        };
-
-        Box::new(
-            tasks
-                .map(move |task| {
-                    task.map(|(task, content_file)| {
-                        let task = v3::Task { uuid: task.uuid, update: task.update.into() };
-
-                        Some((
-                            task,
-                            content_file.map(|content_file| {
-                                Box::new(content_file) as Box<dyn Iterator<Item = Result<Document>>>
-                            }),
-                        ))
-                    })
-                })
-                .filter_map(|res| res.transpose()),
-        )
-    }
-}
-
-pub enum CompatIndexV2ToV3 {
-    V2(v2::V2IndexReader),
-    Compat(Box<CompatIndexV1ToV2>),
-}
-
-impl CompatIndexV2ToV3 {
-    pub fn new(v2: v2::V2IndexReader) -> CompatIndexV2ToV3 {
-        CompatIndexV2ToV3::V2(v2)
-    }
-
-    pub fn metadata(&self) -> &crate::IndexMetadata {
-        match self {
-            CompatIndexV2ToV3::V2(from) => from.metadata(),
-            CompatIndexV2ToV3::Compat(compat) => compat.metadata(),
-        }
-    }
-
-    pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Document>> + '_>> {
-        match self {
-            CompatIndexV2ToV3::V2(from) => from
-                .documents()
-                .map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
-            CompatIndexV2ToV3::Compat(compat) => compat.documents(),
-        }
-    }
-
-    pub fn settings(&mut self) -> Result<v3::Settings<v3::Checked>> {
-        let settings = match self {
-            CompatIndexV2ToV3::V2(from) => from.settings()?,
-            CompatIndexV2ToV3::Compat(compat) => compat.settings()?,
-        };
-        Ok(v3::Settings::<v3::Unchecked>::from(settings).check())
-    }
-}
-
-impl From<v2::updates::UpdateStatus> for v3::updates::UpdateStatus {
-    fn from(update: v2::updates::UpdateStatus) -> Self {
-        match update {
-            v2::updates::UpdateStatus::Processing(processing) => {
-                match (processing.from.meta.clone(), processing.from.content).try_into() {
-                    Ok(meta) => v3::updates::UpdateStatus::Processing(v3::updates::Processing {
-                        from: v3::updates::Enqueued {
-                            update_id: processing.from.update_id,
-                            meta,
-                            enqueued_at: processing.from.enqueued_at,
-                        },
-                        started_processing_at: processing.started_processing_at,
-                    }),
-                    Err(e) => {
-                        log::warn!("Error with task {}: {}", processing.from.update_id, e);
-                        log::warn!("Task will be marked as `Failed`.");
-                        v3::updates::UpdateStatus::Failed(v3::updates::Failed {
-                            from: v3::updates::Processing {
-                                from: v3::updates::Enqueued {
-                                    update_id: processing.from.update_id,
-                                    meta: update_from_unchecked_update_meta(processing.from.meta),
-                                    enqueued_at: processing.from.enqueued_at,
-                                },
-                                started_processing_at: processing.started_processing_at,
-                            },
-                            msg: e.to_string(),
-                            code: v3::Code::MalformedDump,
-                            failed_at: OffsetDateTime::now_utc(),
-                        })
-                    }
-                }
-            }
-            v2::updates::UpdateStatus::Enqueued(enqueued) => {
-                match (enqueued.meta.clone(), enqueued.content).try_into() {
-                    Ok(meta) => v3::updates::UpdateStatus::Enqueued(v3::updates::Enqueued {
-                        update_id: enqueued.update_id,
-                        meta,
-                        enqueued_at: enqueued.enqueued_at,
-                    }),
-                    Err(e) => {
-                        log::warn!("Error with task {}: {}", enqueued.update_id, e);
-                        log::warn!("Task will be marked as `Failed`.");
-                        v3::updates::UpdateStatus::Failed(v3::updates::Failed {
-                            from: v3::updates::Processing {
-                                from: v3::updates::Enqueued {
-                                    update_id: enqueued.update_id,
-                                    meta: update_from_unchecked_update_meta(enqueued.meta),
-                                    enqueued_at: enqueued.enqueued_at,
-                                },
-                                started_processing_at: OffsetDateTime::now_utc(),
-                            },
-                            msg: e.to_string(),
-                            code: v3::Code::MalformedDump,
-                            failed_at: OffsetDateTime::now_utc(),
-                        })
-                    }
-                }
-            }
-            v2::updates::UpdateStatus::Processed(processed) => {
-                v3::updates::UpdateStatus::Processed(v3::updates::Processed {
-                    success: processed.success.into(),
-                    processed_at: processed.processed_at,
-                    from: v3::updates::Processing {
-                        from: v3::updates::Enqueued {
-                            update_id: processed.from.from.update_id,
-                            // since we're never going to read the content_file again it's ok to generate a fake one.
-                            meta: update_from_unchecked_update_meta(processed.from.from.meta),
-                            enqueued_at: processed.from.from.enqueued_at,
-                        },
-                        started_processing_at: processed.from.started_processing_at,
-                    },
-                })
-            }
-            v2::updates::UpdateStatus::Aborted(aborted) => {
-                v3::updates::UpdateStatus::Aborted(v3::updates::Aborted {
-                    from: v3::updates::Enqueued {
-                        update_id: aborted.from.update_id,
-                        // since we're never going to read the content_file again it's ok to generate a fake one.
-                        meta: update_from_unchecked_update_meta(aborted.from.meta),
-                        enqueued_at: aborted.from.enqueued_at,
-                    },
-                    aborted_at: aborted.aborted_at,
-                })
-            }
-            v2::updates::UpdateStatus::Failed(failed) => {
-                v3::updates::UpdateStatus::Failed(v3::updates::Failed {
-                    from: v3::updates::Processing {
-                        from: v3::updates::Enqueued {
-                            update_id: failed.from.from.update_id,
-                            // since we're never going to read the content_file again it's ok to generate a fake one.
-                            meta: update_from_unchecked_update_meta(failed.from.from.meta),
-                            enqueued_at: failed.from.from.enqueued_at,
-                        },
-                        started_processing_at: failed.from.started_processing_at,
-                    },
-                    msg: failed.error.message,
-                    code: failed.error.error_code.into(),
-                    failed_at: failed.failed_at,
-                })
-            }
-        }
-    }
-}
-
-impl TryFrom<(v2::updates::UpdateMeta, Option<Uuid>)> for v3::updates::Update {
-    type Error = crate::Error;
-
-    fn try_from((update, uuid): (v2::updates::UpdateMeta, Option<Uuid>)) -> Result<Self> {
-        Ok(match update {
-            v2::updates::UpdateMeta::DocumentsAddition { method, format: _, primary_key }
-                if uuid.is_some() =>
-            {
-                v3::updates::Update::DocumentAddition {
-                    primary_key,
-                    method: match method {
-                        v2::updates::IndexDocumentsMethod::ReplaceDocuments => {
-                            v3::updates::IndexDocumentsMethod::ReplaceDocuments
-                        }
-                        v2::updates::IndexDocumentsMethod::UpdateDocuments => {
-                            v3::updates::IndexDocumentsMethod::UpdateDocuments
-                        }
-                    },
-                    content_uuid: uuid.unwrap(),
-                }
-            }
-            v2::updates::UpdateMeta::DocumentsAddition { .. } => {
-                return Err(crate::Error::MalformedTask)
-            }
-            v2::updates::UpdateMeta::ClearDocuments => v3::updates::Update::ClearDocuments,
-            v2::updates::UpdateMeta::DeleteDocuments { ids } => {
-                v3::updates::Update::DeleteDocuments(ids)
-            }
-            v2::updates::UpdateMeta::Settings(settings) => {
-                v3::updates::Update::Settings(settings.into())
-            }
-        })
-    }
-}
-
-pub fn update_from_unchecked_update_meta(update: v2::updates::UpdateMeta) -> v3::updates::Update {
-    match update {
-        v2::updates::UpdateMeta::DocumentsAddition { method, format: _, primary_key } => {
-            v3::updates::Update::DocumentAddition {
-                primary_key,
-                method: match method {
-                    v2::updates::IndexDocumentsMethod::ReplaceDocuments => {
-                        v3::updates::IndexDocumentsMethod::ReplaceDocuments
-                    }
-                    v2::updates::IndexDocumentsMethod::UpdateDocuments => {
-                        v3::updates::IndexDocumentsMethod::UpdateDocuments
-                    }
-                },
-                // we use this special uuid so we can recognize it if one day there is a bug related to this field.
-                content_uuid: Uuid::from_str("00112233-4455-6677-8899-aabbccddeeff").unwrap(),
-            }
-        }
-        v2::updates::UpdateMeta::ClearDocuments => v3::updates::Update::ClearDocuments,
-        v2::updates::UpdateMeta::DeleteDocuments { ids } => {
-            v3::updates::Update::DeleteDocuments(ids)
-        }
-        v2::updates::UpdateMeta::Settings(settings) => {
-            v3::updates::Update::Settings(settings.into())
-        }
-    }
-}
-
-impl From<v2::updates::UpdateResult> for v3::updates::UpdateResult {
-    fn from(result: v2::updates::UpdateResult) -> Self {
-        match result {
-            v2::updates::UpdateResult::DocumentsAddition(addition) => {
-                v3::updates::UpdateResult::DocumentsAddition(v3::updates::DocumentAdditionResult {
-                    nb_documents: addition.nb_documents,
-                })
-            }
-            v2::updates::UpdateResult::DocumentDeletion { deleted } => {
-                v3::updates::UpdateResult::DocumentDeletion { deleted }
-            }
-            v2::updates::UpdateResult::Other => v3::updates::UpdateResult::Other,
-        }
-    }
-}
-
-impl From<String> for v3::Code {
-    fn from(code: String) -> Self {
-        match code.as_ref() {
-            "create_index" => v3::Code::CreateIndex,
-            "index_already_exists" => v3::Code::IndexAlreadyExists,
-            "index_not_found" => v3::Code::IndexNotFound,
-            "invalid_index_uid" => v3::Code::InvalidIndexUid,
-            "invalid_state" => v3::Code::InvalidState,
-            "missing_primary_key" => v3::Code::MissingPrimaryKey,
-            "primary_key_already_present" => v3::Code::PrimaryKeyAlreadyPresent,
-            "max_fields_limit_exceeded" => v3::Code::MaxFieldsLimitExceeded,
-            "missing_document_id" => v3::Code::MissingDocumentId,
-            "invalid_document_id" => v3::Code::InvalidDocumentId,
-            "filter" => v3::Code::Filter,
-            "sort" => v3::Code::Sort,
-            "bad_parameter" => v3::Code::BadParameter,
-            "bad_request" => v3::Code::BadRequest,
-            "database_size_limit_reached" => v3::Code::DatabaseSizeLimitReached,
-            "document_not_found" => v3::Code::DocumentNotFound,
-            "internal" => v3::Code::Internal,
-            "invalid_geo_field" => v3::Code::InvalidGeoField,
-            "invalid_ranking_rule" => v3::Code::InvalidRankingRule,
-            "invalid_store" => v3::Code::InvalidStore,
-            "invalid_token" => v3::Code::InvalidToken,
-            "missing_authorization_header" => v3::Code::MissingAuthorizationHeader,
-            "no_space_left_on_device" => v3::Code::NoSpaceLeftOnDevice,
-            "dump_not_found" => v3::Code::DumpNotFound,
-            "task_not_found" => v3::Code::TaskNotFound,
-            "payload_too_large" => v3::Code::PayloadTooLarge,
-            "retrieve_document" => v3::Code::RetrieveDocument,
-            "search_documents" => v3::Code::SearchDocuments,
-            "unsupported_media_type" => v3::Code::UnsupportedMediaType,
-            "dump_already_in_progress" => v3::Code::DumpAlreadyInProgress,
-            "dump_process_failed" => v3::Code::DumpProcessFailed,
-            "invalid_content_type" => v3::Code::InvalidContentType,
-            "missing_content_type" => v3::Code::MissingContentType,
-            "malformed_payload" => v3::Code::MalformedPayload,
-            "missing_payload" => v3::Code::MissingPayload,
-            other => {
-                log::warn!("Unknown error code {}", other);
-                v3::Code::UnretrievableErrorCode
-            }
-        }
-    }
-}
-
-impl<A> From<v2::Setting<A>> for v3::Setting<A> {
-    fn from(setting: v2::Setting<A>) -> Self {
-        match setting {
-            v2::settings::Setting::Set(a) => v3::settings::Setting::Set(a),
-            v2::settings::Setting::Reset => v3::settings::Setting::Reset,
-            v2::settings::Setting::NotSet => v3::settings::Setting::NotSet,
-        }
-    }
-}
-
-impl<T> From<v2::Settings<T>> for v3::Settings<v3::Unchecked> {
-    fn from(settings: v2::Settings<T>) -> Self {
-        v3::Settings {
-            displayed_attributes: settings.displayed_attributes.into(),
-            searchable_attributes: settings.searchable_attributes.into(),
-            filterable_attributes: settings.filterable_attributes.into(),
-            sortable_attributes: settings.sortable_attributes.into(),
-            ranking_rules: v3::Setting::from(settings.ranking_rules).map(|criteria| {
-                criteria.into_iter().map(|criterion| patch_ranking_rules(&criterion)).collect()
-            }),
-            stop_words: settings.stop_words.into(),
-            synonyms: settings.synonyms.into(),
-            distinct_attribute: settings.distinct_attribute.into(),
-            _kind: std::marker::PhantomData,
-        }
-    }
-}
-
-fn patch_ranking_rules(ranking_rule: &str) -> String {
-    match v2::settings::Criterion::from_str(ranking_rule) {
-        Ok(v2::settings::Criterion::Words) => String::from("words"),
-        Ok(v2::settings::Criterion::Typo) => String::from("typo"),
-        Ok(v2::settings::Criterion::Proximity) => String::from("proximity"),
-        Ok(v2::settings::Criterion::Attribute) => String::from("attribute"),
-        Ok(v2::settings::Criterion::Sort) => String::from("sort"),
-        Ok(v2::settings::Criterion::Exactness) => String::from("exactness"),
-        Ok(v2::settings::Criterion::Asc(name)) => format!("{name}:asc"),
-        Ok(v2::settings::Criterion::Desc(name)) => format!("{name}:desc"),
-        // we want to forward the error to the current version of meilisearch
-        Err(_) => ranking_rule.to_string(),
-    }
-}
-
-#[cfg(test)]
-pub(crate) mod test {
-    use std::fs::File;
-    use std::io::BufReader;
-
-    use flate2::bufread::GzDecoder;
-    use meili_snap::insta;
-    use tempfile::TempDir;
-
-    use super::*;
-
-    #[test]
-    fn compat_v2_v3() {
-        let dump = File::open("tests/assets/v2.dump").unwrap();
-        let dir = TempDir::new().unwrap();
-        let mut dump = BufReader::new(dump);
-        let gz = GzDecoder::new(&mut dump);
-        let mut archive = tar::Archive::new(gz);
-        archive.unpack(dir.path()).unwrap();
-
-        let mut dump = v2::V2Reader::open(dir).unwrap().to_v3();
-
-        // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
-
-        // tasks
-        let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, mut update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"9507711db47c7171c79bc6d57d0bed79");
-        assert_eq!(update_files.len(), 9);
-        assert!(update_files[0].is_some()); // the enqueued document addition
-        assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
-
-        let update_file = update_files.remove(0).unwrap().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(update_file), @"7b8889539b669c7b9ddba448bafa385d");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut products = indexes.pop().unwrap();
-        let mut movies2 = indexes.pop().unwrap();
-        let mut movies = indexes.pop().unwrap();
-        let mut spells = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "products",
-          "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(products.settings().unwrap());
-        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
-
-        // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "movies",
-          "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies.settings().unwrap());
-        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 110);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
-
-        // movies2
-        insta::assert_json_snapshot!(movies2.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "movies_2",
-          "primaryKey": null,
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies2.settings().unwrap());
-        let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 0);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
-
-        // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "dnd_spells",
-          "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(spells.settings().unwrap());
-        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
-    }
-}

dump/src/reader/compat/v3_to_v4.rs

@@ -1,449 +0,0 @@
-use super::v2_to_v3::{CompatIndexV2ToV3, CompatV2ToV3};
-use super::v4_to_v5::CompatV4ToV5;
-use crate::reader::{v3, v4, UpdateFile};
-use crate::Result;
-
-pub enum CompatV3ToV4 {
-    V3(v3::V3Reader),
-    Compat(CompatV2ToV3),
-}
-
-impl CompatV3ToV4 {
-    pub fn new(v3: v3::V3Reader) -> CompatV3ToV4 {
-        CompatV3ToV4::V3(v3)
-    }
-
-    pub fn to_v5(self) -> CompatV4ToV5 {
-        CompatV4ToV5::Compat(self)
-    }
-
-    pub fn version(&self) -> crate::Version {
-        match self {
-            CompatV3ToV4::V3(v3) => v3.version(),
-            CompatV3ToV4::Compat(compat) => compat.version(),
-        }
-    }
-
-    pub fn date(&self) -> Option<time::OffsetDateTime> {
-        match self {
-            CompatV3ToV4::V3(v3) => v3.date(),
-            CompatV3ToV4::Compat(compat) => compat.date(),
-        }
-    }
-
-    pub fn instance_uid(&self) -> Result<Option<uuid::Uuid>> {
-        Ok(None)
-    }
-
-    pub fn indexes(&self) -> Result<impl Iterator<Item = Result<CompatIndexV3ToV4>> + '_> {
-        Ok(match self {
-            CompatV3ToV4::V3(v3) => {
-                Box::new(v3.indexes()?.map(|index| index.map(CompatIndexV3ToV4::from)))
-                    as Box<dyn Iterator<Item = Result<CompatIndexV3ToV4>> + '_>
-            }
-
-            CompatV3ToV4::Compat(compat) => {
-                Box::new(compat.indexes()?.map(|index| index.map(CompatIndexV3ToV4::from)))
-                    as Box<dyn Iterator<Item = Result<CompatIndexV3ToV4>> + '_>
-            }
-        })
-    }
-
-    pub fn tasks(
-        &mut self,
-    ) -> Box<dyn Iterator<Item = Result<(v4::Task, Option<Box<UpdateFile>>)>> + '_> {
-        let indexes = match self {
-            CompatV3ToV4::V3(v3) => v3.index_uuid(),
-            CompatV3ToV4::Compat(compat) => compat.index_uuid(),
-        };
-        let tasks = match self {
-            CompatV3ToV4::V3(v3) => v3.tasks(),
-            CompatV3ToV4::Compat(compat) => compat.tasks(),
-        };
-
-        Box::new(
-            tasks
-                // we need to override the old task ids that were generated
-                // by index in favor of a global unique incremental ID.
-                .enumerate()
-                .map(move |(task_id, task)| {
-                    task.map(|(task, content_file)| {
-                        let index_uid = indexes
-                            .iter()
-                            .find(|index| index.uuid == task.uuid)
-                            .map(|index| index.uid.clone());
-
-                        let index_uid = match index_uid {
-                            Some(uid) => uid,
-                            None => {
-                                log::warn!(
-                                    "Error while importing the update {}.",
-                                    task.update.id()
-                                );
-                                log::warn!(
-                                    "The index associated to the uuid `{}` could not be retrieved.",
-                                    task.uuid.to_string()
-                                );
-                                if task.update.is_finished() {
-                                    // we're fucking with his history but not his data, that's ok-ish.
-                                    log::warn!("The index-uuid will be set as `unknown`.");
-                                    String::from("unknown")
-                                } else {
-                                    log::warn!("The task will be ignored.");
-                                    return None;
-                                }
-                            }
-                        };
-
-                        let task = v4::Task {
-                            id: task_id as u32,
-                            index_uid: v4::meta::IndexUid(index_uid),
-                            content: match task.update.meta() {
-                                v3::Kind::DeleteDocuments(documents) => {
-                                    v4::tasks::TaskContent::DocumentDeletion(
-                                        v4::tasks::DocumentDeletion::Ids(documents.clone()),
-                                    )
-                                }
-                                v3::Kind::DocumentAddition {
-                                    primary_key,
-                                    method,
-                                    content_uuid,
-                                } => v4::tasks::TaskContent::DocumentAddition {
-                                    merge_strategy: match method {
-                                        v3::updates::IndexDocumentsMethod::ReplaceDocuments => {
-                                            v4::tasks::IndexDocumentsMethod::ReplaceDocuments
-                                        }
-                                        v3::updates::IndexDocumentsMethod::UpdateDocuments => {
-                                            v4::tasks::IndexDocumentsMethod::UpdateDocuments
-                                        }
-                                    },
-                                    primary_key: primary_key.clone(),
-                                    documents_count: 0, // we don't have this info
-                                    allow_index_creation: true, // there was no API-key in the v3
-                                    content_uuid: *content_uuid,
-                                },
-                                v3::Kind::Settings(settings) => {
-                                    v4::tasks::TaskContent::SettingsUpdate {
-                                        settings: v4::Settings::from(settings.clone()),
-                                        is_deletion: false, // that didn't exist at this time
-                                        allow_index_creation: true, // there was no API-key in the v3
-                                    }
-                                }
-                                v3::Kind::ClearDocuments => {
-                                    v4::tasks::TaskContent::DocumentDeletion(
-                                        v4::tasks::DocumentDeletion::Clear,
-                                    )
-                                }
-                            },
-                            events: match task.update {
-                                v3::Status::Processing(processing) => {
-                                    vec![v4::tasks::TaskEvent::Created(processing.from.enqueued_at)]
-                                }
-                                v3::Status::Enqueued(enqueued) => {
-                                    vec![v4::tasks::TaskEvent::Created(enqueued.enqueued_at)]
-                                }
-                                v3::Status::Processed(processed) => {
-                                    vec![
-                                        v4::tasks::TaskEvent::Created(
-                                            processed.from.from.enqueued_at,
-                                        ),
-                                        v4::tasks::TaskEvent::Processing(
-                                            processed.from.started_processing_at,
-                                        ),
-                                        v4::tasks::TaskEvent::Succeded {
-                                            result: match processed.success {
-                                                v3::updates::UpdateResult::DocumentsAddition(
-                                                    document_addition,
-                                                ) => v4::tasks::TaskResult::DocumentAddition {
-                                                    indexed_documents: document_addition
-                                                        .nb_documents
-                                                        as u64,
-                                                },
-                                                v3::updates::UpdateResult::DocumentDeletion {
-                                                    deleted,
-                                                } => v4::tasks::TaskResult::DocumentDeletion {
-                                                    deleted_documents: deleted,
-                                                },
-                                                v3::updates::UpdateResult::Other => {
-                                                    v4::tasks::TaskResult::Other
-                                                }
-                                            },
-                                            timestamp: processed.processed_at,
-                                        },
-                                    ]
-                                }
-                                v3::Status::Failed(failed) => vec![
-                                    v4::tasks::TaskEvent::Created(failed.from.from.enqueued_at),
-                                    v4::tasks::TaskEvent::Processing(
-                                        failed.from.started_processing_at,
-                                    ),
-                                    v4::tasks::TaskEvent::Failed {
-                                        error: v4::ResponseError::from_msg(
-                                            failed.msg.to_string(),
-                                            failed.code.into(),
-                                        ),
-                                        timestamp: failed.failed_at,
-                                    },
-                                ],
-                                v3::Status::Aborted(aborted) => vec![
-                            v4::tasks::TaskEvent::Created(aborted.from.enqueued_at),
-                            v4::tasks::TaskEvent::Failed {
-                                error: v4::ResponseError::from_msg(
-                                    "Task was aborted in a previous version of meilisearch."
-                                        .to_string(),
-                                    v4::errors::Code::UnretrievableErrorCode,
-                                ),
-                                timestamp: aborted.aborted_at,
-                            },
-                        ],
-                            },
-                        };
-
-                        Some((task, content_file))
-                    })
-                })
-                .filter_map(|res| res.transpose()),
-        )
-    }
-
-    pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v4::Key>> + '_> {
-        Box::new(std::iter::empty())
-    }
-}
-
-pub enum CompatIndexV3ToV4 {
-    V3(v3::V3IndexReader),
-    Compat(CompatIndexV2ToV3),
-}
-
-impl From<v3::V3IndexReader> for CompatIndexV3ToV4 {
-    fn from(index_reader: v3::V3IndexReader) -> Self {
-        Self::V3(index_reader)
-    }
-}
-
-impl From<CompatIndexV2ToV3> for CompatIndexV3ToV4 {
-    fn from(index_reader: CompatIndexV2ToV3) -> Self {
-        Self::Compat(index_reader)
-    }
-}
-
-impl CompatIndexV3ToV4 {
-    pub fn new(v3: v3::V3IndexReader) -> CompatIndexV3ToV4 {
-        CompatIndexV3ToV4::V3(v3)
-    }
-
-    pub fn metadata(&self) -> &crate::IndexMetadata {
-        match self {
-            CompatIndexV3ToV4::V3(v3) => v3.metadata(),
-            CompatIndexV3ToV4::Compat(compat) => compat.metadata(),
-        }
-    }
-
-    pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<v4::Document>> + '_>> {
-        match self {
-            CompatIndexV3ToV4::V3(v3) => v3
-                .documents()
-                .map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<v4::Document>> + '_>),
-
-            CompatIndexV3ToV4::Compat(compat) => compat
-                .documents()
-                .map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<v4::Document>> + '_>),
-        }
-    }
-
-    pub fn settings(&mut self) -> Result<v4::Settings<v4::Checked>> {
-        Ok(match self {
-            CompatIndexV3ToV4::V3(v3) => {
-                v4::Settings::<v4::Unchecked>::from(v3.settings()?).check()
-            }
-            CompatIndexV3ToV4::Compat(compat) => {
-                v4::Settings::<v4::Unchecked>::from(compat.settings()?).check()
-            }
-        })
-    }
-}
-
-impl<T> From<v3::Setting<T>> for v4::Setting<T> {
-    fn from(setting: v3::Setting<T>) -> Self {
-        match setting {
-            v3::Setting::Set(t) => v4::Setting::Set(t),
-            v3::Setting::Reset => v4::Setting::Reset,
-            v3::Setting::NotSet => v4::Setting::NotSet,
-        }
-    }
-}
-
-impl From<v3::Code> for v4::Code {
-    fn from(code: v3::Code) -> Self {
-        match code {
-            v3::Code::CreateIndex => v4::Code::CreateIndex,
-            v3::Code::IndexAlreadyExists => v4::Code::IndexAlreadyExists,
-            v3::Code::IndexNotFound => v4::Code::IndexNotFound,
-            v3::Code::InvalidIndexUid => v4::Code::InvalidIndexUid,
-            v3::Code::InvalidState => v4::Code::InvalidState,
-            v3::Code::MissingPrimaryKey => v4::Code::MissingPrimaryKey,
-            v3::Code::PrimaryKeyAlreadyPresent => v4::Code::PrimaryKeyAlreadyPresent,
-            v3::Code::MaxFieldsLimitExceeded => v4::Code::MaxFieldsLimitExceeded,
-            v3::Code::MissingDocumentId => v4::Code::MissingDocumentId,
-            v3::Code::InvalidDocumentId => v4::Code::InvalidDocumentId,
-            v3::Code::Filter => v4::Code::Filter,
-            v3::Code::Sort => v4::Code::Sort,
-            v3::Code::BadParameter => v4::Code::BadParameter,
-            v3::Code::BadRequest => v4::Code::BadRequest,
-            v3::Code::DatabaseSizeLimitReached => v4::Code::DatabaseSizeLimitReached,
-            v3::Code::DocumentNotFound => v4::Code::DocumentNotFound,
-            v3::Code::Internal => v4::Code::Internal,
-            v3::Code::InvalidGeoField => v4::Code::InvalidGeoField,
-            v3::Code::InvalidRankingRule => v4::Code::InvalidRankingRule,
-            v3::Code::InvalidStore => v4::Code::InvalidStore,
-            v3::Code::InvalidToken => v4::Code::InvalidToken,
-            v3::Code::MissingAuthorizationHeader => v4::Code::MissingAuthorizationHeader,
-            v3::Code::NoSpaceLeftOnDevice => v4::Code::NoSpaceLeftOnDevice,
-            v3::Code::DumpNotFound => v4::Code::DumpNotFound,
-            v3::Code::TaskNotFound => v4::Code::TaskNotFound,
-            v3::Code::PayloadTooLarge => v4::Code::PayloadTooLarge,
-            v3::Code::RetrieveDocument => v4::Code::RetrieveDocument,
-            v3::Code::SearchDocuments => v4::Code::SearchDocuments,
-            v3::Code::UnsupportedMediaType => v4::Code::UnsupportedMediaType,
-            v3::Code::DumpAlreadyInProgress => v4::Code::DumpAlreadyInProgress,
-            v3::Code::DumpProcessFailed => v4::Code::DumpProcessFailed,
-            v3::Code::InvalidContentType => v4::Code::InvalidContentType,
-            v3::Code::MissingContentType => v4::Code::MissingContentType,
-            v3::Code::MalformedPayload => v4::Code::MalformedPayload,
-            v3::Code::MissingPayload => v4::Code::MissingPayload,
-            v3::Code::UnretrievableErrorCode => v4::Code::UnretrievableErrorCode,
-            v3::Code::MalformedDump => v4::Code::MalformedDump,
-        }
-    }
-}
-
-impl<T> From<v3::Settings<T>> for v4::Settings<v4::Unchecked> {
-    fn from(settings: v3::Settings<T>) -> Self {
-        v4::Settings {
-            displayed_attributes: settings.displayed_attributes.into(),
-            searchable_attributes: settings.searchable_attributes.into(),
-            filterable_attributes: settings.filterable_attributes.into(),
-            sortable_attributes: settings.sortable_attributes.into(),
-            ranking_rules: settings.ranking_rules.into(),
-            stop_words: settings.stop_words.into(),
-            synonyms: settings.synonyms.into(),
-            distinct_attribute: settings.distinct_attribute.into(),
-            typo_tolerance: v4::Setting::NotSet,
-            _kind: std::marker::PhantomData,
-        }
-    }
-}
-
-#[cfg(test)]
-pub(crate) mod test {
-    use std::fs::File;
-    use std::io::BufReader;
-
-    use flate2::bufread::GzDecoder;
-    use meili_snap::insta;
-    use tempfile::TempDir;
-
-    use super::*;
-
-    #[test]
-    fn compat_v3_v4() {
-        let dump = File::open("tests/assets/v3.dump").unwrap();
-        let dir = TempDir::new().unwrap();
-        let mut dump = BufReader::new(dump);
-        let gz = GzDecoder::new(&mut dump);
-        let mut archive = tar::Archive::new(gz);
-        archive.unpack(dir.path()).unwrap();
-
-        let mut dump = v3::V3Reader::open(dir).unwrap().to_v4();
-
-        // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
-
-        // tasks
-        let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, mut update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"79bc053583a1a7172bbaaafb1edaeb78");
-        assert_eq!(update_files.len(), 10);
-        assert!(update_files[0].is_some()); // the enqueued document addition
-        assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
-
-        let update_file = update_files.remove(0).unwrap().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(update_file), @"7b8889539b669c7b9ddba448bafa385d");
-
-        // keys
-        let keys = dump.keys().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(keys, { "[].uid" => "[uuid]" }), @"d751713988987e9331980363e24189ce");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut products = indexes.pop().unwrap();
-        let mut movies2 = indexes.pop().unwrap();
-        let mut movies = indexes.pop().unwrap();
-        let mut spells = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "products",
-          "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(products.settings().unwrap());
-        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
-
-        // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "movies",
-          "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies.settings().unwrap());
-        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 110);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
-
-        // movies2
-        insta::assert_json_snapshot!(movies2.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "movies_2",
-          "primaryKey": null,
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies2.settings().unwrap());
-        let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 0);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
-
-        // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "dnd_spells",
-          "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(spells.settings().unwrap());
-        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
-    }
-}

dump/src/reader/compat/v4_to_v5.rs

@@ -1,467 +0,0 @@
-use super::v3_to_v4::{CompatIndexV3ToV4, CompatV3ToV4};
-use super::v5_to_v6::CompatV5ToV6;
-use crate::reader::{v4, v5, Document};
-use crate::Result;
-
-pub enum CompatV4ToV5 {
-    V4(v4::V4Reader),
-    Compat(CompatV3ToV4),
-}
-
-impl CompatV4ToV5 {
-    pub fn new(v4: v4::V4Reader) -> CompatV4ToV5 {
-        CompatV4ToV5::V4(v4)
-    }
-
-    pub fn to_v6(self) -> CompatV5ToV6 {
-        CompatV5ToV6::Compat(self)
-    }
-
-    pub fn version(&self) -> crate::Version {
-        match self {
-            CompatV4ToV5::V4(v4) => v4.version(),
-            CompatV4ToV5::Compat(compat) => compat.version(),
-        }
-    }
-
-    pub fn date(&self) -> Option<time::OffsetDateTime> {
-        match self {
-            CompatV4ToV5::V4(v4) => v4.date(),
-            CompatV4ToV5::Compat(compat) => compat.date(),
-        }
-    }
-
-    pub fn instance_uid(&self) -> Result<Option<uuid::Uuid>> {
-        match self {
-            CompatV4ToV5::V4(v4) => v4.instance_uid(),
-            CompatV4ToV5::Compat(compat) => compat.instance_uid(),
-        }
-    }
-
-    pub fn indexes(&self) -> Result<Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>> {
-        Ok(match self {
-            CompatV4ToV5::V4(v4) => {
-                Box::new(v4.indexes()?.map(|index| index.map(CompatIndexV4ToV5::from)))
-                    as Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>
-            }
-
-            CompatV4ToV5::Compat(compat) => {
-                Box::new(compat.indexes()?.map(|index| index.map(CompatIndexV4ToV5::from)))
-                    as Box<dyn Iterator<Item = Result<CompatIndexV4ToV5>> + '_>
-            }
-        })
-    }
-
-    pub fn tasks(
-        &mut self,
-    ) -> Box<dyn Iterator<Item = Result<(v5::Task, Option<Box<crate::reader::UpdateFile>>)>> + '_>
-    {
-        let tasks = match self {
-            CompatV4ToV5::V4(v4) => v4.tasks(),
-            CompatV4ToV5::Compat(compat) => compat.tasks(),
-        };
-        Box::new(tasks.map(|task| {
-            task.map(|(task, content_file)| {
-                let task = v5::Task {
-                    id: task.id,
-                    content: match task.content {
-                        v4::tasks::TaskContent::DocumentAddition {
-                            content_uuid,
-                            merge_strategy,
-                            primary_key,
-                            documents_count,
-                            allow_index_creation,
-                        } => v5::tasks::TaskContent::DocumentAddition {
-                            index_uid: v5::meta::IndexUid(task.index_uid.0),
-                            content_uuid,
-                            merge_strategy: match merge_strategy {
-                                v4::tasks::IndexDocumentsMethod::ReplaceDocuments => {
-                                    v5::tasks::IndexDocumentsMethod::ReplaceDocuments
-                                }
-                                v4::tasks::IndexDocumentsMethod::UpdateDocuments => {
-                                    v5::tasks::IndexDocumentsMethod::UpdateDocuments
-                                }
-                            },
-                            primary_key,
-                            documents_count,
-                            allow_index_creation,
-                        },
-                        v4::tasks::TaskContent::DocumentDeletion(deletion) => {
-                            v5::tasks::TaskContent::DocumentDeletion {
-                                index_uid: v5::meta::IndexUid(task.index_uid.0),
-                                deletion: match deletion {
-                                    v4::tasks::DocumentDeletion::Clear => {
-                                        v5::tasks::DocumentDeletion::Clear
-                                    }
-                                    v4::tasks::DocumentDeletion::Ids(ids) => {
-                                        v5::tasks::DocumentDeletion::Ids(ids)
-                                    }
-                                },
-                            }
-                        }
-                        v4::tasks::TaskContent::SettingsUpdate {
-                            settings,
-                            is_deletion,
-                            allow_index_creation,
-                        } => v5::tasks::TaskContent::SettingsUpdate {
-                            index_uid: v5::meta::IndexUid(task.index_uid.0),
-                            settings: settings.into(),
-                            is_deletion,
-                            allow_index_creation,
-                        },
-                        v4::tasks::TaskContent::IndexDeletion => {
-                            v5::tasks::TaskContent::IndexDeletion {
-                                index_uid: v5::meta::IndexUid(task.index_uid.0),
-                            }
-                        }
-                        v4::tasks::TaskContent::IndexCreation { primary_key } => {
-                            v5::tasks::TaskContent::IndexCreation {
-                                index_uid: v5::meta::IndexUid(task.index_uid.0),
-                                primary_key,
-                            }
-                        }
-                        v4::tasks::TaskContent::IndexUpdate { primary_key } => {
-                            v5::tasks::TaskContent::IndexUpdate {
-                                index_uid: v5::meta::IndexUid(task.index_uid.0),
-                                primary_key,
-                            }
-                        }
-                    },
-                    events: task
-                        .events
-                        .into_iter()
-                        .map(|event| match event {
-                            v4::tasks::TaskEvent::Created(date) => {
-                                v5::tasks::TaskEvent::Created(date)
-                            }
-                            v4::tasks::TaskEvent::Batched { timestamp, batch_id } => {
-                                v5::tasks::TaskEvent::Batched { timestamp, batch_id }
-                            }
-                            v4::tasks::TaskEvent::Processing(date) => {
-                                v5::tasks::TaskEvent::Processing(date)
-                            }
-                            v4::tasks::TaskEvent::Succeded { result, timestamp } => {
-                                v5::tasks::TaskEvent::Succeeded {
-                                    result: match result {
-                                        v4::tasks::TaskResult::DocumentAddition {
-                                            indexed_documents,
-                                        } => v5::tasks::TaskResult::DocumentAddition {
-                                            indexed_documents,
-                                        },
-                                        v4::tasks::TaskResult::DocumentDeletion {
-                                            deleted_documents,
-                                        } => v5::tasks::TaskResult::DocumentDeletion {
-                                            deleted_documents,
-                                        },
-                                        v4::tasks::TaskResult::ClearAll { deleted_documents } => {
-                                            v5::tasks::TaskResult::ClearAll { deleted_documents }
-                                        }
-                                        v4::tasks::TaskResult::Other => {
-                                            v5::tasks::TaskResult::Other
-                                        }
-                                    },
-                                    timestamp,
-                                }
-                            }
-                            v4::tasks::TaskEvent::Failed { error, timestamp } => {
-                                v5::tasks::TaskEvent::Failed {
-                                    error: v5::ResponseError::from(error),
-                                    timestamp,
-                                }
-                            }
-                        })
-                        .collect(),
-                };
-
-                (task, content_file)
-            })
-        }))
-    }
-
-    pub fn keys(&mut self) -> Box<dyn Iterator<Item = Result<v5::Key>> + '_> {
-        let keys = match self {
-            CompatV4ToV5::V4(v4) => v4.keys(),
-            CompatV4ToV5::Compat(compat) => compat.keys(),
-        };
-        Box::new(keys.map(|key| {
-            key.map(|key| v5::Key {
-                description: key.description,
-                name: None,
-                uid: v5::keys::KeyId::new_v4(),
-                actions: key.actions.into_iter().filter_map(|action| action.into()).collect(),
-                indexes: key
-                    .indexes
-                    .into_iter()
-                    .map(|index| match index.as_str() {
-                        "*" => v5::StarOr::Star,
-                        _ => v5::StarOr::Other(v5::meta::IndexUid(index)),
-                    })
-                    .collect(),
-                expires_at: key.expires_at,
-                created_at: key.created_at,
-                updated_at: key.updated_at,
-            })
-        }))
-    }
-}
-
-pub enum CompatIndexV4ToV5 {
-    V4(v4::V4IndexReader),
-    Compat(CompatIndexV3ToV4),
-}
-
-impl From<v4::V4IndexReader> for CompatIndexV4ToV5 {
-    fn from(index_reader: v4::V4IndexReader) -> Self {
-        Self::V4(index_reader)
-    }
-}
-
-impl From<CompatIndexV3ToV4> for CompatIndexV4ToV5 {
-    fn from(index_reader: CompatIndexV3ToV4) -> Self {
-        Self::Compat(index_reader)
-    }
-}
-
-impl CompatIndexV4ToV5 {
-    pub fn metadata(&self) -> &crate::IndexMetadata {
-        match self {
-            CompatIndexV4ToV5::V4(v4) => v4.metadata(),
-            CompatIndexV4ToV5::Compat(compat) => compat.metadata(),
-        }
-    }
-
-    pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Document>> + '_>> {
-        match self {
-            CompatIndexV4ToV5::V4(v4) => v4
-                .documents()
-                .map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
-            CompatIndexV4ToV5::Compat(compat) => compat
-                .documents()
-                .map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
-        }
-    }
-
-    pub fn settings(&mut self) -> Result<v5::Settings<v5::Checked>> {
-        match self {
-            CompatIndexV4ToV5::V4(v4) => Ok(v5::Settings::from(v4.settings()?).check()),
-            CompatIndexV4ToV5::Compat(compat) => Ok(v5::Settings::from(compat.settings()?).check()),
-        }
-    }
-}
-
-impl<T> From<v4::Setting<T>> for v5::Setting<T> {
-    fn from(setting: v4::Setting<T>) -> Self {
-        match setting {
-            v4::Setting::Set(t) => v5::Setting::Set(t),
-            v4::Setting::Reset => v5::Setting::Reset,
-            v4::Setting::NotSet => v5::Setting::NotSet,
-        }
-    }
-}
-
-impl From<v4::ResponseError> for v5::ResponseError {
-    fn from(error: v4::ResponseError) -> Self {
-        let code = match error.error_code.as_ref() {
-            "index_creation_failed" => v5::Code::CreateIndex,
-            "index_already_exists" => v5::Code::IndexAlreadyExists,
-            "index_not_found" => v5::Code::IndexNotFound,
-            "invalid_index_uid" => v5::Code::InvalidIndexUid,
-            "invalid_min_word_length_for_typo" => v5::Code::InvalidMinWordLengthForTypo,
-            "invalid_state" => v5::Code::InvalidState,
-            "primary_key_inference_failed" => v5::Code::MissingPrimaryKey,
-            "index_primary_key_already_exists" => v5::Code::PrimaryKeyAlreadyPresent,
-            "max_fields_limit_exceeded" => v5::Code::MaxFieldsLimitExceeded,
-            "missing_document_id" => v5::Code::MissingDocumentId,
-            "invalid_document_id" => v5::Code::InvalidDocumentId,
-            "invalid_filter" => v5::Code::Filter,
-            "invalid_sort" => v5::Code::Sort,
-            "bad_parameter" => v5::Code::BadParameter,
-            "bad_request" => v5::Code::BadRequest,
-            "database_size_limit_reached" => v5::Code::DatabaseSizeLimitReached,
-            "document_not_found" => v5::Code::DocumentNotFound,
-            "internal" => v5::Code::Internal,
-            "invalid_geo_field" => v5::Code::InvalidGeoField,
-            "invalid_ranking_rule" => v5::Code::InvalidRankingRule,
-            "invalid_store_file" => v5::Code::InvalidStore,
-            "invalid_api_key" => v5::Code::InvalidToken,
-            "missing_authorization_header" => v5::Code::MissingAuthorizationHeader,
-            "no_space_left_on_device" => v5::Code::NoSpaceLeftOnDevice,
-            "dump_not_found" => v5::Code::DumpNotFound,
-            "task_not_found" => v5::Code::TaskNotFound,
-            "payload_too_large" => v5::Code::PayloadTooLarge,
-            "unretrievable_document" => v5::Code::RetrieveDocument,
-            "search_error" => v5::Code::SearchDocuments,
-            "unsupported_media_type" => v5::Code::UnsupportedMediaType,
-            "dump_already_processing" => v5::Code::DumpAlreadyInProgress,
-            "dump_process_failed" => v5::Code::DumpProcessFailed,
-            "invalid_content_type" => v5::Code::InvalidContentType,
-            "missing_content_type" => v5::Code::MissingContentType,
-            "malformed_payload" => v5::Code::MalformedPayload,
-            "missing_payload" => v5::Code::MissingPayload,
-            "api_key_not_found" => v5::Code::ApiKeyNotFound,
-            "missing_parameter" => v5::Code::MissingParameter,
-            "invalid_api_key_actions" => v5::Code::InvalidApiKeyActions,
-            "invalid_api_key_indexes" => v5::Code::InvalidApiKeyIndexes,
-            "invalid_api_key_expires_at" => v5::Code::InvalidApiKeyExpiresAt,
-            "invalid_api_key_description" => v5::Code::InvalidApiKeyDescription,
-            other => {
-                log::warn!("Unknown error code {}", other);
-                v5::Code::UnretrievableErrorCode
-            }
-        };
-        v5::ResponseError::from_msg(error.message, code)
-    }
-}
-
-impl<T> From<v4::Settings<T>> for v5::Settings<v5::Unchecked> {
-    fn from(settings: v4::Settings<T>) -> Self {
-        v5::Settings {
-            displayed_attributes: settings.displayed_attributes.into(),
-            searchable_attributes: settings.searchable_attributes.into(),
-            filterable_attributes: settings.filterable_attributes.into(),
-            sortable_attributes: settings.sortable_attributes.into(),
-            ranking_rules: settings.ranking_rules.into(),
-            stop_words: settings.stop_words.into(),
-            synonyms: settings.synonyms.into(),
-            distinct_attribute: settings.distinct_attribute.into(),
-            typo_tolerance: match settings.typo_tolerance {
-                v4::Setting::Set(typo) => v5::Setting::Set(v5::TypoTolerance {
-                    enabled: typo.enabled.into(),
-                    min_word_size_for_typos: match typo.min_word_size_for_typos {
-                        v4::Setting::Set(t) => v5::Setting::Set(v5::MinWordSizeForTypos {
-                            one_typo: t.one_typo.into(),
-                            two_typos: t.two_typos.into(),
-                        }),
-                        v4::Setting::Reset => v5::Setting::Reset,
-                        v4::Setting::NotSet => v5::Setting::NotSet,
-                    },
-                    disable_on_words: typo.disable_on_words.into(),
-                    disable_on_attributes: typo.disable_on_attributes.into(),
-                }),
-                v4::Setting::Reset => v5::Setting::Reset,
-                v4::Setting::NotSet => v5::Setting::NotSet,
-            },
-            faceting: v5::Setting::NotSet,
-            pagination: v5::Setting::NotSet,
-            _kind: std::marker::PhantomData,
-        }
-    }
-}
-
-impl From<v4::Action> for Option<v5::Action> {
-    fn from(key: v4::Action) -> Self {
-        match key {
-            v4::Action::All => Some(v5::Action::All),
-            v4::Action::Search => Some(v5::Action::Search),
-            v4::Action::DocumentsAdd => Some(v5::Action::DocumentsAdd),
-            v4::Action::DocumentsGet => Some(v5::Action::DocumentsGet),
-            v4::Action::DocumentsDelete => Some(v5::Action::DocumentsDelete),
-            v4::Action::IndexesAdd => Some(v5::Action::IndexesAdd),
-            v4::Action::IndexesGet => Some(v5::Action::IndexesGet),
-            v4::Action::IndexesUpdate => Some(v5::Action::IndexesUpdate),
-            v4::Action::IndexesDelete => Some(v5::Action::IndexesDelete),
-            v4::Action::TasksGet => Some(v5::Action::TasksGet),
-            v4::Action::SettingsGet => Some(v5::Action::SettingsGet),
-            v4::Action::SettingsUpdate => Some(v5::Action::SettingsUpdate),
-            v4::Action::StatsGet => Some(v5::Action::StatsGet),
-            v4::Action::DumpsCreate => Some(v5::Action::DumpsCreate),
-            v4::Action::DumpsGet => None,
-            v4::Action::Version => Some(v5::Action::Version),
-        }
-    }
-}
-
-#[cfg(test)]
-pub(crate) mod test {
-    use std::fs::File;
-    use std::io::BufReader;
-
-    use flate2::bufread::GzDecoder;
-    use meili_snap::insta;
-    use tempfile::TempDir;
-
-    use super::*;
-
-    #[test]
-    fn compat_v4_v5() {
-        let dump = File::open("tests/assets/v4.dump").unwrap();
-        let dir = TempDir::new().unwrap();
-        let mut dump = BufReader::new(dump);
-        let gz = GzDecoder::new(&mut dump);
-        let mut archive = tar::Archive::new(gz);
-        archive.unpack(dir.path()).unwrap();
-
-        let mut dump = v4::V4Reader::open(dir).unwrap().to_v5();
-
-        // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
-
-        // tasks
-        let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"ed9a30cded4c046ef46f7cff7450347e");
-        assert_eq!(update_files.len(), 10);
-        assert!(update_files[0].is_some()); // the enqueued document addition
-        assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
-
-        // keys
-        let keys = dump.keys().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(keys, { "[].uid" => "[uuid]" }), @"1384361d734fd77c23804c9696228660");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut products = indexes.pop().unwrap();
-        let mut movies = indexes.pop().unwrap();
-        let mut spells = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "products",
-          "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(products.settings().unwrap());
-        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
-
-        // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "movies",
-          "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies.settings().unwrap());
-        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 110);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab");
-
-        // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "dnd_spells",
-          "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(spells.settings().unwrap());
-        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
-    }
-}

dump/src/reader/compat/v5_to_v6.rs

@@ -1,513 +0,0 @@
-use std::str::FromStr;
-
-use super::v4_to_v5::{CompatIndexV4ToV5, CompatV4ToV5};
-use crate::reader::{v5, v6, Document, UpdateFile};
-use crate::Result;
-
-pub enum CompatV5ToV6 {
-    V5(v5::V5Reader),
-    Compat(CompatV4ToV5),
-}
-
-impl CompatV5ToV6 {
-    pub fn new_v5(v5: v5::V5Reader) -> CompatV5ToV6 {
-        CompatV5ToV6::V5(v5)
-    }
-
-    pub fn version(&self) -> crate::Version {
-        match self {
-            CompatV5ToV6::V5(v5) => v5.version(),
-            CompatV5ToV6::Compat(compat) => compat.version(),
-        }
-    }
-
-    pub fn date(&self) -> Option<time::OffsetDateTime> {
-        match self {
-            CompatV5ToV6::V5(v5) => v5.date(),
-            CompatV5ToV6::Compat(compat) => compat.date(),
-        }
-    }
-
-    pub fn instance_uid(&self) -> Result<Option<uuid::Uuid>> {
-        match self {
-            CompatV5ToV6::V5(v5) => v5.instance_uid(),
-            CompatV5ToV6::Compat(compat) => compat.instance_uid(),
-        }
-    }
-
-    pub fn indexes(&self) -> Result<Box<dyn Iterator<Item = Result<CompatIndexV5ToV6>> + '_>> {
-        let indexes = match self {
-            CompatV5ToV6::V5(v5) => {
-                Box::new(v5.indexes()?.map(|index| index.map(CompatIndexV5ToV6::from)))
-                    as Box<dyn Iterator<Item = Result<CompatIndexV5ToV6>> + '_>
-            }
-
-            CompatV5ToV6::Compat(compat) => {
-                Box::new(compat.indexes()?.map(|index| index.map(CompatIndexV5ToV6::from)))
-                    as Box<dyn Iterator<Item = Result<CompatIndexV5ToV6>> + '_>
-            }
-        };
-        Ok(indexes)
-    }
-
-    pub fn tasks(
-        &mut self,
-    ) -> Result<Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_>> {
-        let instance_uid = self.instance_uid().ok().flatten();
-        let keys = self.keys()?.collect::<Result<Vec<_>>>()?;
-
-        let tasks = match self {
-            CompatV5ToV6::V5(v5) => v5.tasks(),
-            CompatV5ToV6::Compat(compat) => compat.tasks(),
-        };
-        Ok(Box::new(tasks.map(move |task| {
-            task.map(|(task, content_file)| {
-                let mut task_view: v5::tasks::TaskView = task.clone().into();
-
-                if task_view.status == v5::Status::Processing {
-                    task_view.started_at = None;
-                }
-
-                let task = v6::Task {
-                    uid: task_view.uid,
-                    index_uid: task_view.index_uid,
-                    status: match task_view.status {
-                        v5::Status::Enqueued => v6::Status::Enqueued,
-                        v5::Status::Processing => v6::Status::Enqueued,
-                        v5::Status::Succeeded => v6::Status::Succeeded,
-                        v5::Status::Failed => v6::Status::Failed,
-                    },
-                    kind: match task.content {
-                        v5::tasks::TaskContent::IndexCreation { primary_key, .. } => {
-                            v6::Kind::IndexCreation { primary_key }
-                        }
-                        v5::tasks::TaskContent::IndexUpdate { primary_key, .. } => {
-                            v6::Kind::IndexUpdate { primary_key }
-                        }
-                        v5::tasks::TaskContent::IndexDeletion { .. } => v6::Kind::IndexDeletion,
-                        v5::tasks::TaskContent::DocumentAddition {
-                            merge_strategy,
-                            allow_index_creation,
-                            primary_key,
-                            documents_count,
-                            ..
-                        } => v6::Kind::DocumentImport {
-                            primary_key,
-                            documents_count: documents_count as u64,
-                            method: match merge_strategy {
-                                v5::tasks::IndexDocumentsMethod::ReplaceDocuments => {
-                                    v6::milli::update::IndexDocumentsMethod::ReplaceDocuments
-                                }
-                                v5::tasks::IndexDocumentsMethod::UpdateDocuments => {
-                                    v6::milli::update::IndexDocumentsMethod::UpdateDocuments
-                                }
-                            },
-                            allow_index_creation,
-                        },
-                        v5::tasks::TaskContent::DocumentDeletion { deletion, .. } => match deletion
-                        {
-                            v5::tasks::DocumentDeletion::Clear => v6::Kind::DocumentClear,
-                            v5::tasks::DocumentDeletion::Ids(documents_ids) => {
-                                v6::Kind::DocumentDeletion { documents_ids }
-                            }
-                        },
-                        v5::tasks::TaskContent::SettingsUpdate {
-                            allow_index_creation,
-                            is_deletion,
-                            settings,
-                            ..
-                        } => v6::Kind::Settings {
-                            is_deletion,
-                            allow_index_creation,
-                            settings: Box::new(settings.into()),
-                        },
-                        v5::tasks::TaskContent::Dump { uid: _ } => {
-                            // in v6 we compute the dump_uid from the started_at processing time
-                            v6::Kind::DumpCreation { keys: keys.clone(), instance_uid }
-                        }
-                    },
-                    canceled_by: None,
-                    details: task_view.details.map(|details| match details {
-                        v5::Details::DocumentAddition { received_documents, indexed_documents } => {
-                            v6::Details::DocumentAdditionOrUpdate {
-                                received_documents: received_documents as u64,
-                                indexed_documents,
-                            }
-                        }
-                        v5::Details::Settings { settings } => {
-                            v6::Details::SettingsUpdate { settings: Box::new(settings.into()) }
-                        }
-                        v5::Details::IndexInfo { primary_key } => {
-                            v6::Details::IndexInfo { primary_key }
-                        }
-                        v5::Details::DocumentDeletion {
-                            received_document_ids,
-                            deleted_documents,
-                        } => v6::Details::DocumentDeletion {
-                            provided_ids: received_document_ids,
-                            deleted_documents,
-                        },
-                        v5::Details::ClearAll { deleted_documents } => {
-                            v6::Details::ClearAll { deleted_documents }
-                        }
-                        v5::Details::Dump { dump_uid } => {
-                            v6::Details::Dump { dump_uid: Some(dump_uid) }
-                        }
-                    }),
-                    error: task_view.error.map(|e| e.into()),
-                    enqueued_at: task_view.enqueued_at,
-                    started_at: task_view.started_at,
-                    finished_at: task_view.finished_at,
-                };
-
-                (task, content_file)
-            })
-        })))
-    }
-
-    pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
-        let keys = match self {
-            CompatV5ToV6::V5(v5) => v5.keys()?,
-            CompatV5ToV6::Compat(compat) => compat.keys(),
-        };
-
-        Ok(Box::new(keys.map(|key| {
-            key.map(|key| v6::Key {
-                description: key.description,
-                name: key.name,
-                uid: key.uid,
-                actions: key.actions.into_iter().map(|action| action.into()).collect(),
-                indexes: key
-                    .indexes
-                    .into_iter()
-                    .map(|index| match index {
-                        v5::StarOr::Star => v6::IndexUidPattern::all(),
-                        v5::StarOr::Other(uid) => v6::IndexUidPattern::new_unchecked(uid.as_str()),
-                    })
-                    .collect(),
-                expires_at: key.expires_at,
-                created_at: key.created_at,
-                updated_at: key.updated_at,
-            })
-        })))
-    }
-
-    pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
-        Ok(None)
-    }
-}
-
-pub enum CompatIndexV5ToV6 {
-    V5(v5::V5IndexReader),
-    Compat(CompatIndexV4ToV5),
-}
-
-impl From<v5::V5IndexReader> for CompatIndexV5ToV6 {
-    fn from(index_reader: v5::V5IndexReader) -> Self {
-        Self::V5(index_reader)
-    }
-}
-
-impl From<CompatIndexV4ToV5> for CompatIndexV5ToV6 {
-    fn from(index_reader: CompatIndexV4ToV5) -> Self {
-        Self::Compat(index_reader)
-    }
-}
-
-impl CompatIndexV5ToV6 {
-    pub fn new_v5(v5: v5::V5IndexReader) -> CompatIndexV5ToV6 {
-        CompatIndexV5ToV6::V5(v5)
-    }
-
-    pub fn metadata(&self) -> &crate::IndexMetadata {
-        match self {
-            CompatIndexV5ToV6::V5(v5) => v5.metadata(),
-            CompatIndexV5ToV6::Compat(compat) => compat.metadata(),
-        }
-    }
-
-    pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Document>> + '_>> {
-        match self {
-            CompatIndexV5ToV6::V5(v5) => v5
-                .documents()
-                .map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
-            CompatIndexV5ToV6::Compat(compat) => compat
-                .documents()
-                .map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
-        }
-    }
-
-    pub fn settings(&mut self) -> Result<v6::Settings<v6::Checked>> {
-        match self {
-            CompatIndexV5ToV6::V5(v5) => Ok(v6::Settings::from(v5.settings()?).check()),
-            CompatIndexV5ToV6::Compat(compat) => Ok(v6::Settings::from(compat.settings()?).check()),
-        }
-    }
-}
-
-impl<T> From<v5::Setting<T>> for v6::Setting<T> {
-    fn from(setting: v5::Setting<T>) -> Self {
-        match setting {
-            v5::Setting::Set(t) => v6::Setting::Set(t),
-            v5::Setting::Reset => v6::Setting::Reset,
-            v5::Setting::NotSet => v6::Setting::NotSet,
-        }
-    }
-}
-
-impl From<v5::ResponseError> for v6::ResponseError {
-    fn from(error: v5::ResponseError) -> Self {
-        let code = match error.error_code.as_ref() {
-            "index_creation_failed" => v6::Code::IndexCreationFailed,
-            "index_already_exists" => v6::Code::IndexAlreadyExists,
-            "index_not_found" => v6::Code::IndexNotFound,
-            "invalid_index_uid" => v6::Code::InvalidIndexUid,
-            "invalid_min_word_length_for_typo" => v6::Code::InvalidSettingsTypoTolerance,
-            "invalid_state" => v6::Code::InvalidState,
-            "primary_key_inference_failed" => v6::Code::IndexPrimaryKeyNoCandidateFound,
-            "index_primary_key_already_exists" => v6::Code::IndexPrimaryKeyAlreadyExists,
-            "max_fields_limit_exceeded" => v6::Code::MaxFieldsLimitExceeded,
-            "missing_document_id" => v6::Code::MissingDocumentId,
-            "invalid_document_id" => v6::Code::InvalidDocumentId,
-            "invalid_filter" => v6::Code::InvalidSettingsFilterableAttributes,
-            "invalid_sort" => v6::Code::InvalidSettingsSortableAttributes,
-            "bad_parameter" => v6::Code::BadParameter,
-            "bad_request" => v6::Code::BadRequest,
-            "database_size_limit_reached" => v6::Code::DatabaseSizeLimitReached,
-            "document_not_found" => v6::Code::DocumentNotFound,
-            "internal" => v6::Code::Internal,
-            "invalid_geo_field" => v6::Code::InvalidDocumentGeoField,
-            "invalid_ranking_rule" => v6::Code::InvalidSettingsRankingRules,
-            "invalid_store_file" => v6::Code::InvalidStoreFile,
-            "invalid_api_key" => v6::Code::InvalidApiKey,
-            "missing_authorization_header" => v6::Code::MissingAuthorizationHeader,
-            "no_space_left_on_device" => v6::Code::NoSpaceLeftOnDevice,
-            "dump_not_found" => v6::Code::DumpNotFound,
-            "task_not_found" => v6::Code::TaskNotFound,
-            "payload_too_large" => v6::Code::PayloadTooLarge,
-            "unretrievable_document" => v6::Code::UnretrievableDocument,
-            "unsupported_media_type" => v6::Code::UnsupportedMediaType,
-            "dump_already_processing" => v6::Code::DumpAlreadyProcessing,
-            "dump_process_failed" => v6::Code::DumpProcessFailed,
-            "invalid_content_type" => v6::Code::InvalidContentType,
-            "missing_content_type" => v6::Code::MissingContentType,
-            "malformed_payload" => v6::Code::MalformedPayload,
-            "missing_payload" => v6::Code::MissingPayload,
-            "api_key_not_found" => v6::Code::ApiKeyNotFound,
-            "missing_parameter" => v6::Code::BadRequest,
-            "invalid_api_key_actions" => v6::Code::InvalidApiKeyActions,
-            "invalid_api_key_indexes" => v6::Code::InvalidApiKeyIndexes,
-            "invalid_api_key_expires_at" => v6::Code::InvalidApiKeyExpiresAt,
-            "invalid_api_key_description" => v6::Code::InvalidApiKeyDescription,
-            "invalid_api_key_name" => v6::Code::InvalidApiKeyName,
-            "invalid_api_key_uid" => v6::Code::InvalidApiKeyUid,
-            "immutable_field" => v6::Code::BadRequest,
-            "api_key_already_exists" => v6::Code::ApiKeyAlreadyExists,
-            other => {
-                log::warn!("Unknown error code {}", other);
-                v6::Code::UnretrievableErrorCode
-            }
-        };
-        v6::ResponseError::from_msg(error.message, code)
-    }
-}
-
-impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
-    fn from(settings: v5::Settings<T>) -> Self {
-        v6::Settings {
-            displayed_attributes: settings.displayed_attributes.into(),
-            searchable_attributes: settings.searchable_attributes.into(),
-            filterable_attributes: settings.filterable_attributes.into(),
-            sortable_attributes: settings.sortable_attributes.into(),
-            ranking_rules: {
-                match settings.ranking_rules {
-                    v5::settings::Setting::Set(ranking_rules) => {
-                        let mut new_ranking_rules = vec![];
-                        for rule in ranking_rules {
-                            match v6::RankingRuleView::from_str(&rule) {
-                                Ok(new_rule) => {
-                                    new_ranking_rules.push(new_rule);
-                                }
-                                Err(_) => {
-                                    log::warn!("Error while importing settings. The ranking rule `{rule}` does not exist anymore.")
-                                }
-                            }
-                        }
-                        v6::Setting::Set(new_ranking_rules)
-                    }
-                    v5::settings::Setting::Reset => v6::Setting::Reset,
-                    v5::settings::Setting::NotSet => v6::Setting::NotSet,
-                }
-            },
-            stop_words: settings.stop_words.into(),
-            non_separator_tokens: v6::Setting::NotSet,
-            separator_tokens: v6::Setting::NotSet,
-            dictionary: v6::Setting::NotSet,
-            synonyms: settings.synonyms.into(),
-            distinct_attribute: settings.distinct_attribute.into(),
-            typo_tolerance: match settings.typo_tolerance {
-                v5::Setting::Set(typo) => v6::Setting::Set(v6::TypoTolerance {
-                    enabled: typo.enabled.into(),
-                    min_word_size_for_typos: match typo.min_word_size_for_typos {
-                        v5::Setting::Set(t) => v6::Setting::Set(v6::MinWordSizeForTypos {
-                            one_typo: t.one_typo.into(),
-                            two_typos: t.two_typos.into(),
-                        }),
-                        v5::Setting::Reset => v6::Setting::Reset,
-                        v5::Setting::NotSet => v6::Setting::NotSet,
-                    },
-                    disable_on_words: typo.disable_on_words.into(),
-                    disable_on_attributes: typo.disable_on_attributes.into(),
-                }),
-                v5::Setting::Reset => v6::Setting::Reset,
-                v5::Setting::NotSet => v6::Setting::NotSet,
-            },
-            faceting: match settings.faceting {
-                v5::Setting::Set(faceting) => v6::Setting::Set(v6::FacetingSettings {
-                    max_values_per_facet: faceting.max_values_per_facet.into(),
-                    sort_facet_values_by: v6::Setting::NotSet,
-                }),
-                v5::Setting::Reset => v6::Setting::Reset,
-                v5::Setting::NotSet => v6::Setting::NotSet,
-            },
-            pagination: match settings.pagination {
-                v5::Setting::Set(pagination) => v6::Setting::Set(v6::PaginationSettings {
-                    max_total_hits: pagination.max_total_hits.into(),
-                }),
-                v5::Setting::Reset => v6::Setting::Reset,
-                v5::Setting::NotSet => v6::Setting::NotSet,
-            },
-            _kind: std::marker::PhantomData,
-        }
-    }
-}
-
-impl From<v5::Action> for v6::Action {
-    fn from(key: v5::Action) -> Self {
-        match key {
-            v5::Action::All => v6::Action::All,
-            v5::Action::Search => v6::Action::Search,
-            v5::Action::DocumentsAll => v6::Action::DocumentsAll,
-            v5::Action::DocumentsAdd => v6::Action::DocumentsAdd,
-            v5::Action::DocumentsGet => v6::Action::DocumentsGet,
-            v5::Action::DocumentsDelete => v6::Action::DocumentsDelete,
-            v5::Action::IndexesAll => v6::Action::IndexesAll,
-            v5::Action::IndexesAdd => v6::Action::IndexesAdd,
-            v5::Action::IndexesGet => v6::Action::IndexesGet,
-            v5::Action::IndexesUpdate => v6::Action::IndexesUpdate,
-            v5::Action::IndexesDelete => v6::Action::IndexesDelete,
-            v5::Action::TasksAll => v6::Action::TasksAll,
-            v5::Action::TasksGet => v6::Action::TasksGet,
-            v5::Action::SettingsAll => v6::Action::SettingsAll,
-            v5::Action::SettingsGet => v6::Action::SettingsGet,
-            v5::Action::SettingsUpdate => v6::Action::SettingsUpdate,
-            v5::Action::StatsAll => v6::Action::StatsAll,
-            v5::Action::StatsGet => v6::Action::StatsGet,
-            v5::Action::MetricsAll => v6::Action::MetricsAll,
-            v5::Action::MetricsGet => v6::Action::MetricsGet,
-            v5::Action::DumpsAll => v6::Action::DumpsAll,
-            v5::Action::DumpsCreate => v6::Action::DumpsCreate,
-            v5::Action::Version => v6::Action::Version,
-            v5::Action::KeysAdd => v6::Action::KeysAdd,
-            v5::Action::KeysGet => v6::Action::KeysGet,
-            v5::Action::KeysUpdate => v6::Action::KeysUpdate,
-            v5::Action::KeysDelete => v6::Action::KeysDelete,
-        }
-    }
-}
-
-#[cfg(test)]
-pub(crate) mod test {
-    use std::fs::File;
-    use std::io::BufReader;
-
-    use flate2::bufread::GzDecoder;
-    use meili_snap::insta;
-    use tempfile::TempDir;
-
-    use super::*;
-
-    #[test]
-    fn compat_v5_v6() {
-        let dump = File::open("tests/assets/v5.dump").unwrap();
-        let dir = TempDir::new().unwrap();
-        let mut dump = BufReader::new(dump);
-        let gz = GzDecoder::new(&mut dump);
-        let mut archive = tar::Archive::new(gz);
-        archive.unpack(dir.path()).unwrap();
-
-        let mut dump = v5::V5Reader::open(dir).unwrap().to_v6();
-
-        // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
-
-        // tasks
-        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"41f91d3a94911b2735ec41b07540df5c");
-        assert_eq!(update_files.len(), 22);
-        assert!(update_files[0].is_none()); // the dump creation
-        assert!(update_files[1].is_some()); // the enqueued document addition
-        assert!(update_files[2..].iter().all(|u| u.is_none())); // everything already processed
-
-        // keys
-        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut products = indexes.pop().unwrap();
-        let mut movies = indexes.pop().unwrap();
-        let mut spells = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "products",
-          "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(products.settings().unwrap());
-        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
-
-        // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "movies",
-          "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies.settings().unwrap());
-        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 200);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"e962baafd2fbae4cdd14e876053b0c5a");
-
-        // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "dnd_spells",
-          "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(spells.settings().unwrap());
-        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
-    }
-}

dump/src/reader/mod.rs

@@ -1,694 +0,0 @@
-use std::fs::File;
-use std::io::{BufReader, Read};
-
-use flate2::bufread::GzDecoder;
-use serde::Deserialize;
-use tempfile::TempDir;
-
-use self::compat::v4_to_v5::CompatV4ToV5;
-use self::compat::v5_to_v6::{CompatIndexV5ToV6, CompatV5ToV6};
-use self::v5::V5Reader;
-use self::v6::{V6IndexReader, V6Reader};
-use crate::{Result, Version};
-
-mod compat;
-
-pub(self) mod v1;
-pub(self) mod v2;
-pub(self) mod v3;
-pub(self) mod v4;
-pub(self) mod v5;
-pub(self) mod v6;
-
-pub type Document = serde_json::Map<String, serde_json::Value>;
-pub type UpdateFile = dyn Iterator<Item = Result<Document>>;
-
-pub enum DumpReader {
-    Current(V6Reader),
-    Compat(CompatV5ToV6),
-}
-
-impl DumpReader {
-    pub fn open(dump: impl Read) -> Result<DumpReader> {
-        let path = TempDir::new()?;
-        let mut dump = BufReader::new(dump);
-        let gz = GzDecoder::new(&mut dump);
-        let mut archive = tar::Archive::new(gz);
-        archive.unpack(path.path())?;
-
-        #[derive(Deserialize)]
-        #[serde(rename_all = "camelCase")]
-        struct MetadataVersion {
-            pub dump_version: Version,
-        }
-        let mut meta_file = File::open(path.path().join("metadata.json"))?;
-        let MetadataVersion { dump_version } = serde_json::from_reader(&mut meta_file)?;
-
-        match dump_version {
-            Version::V1 => {
-                Ok(v1::V1Reader::open(path)?.to_v2().to_v3().to_v4().to_v5().to_v6().into())
-            }
-            Version::V2 => Ok(v2::V2Reader::open(path)?.to_v3().to_v4().to_v5().to_v6().into()),
-            Version::V3 => Ok(v3::V3Reader::open(path)?.to_v4().to_v5().to_v6().into()),
-            Version::V4 => Ok(v4::V4Reader::open(path)?.to_v5().to_v6().into()),
-            Version::V5 => Ok(v5::V5Reader::open(path)?.to_v6().into()),
-            Version::V6 => Ok(v6::V6Reader::open(path)?.into()),
-        }
-    }
-
-    pub fn version(&self) -> crate::Version {
-        match self {
-            DumpReader::Current(current) => current.version(),
-            DumpReader::Compat(compat) => compat.version(),
-        }
-    }
-
-    pub fn date(&self) -> Option<time::OffsetDateTime> {
-        match self {
-            DumpReader::Current(current) => current.date(),
-            DumpReader::Compat(compat) => compat.date(),
-        }
-    }
-
-    pub fn instance_uid(&self) -> Result<Option<uuid::Uuid>> {
-        match self {
-            DumpReader::Current(current) => current.instance_uid(),
-            DumpReader::Compat(compat) => compat.instance_uid(),
-        }
-    }
-
-    pub fn indexes(&self) -> Result<Box<dyn Iterator<Item = Result<DumpIndexReader>> + '_>> {
-        match self {
-            DumpReader::Current(current) => {
-                let indexes = Box::new(current.indexes()?.map(|res| res.map(DumpIndexReader::from)))
-                    as Box<dyn Iterator<Item = Result<DumpIndexReader>> + '_>;
-                Ok(indexes)
-            }
-            DumpReader::Compat(compat) => {
-                let indexes = Box::new(compat.indexes()?.map(|res| res.map(DumpIndexReader::from)))
-                    as Box<dyn Iterator<Item = Result<DumpIndexReader>> + '_>;
-                Ok(indexes)
-            }
-        }
-    }
-
-    pub fn tasks(
-        &mut self,
-    ) -> Result<Box<dyn Iterator<Item = Result<(v6::Task, Option<Box<UpdateFile>>)>> + '_>> {
-        match self {
-            DumpReader::Current(current) => Ok(current.tasks()),
-            DumpReader::Compat(compat) => compat.tasks(),
-        }
-    }
-
-    pub fn keys(&mut self) -> Result<Box<dyn Iterator<Item = Result<v6::Key>> + '_>> {
-        match self {
-            DumpReader::Current(current) => Ok(current.keys()),
-            DumpReader::Compat(compat) => compat.keys(),
-        }
-    }
-
-    pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
-        match self {
-            DumpReader::Current(current) => Ok(current.features()),
-            DumpReader::Compat(compat) => compat.features(),
-        }
-    }
-}
-
-impl From<V6Reader> for DumpReader {
-    fn from(value: V6Reader) -> Self {
-        DumpReader::Current(value)
-    }
-}
-
-impl From<CompatV5ToV6> for DumpReader {
-    fn from(value: CompatV5ToV6) -> Self {
-        DumpReader::Compat(value)
-    }
-}
-
-impl From<V5Reader> for DumpReader {
-    fn from(value: V5Reader) -> Self {
-        DumpReader::Compat(value.to_v6())
-    }
-}
-
-impl From<CompatV4ToV5> for DumpReader {
-    fn from(value: CompatV4ToV5) -> Self {
-        DumpReader::Compat(value.to_v6())
-    }
-}
-
-pub enum DumpIndexReader {
-    Current(v6::V6IndexReader),
-    Compat(Box<CompatIndexV5ToV6>),
-}
-
-impl DumpIndexReader {
-    pub fn new_v6(v6: v6::V6IndexReader) -> DumpIndexReader {
-        DumpIndexReader::Current(v6)
-    }
-
-    pub fn metadata(&self) -> &crate::IndexMetadata {
-        match self {
-            DumpIndexReader::Current(v6) => v6.metadata(),
-            DumpIndexReader::Compat(compat) => compat.metadata(),
-        }
-    }
-
-    pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Document>> + '_>> {
-        match self {
-            DumpIndexReader::Current(v6) => v6
-                .documents()
-                .map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
-            DumpIndexReader::Compat(compat) => compat
-                .documents()
-                .map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
-        }
-    }
-
-    pub fn settings(&mut self) -> Result<v6::Settings<v6::Checked>> {
-        match self {
-            DumpIndexReader::Current(v6) => v6.settings(),
-            DumpIndexReader::Compat(compat) => compat.settings(),
-        }
-    }
-}
-
-impl From<V6IndexReader> for DumpIndexReader {
-    fn from(value: V6IndexReader) -> Self {
-        DumpIndexReader::Current(value)
-    }
-}
-
-impl From<CompatIndexV5ToV6> for DumpIndexReader {
-    fn from(value: CompatIndexV5ToV6) -> Self {
-        DumpIndexReader::Compat(Box::new(value))
-    }
-}
-
-#[cfg(test)]
-pub(crate) mod test {
-    use std::fs::File;
-
-    use meili_snap::insta;
-
-    use super::*;
-
-    // TODO: add `features` to tests
-
-    #[test]
-    fn import_dump_v5() {
-        let dump = File::open("tests/assets/v5.dump").unwrap();
-        let mut dump = DumpReader::open(dump).unwrap();
-
-        // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-04 15:55:10.344982459 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
-
-        // tasks
-        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"41f91d3a94911b2735ec41b07540df5c");
-        assert_eq!(update_files.len(), 22);
-        assert!(update_files[0].is_none()); // the dump creation
-        assert!(update_files[1].is_some()); // the enqueued document addition
-        assert!(update_files[2..].iter().all(|u| u.is_none())); // everything already processed
-
-        // keys
-        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"c9d2b467fe2fca0b35580d8a999808fb");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut products = indexes.pop().unwrap();
-        let mut movies = indexes.pop().unwrap();
-        let mut spells = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
-        {
-          "uid": "products",
-          "primaryKey": "sku",
-          "createdAt": "2022-10-04T15:51:35.939396731Z",
-          "updatedAt": "2022-10-04T15:55:01.897325373Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(products.settings().unwrap());
-        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
-
-        // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
-        {
-          "uid": "movies",
-          "primaryKey": "id",
-          "createdAt": "2022-10-04T15:51:35.291992167Z",
-          "updatedAt": "2022-10-04T15:55:10.33561842Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies.settings().unwrap());
-        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 200);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"e962baafd2fbae4cdd14e876053b0c5a");
-
-        // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
-        {
-          "uid": "dnd_spells",
-          "primaryKey": "index",
-          "createdAt": "2022-10-04T15:51:37.381094632Z",
-          "updatedAt": "2022-10-04T15:55:02.394503431Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(spells.settings().unwrap());
-        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
-    }
-
-    #[test]
-    fn import_dump_v4() {
-        let dump = File::open("tests/assets/v4.dump").unwrap();
-        let mut dump = DumpReader::open(dump).unwrap();
-
-        // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-06 12:53:49.131989609 +00:00:00");
-        insta::assert_display_snapshot!(dump.instance_uid().unwrap().unwrap(), @"9e15e977-f2ae-4761-943f-1eaf75fd736d");
-
-        // tasks
-        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"c2445ddd1785528b80f2ba534d3bd00c");
-        assert_eq!(update_files.len(), 10);
-        assert!(update_files[0].is_some()); // the enqueued document addition
-        assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
-
-        // keys
-        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(keys, { "[].uid" => "[uuid]" }), @"d751713988987e9331980363e24189ce");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut products = indexes.pop().unwrap();
-        let mut movies = indexes.pop().unwrap();
-        let mut spells = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
-        {
-          "uid": "products",
-          "primaryKey": "sku",
-          "createdAt": "2022-10-06T12:53:39.360187055Z",
-          "updatedAt": "2022-10-06T12:53:40.603035979Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(products.settings().unwrap());
-        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
-
-        // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
-        {
-          "uid": "movies",
-          "primaryKey": "id",
-          "createdAt": "2022-10-06T12:53:38.710611568Z",
-          "updatedAt": "2022-10-06T12:53:49.785862546Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies.settings().unwrap());
-        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 110);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab");
-
-        // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
-        {
-          "uid": "dnd_spells",
-          "primaryKey": "index",
-          "createdAt": "2022-10-06T12:53:40.831649057Z",
-          "updatedAt": "2022-10-06T12:53:41.116036186Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(spells.settings().unwrap());
-        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
-    }
-
-    #[test]
-    fn import_dump_v3() {
-        let dump = File::open("tests/assets/v3.dump").unwrap();
-        let mut dump = DumpReader::open(dump).unwrap();
-
-        // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-07 11:39:03.709153554 +00:00:00");
-        assert_eq!(dump.instance_uid().unwrap(), None);
-
-        // tasks
-        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"cd12efd308fe3ed226356a727ab42ed3");
-        assert_eq!(update_files.len(), 10);
-        assert!(update_files[0].is_some()); // the enqueued document addition
-        assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
-
-        // keys
-        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"d751713988987e9331980363e24189ce");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut products = indexes.pop().unwrap();
-        let mut movies2 = indexes.pop().unwrap();
-        let mut movies = indexes.pop().unwrap();
-        let mut spells = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "products",
-          "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(products.settings().unwrap());
-        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
-
-        // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "movies",
-          "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies.settings().unwrap());
-        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 110);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
-
-        // movies2
-        insta::assert_json_snapshot!(movies2.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "movies_2",
-          "primaryKey": null,
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies2.settings().unwrap());
-        let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 0);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
-
-        // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "dnd_spells",
-          "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(spells.settings().unwrap());
-        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
-    }
-
-    #[test]
-    fn import_dump_v2() {
-        let dump = File::open("tests/assets/v2.dump").unwrap();
-        let mut dump = DumpReader::open(dump).unwrap();
-
-        // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2022-10-09 20:27:59.904096267 +00:00:00");
-        assert_eq!(dump.instance_uid().unwrap(), None);
-
-        // tasks
-        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"bc616290adfe7d09a624cf6065ca9069");
-        assert_eq!(update_files.len(), 9);
-        assert!(update_files[0].is_some()); // the enqueued document addition
-        assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
-
-        // keys
-        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"d751713988987e9331980363e24189ce");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut products = indexes.pop().unwrap();
-        let mut movies2 = indexes.pop().unwrap();
-        let mut movies = indexes.pop().unwrap();
-        let mut spells = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "products",
-          "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(products.settings().unwrap());
-        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
-
-        // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "movies",
-          "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies.settings().unwrap());
-        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 110);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
-
-        // movies2
-        insta::assert_json_snapshot!(movies2.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "movies_2",
-          "primaryKey": null,
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies2.settings().unwrap());
-        let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 0);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
-
-        // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "dnd_spells",
-          "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(spells.settings().unwrap());
-        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
-    }
-
-    #[test]
-    fn import_dump_v2_from_meilisearch_v0_22_0_issue_3435() {
-        let dump = File::open("tests/assets/v2-v0.22.0.dump").unwrap();
-        let mut dump = DumpReader::open(dump).unwrap();
-
-        // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
-        assert_eq!(dump.instance_uid().unwrap(), None);
-
-        // tasks
-        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"2db37756d8af1fb7623436b76e8956a6");
-        assert_eq!(update_files.len(), 8);
-        assert!(update_files[0..].iter().all(|u| u.is_none())); // everything already processed
-
-        // keys
-        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"d751713988987e9331980363e24189ce");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut products = indexes.pop().unwrap();
-        let mut movies = indexes.pop().unwrap();
-        let mut spells = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "products",
-          "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(products.settings().unwrap());
-        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
-
-        // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "movies",
-          "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies.settings().unwrap());
-        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"0227598af846e574139ee0b80e03a720");
-
-        // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
-        {
-          "uid": "dnd_spells",
-          "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(spells.settings().unwrap());
-        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
-    }
-
-    #[test]
-    fn import_dump_v1() {
-        let dump = File::open("tests/assets/v1.dump").unwrap();
-        let mut dump = DumpReader::open(dump).unwrap();
-
-        // top level infos
-        assert_eq!(dump.date(), None);
-        assert_eq!(dump.instance_uid().unwrap(), None);
-
-        // tasks
-        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"8df6eab075a44b3c1af6b726f9fd9a43");
-        assert_eq!(update_files.len(), 9);
-        assert!(update_files[..].iter().all(|u| u.is_none())); // no update file in dump v1
-
-        // keys
-        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot!(meili_snap::json_string!(keys), @"[]");
-        meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"d751713988987e9331980363e24189ce");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut products = indexes.pop().unwrap();
-        let mut movies = indexes.pop().unwrap();
-        let mut spells = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
-        {
-          "uid": "products",
-          "primaryKey": "sku",
-          "createdAt": "2022-10-02T13:23:39.976870431Z",
-          "updatedAt": "2022-10-02T13:27:54.353262482Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(products.settings().unwrap());
-        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
-
-        // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
-        {
-          "uid": "movies",
-          "primaryKey": "id",
-          "createdAt": "2022-10-02T13:15:29.477512777Z",
-          "updatedAt": "2022-10-02T13:21:12.671204856Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(movies.settings().unwrap());
-        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5");
-
-        // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
-        {
-          "uid": "dnd_spells",
-          "primaryKey": "index",
-          "createdAt": "2022-10-02T13:38:26.358882984Z",
-          "updatedAt": "2022-10-02T13:38:26.385609433Z"
-        }
-        "###);
-
-        insta::assert_json_snapshot!(spells.settings().unwrap());
-        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(documents.len(), 10);
-        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"aa24c0cfc733d66c396237ad44263bed");
-    }
-}

dump/src/reader/snapshots/dump__reader__test__import_dump_v1-11.snap

@@ -1,24 +0,0 @@
----
-source: dump/src/reader/mod.rs
-expression: spells.settings().unwrap()
----
-{
-  "displayedAttributes": [
-    "*"
-  ],
-  "searchableAttributes": [
-    "*"
-  ],
-  "filterableAttributes": [],
-  "sortableAttributes": [],
-  "rankingRules": [
-    "typo",
-    "words",
-    "proximity",
-    "attribute",
-    "exactness"
-  ],
-  "stopWords": [],
-  "synonyms": {},
-  "distinctAttribute": null
-}