Use Debian instead of Alpine in Dockerfile

3505: Csv delimiter r=irevoire a=irevoire

Fixes https://github.com/meilisearch/meilisearch/issues/3442
Closes https://github.com/meilisearch/meilisearch/pull/2803
Specified in https://github.com/meilisearch/specifications/pull/221

This PR is a reimplementation of https://github.com/meilisearch/meilisearch/pull/2803, on the new engine. Thanks for your idea and initial PR `@MixusMinimax;` sorry I couldn’t update/merge your PR. Way too many changes happened on the engine in the meantime.

**Attention to reviewer**; I had to update deserr to implement the support of deserializing `char`s

-------

It introduces four new error messages;
- Invalid value in parameter csvDelimiter: expected a string of one character, but found an empty string
- Invalid value in parameter csvDelimiter: expected a string of one character, but found the following string of 5 characters: doggo
- csv delimiter must be an ascii character. Found: 🍰 
- The Content-Type application/json does not support the use of a csv delimiter. The csv delimiter can only be used with the Content-Type text/csv.

And one error code;
- `invalid_index_csv_delimiter`

The `invalid_content_type` error code is now also used when we encounter the `csvDelimiter` query parameter with a non-csv content type.

Co-authored-by: Tamo <tamo@meilisearch.com>

.github/dependabot.yml

@@ -2,7 +2,6 @@
 
 version: 2
 updates:
-
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/scripts/check-release.sh

@@ -3,7 +3,7 @@
 # check_tag $current_tag $file_tag $file_name
 function check_tag {
   if [[ "$1" != "$2" ]]; then
-      echo "Error: the current tag does not match the version in $3: found $2 - expected $1"
+      echo "Error: the current tag does not match the version in Cargo.toml: found $2 - expected $1"
       ret=1
   fi
 }
@@ -11,12 +11,8 @@ function check_tag {
 ret=0
 current_tag=${GITHUB_REF#'refs/tags/v'}
 
-toml_files='*/Cargo.toml'
-for toml_file in $toml_files;
-do
-    file_tag="$(grep '^version = ' $toml_file | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')"
-    check_tag $current_tag $file_tag $toml_file
-done
+file_tag="$(grep '^version = ' Cargo.toml | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')"
+check_tag $current_tag $file_tag
 
 lock_file='Cargo.lock'
 lock_tag=$(grep -A 1 'name = "meilisearch-auth"' $lock_file | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')

.github/scripts/is-latest-release.sh

@@ -1,127 +1,41 @@
 #!/bin/sh
 
-# Was used in our CIs to publish the latest docker image. Not used anymore, will be used again when v1 and v2 will be out and we will want to maintain multiple stable versions.
-# Returns "true" or "false" (as a string) to be used in the `if` in GHA
+# Used in our CIs to publish the latest Docker image.
 
-# Checks if the current tag should be the latest (in terms of semver and not of release date).
-# Ex: previous tag -> v2.1.1
-#     new tag -> v1.20.3
-#     The new tag (v1.20.3) should NOT be the latest
-#     So it returns "false", the `latest` tag should not be updated for the release v1.20.3 and still need to correspond to v2.1.1
+# Checks if the current tag ($GITHUB_REF) corresponds to the latest release tag on GitHub
+# Returns "true" or "false" (as a string).
 
-# GLOBAL
-GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
+GITHUB_API='https://api.github.com/repos/meilisearch/meilisearch/releases'
+PNAME='meilisearch'
 
 # FUNCTIONS
 
-# semverParseInto and semverLT from https://github.com/cloudflare/semver_bash/blob/master/semver.sh
-
-# usage: semverParseInto version major minor patch special
-# version: the string version
-# major, minor, patch, special: will be assigned by the function
-semverParseInto() {
-    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
-    #MAJOR
-    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
-    #MINOR
-    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
-    #MINOR
-    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
-    #SPECIAL
-    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
-}
-
-# usage: semverLT version1 version2
-semverLT() {
-    local MAJOR_A=0
-    local MINOR_A=0
-    local PATCH_A=0
-    local SPECIAL_A=0
-
-    local MAJOR_B=0
-    local MINOR_B=0
-    local PATCH_B=0
-    local SPECIAL_B=0
-
-    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
-    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
-
-    if [ $MAJOR_A -lt $MAJOR_B ]; then
-        return 0
-    fi
-    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then
-        return 0
-    fi
-    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
-        return 0
-    fi
-    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
-        return 1
-    fi
-    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  != "_" ] ; then
-        return 1
-    fi
-    if [ "_$SPECIAL_A"  != "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
-        return 0
-    fi
-    if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
-        return 0
-    fi
-
-    return 1
-}
-
-# Returns the tag of the latest stable release (in terms of semver and not of release date)
+# Returns the version of the latest stable version of Meilisearch by setting the $latest variable.
 get_latest() {
-    temp_file='temp_file' # temp_file needed because the grep would start before the download is over
-    curl -s 'https://api.github.com/repos/meilisearch/meilisearch/releases' > "$temp_file"
-    releases=$(cat "$temp_file" | \
-        grep -E "tag_name|draft|prerelease" \
-        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
-        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
-        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
+    # temp_file is needed because the grep would start before the download is over
+    temp_file=$(mktemp -q /tmp/$PNAME.XXXXXXXXX)
+    latest_release="$GITHUB_API/latest"
 
-    i=0
-    latest=""
-    current_tag=""
-    for release_info in $releases; do
-        if [ $i -eq 0 ]; then # Checking tag_name
-            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
-                current_tag=$release_info
-            else
-                current_tag=""
-            fi
-            i=1
-        elif [ $i -eq 1 ]; then # Checking draft boolean
-            if [ "$release_info" = "true" ]; then
-                current_tag=""
-            fi
-            i=2
-        elif [ $i -eq 2 ]; then # Checking prerelease boolean
-            if [ "$release_info" = "true" ]; then
-                current_tag=""
-            fi
-            i=0
-            if [ "$current_tag" != "" ]; then # If the current_tag is valid
-                if [ "$latest" = "" ]; then # If there is no latest yet
-                    latest="$current_tag"
-                else
-                    semverLT $current_tag $latest # Comparing latest and the current tag
-                    if [ $? -eq 1 ]; then
-                        latest="$current_tag"
-                    fi
-                fi
-            fi
-        fi
-    done
+    if [ $? -ne 0 ]; then
+        echo "$0: Can't create temp file."
+        exit 1
+    fi
+
+    if [ -z "$GITHUB_PAT" ]; then
+        curl -s "$latest_release" > "$temp_file" || return 1
+    else
+        curl -H "Authorization: token $GITHUB_PAT" -s "$latest_release" > "$temp_file" || return 1
+    fi
+
+    latest="$(cat "$temp_file" | grep '"tag_name":' | cut -d ':' -f2 | tr -d '"' | tr -d ',' | tr -d ' ')"
 
     rm -f "$temp_file"
-    echo $latest
+    return 0
 }
 
 # MAIN
 current_tag="$(echo $GITHUB_REF | tr -d 'refs/tags/')"
-latest="$(get_latest)"
+get_latest
 
 if [ "$current_tag" != "$latest" ]; then
     # The current release tag is not the latest
@@ -130,3 +44,5 @@ else
     # The current release tag is the latest
     echo "true"
 fi
+
+exit 0

.github/uffizzi/Dockerfile

@@ -0,0 +1,48 @@
+# Compile
+FROM    rust:alpine3.16 AS compiler
+
+RUN     apk add -q --update-cache --no-cache build-base openssl-dev
+
+WORKDIR /meilisearch
+
+ARG     COMMIT_SHA
+ARG     COMMIT_DATE
+ARG     GIT_TAG
+ENV     COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE} VERGEN_GIT_SEMVER_LIGHTWEIGHT=${GIT_TAG}
+ENV     RUSTFLAGS="-C target-feature=-crt-static"
+
+COPY    . .
+RUN     set -eux; \
+        apkArch="$(apk --print-arch)"; \
+        if [ "$apkArch" = "aarch64" ]; then \
+            export JEMALLOC_SYS_WITH_LG_PAGE=16; \
+        fi && \
+        cargo build --release
+
+# Run
+FROM    uffizzi/ttyd:alpine
+
+ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+ENV     MEILI_SERVER_PROVIDER docker
+ENV     MEILI_NO_ANALYTICS true
+
+RUN     apk update --quiet \
+        && apk add -q --no-cache libgcc tini curl
+
+# add meilisearch to the `/bin` so you can run it from anywhere and it's easy
+# to find.
+COPY    --from=compiler /meilisearch/target/release/meilisearch /bin/meilisearch
+# To stay compatible with the older version of the container (pre v0.27.0) we're
+# going to symlink the meilisearch binary in the path to `/meilisearch`
+RUN     ln -s /bin/meilisearch /meilisearch
+
+# This directory should hold all the data related to meilisearch so we're going
+# to move our PWD in there.
+# We don't want to put the meilisearch binary
+WORKDIR /meili_data
+
+
+EXPOSE  7700/tcp
+
+ENTRYPOINT ["tini", "--"]
+CMD     ["ttyd", "/bin/zsh"]

.github/uffizzi/docker-compose.uffizzi.yml

@@ -0,0 +1,26 @@
+version: "3"
+
+x-uffizzi:
+  ingress:
+    service: nginx
+    port: 8081
+
+services:
+  meilisearch:
+    image: "${MEILISEARCH_IMAGE}"
+    restart: unless-stopped
+    ports:
+    - "7681:7681"
+    - "7700:7700"
+    deploy:
+      resources:
+        limits:
+          memory: 500M
+  
+  nginx:
+    image: nginx:alpine
+    restart: unless-stopped
+    ports:
+    - "8081:8081"
+    volumes:
+    - ./.github/uffizzi/nginx:/etc/nginx

.github/uffizzi/nginx/nginx.conf

@@ -0,0 +1,28 @@
+
+events {
+  worker_connections  4096;  ## Default: 1024
+}
+
+http {
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        '' close;
+    }
+
+    server {
+        listen 8081;
+
+        location / {
+            proxy_pass http://localhost:7681;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection $connection_upgrade;
+        }
+
+        location /meilisearch/ {
+            # rewrite /meilisearch/(.*) /$1  break;
+            proxy_pass http://localhost:7700/;
+        }
+    }
+}
+

.github/workflows/coverage.yml

@@ -1,33 +0,0 @@
----
-on:
-  workflow_dispatch:
-
-name: Execute code coverage
-
-jobs:
-  nightly-coverage:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
-      - uses: actions-rs/cargo@v1
-        with:
-          command: clean
-      - uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --all-features --no-fail-fast
-        env:
-          CARGO_INCREMENTAL: "0"
-          RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=unwind -Zpanic_abort_tests"
-      - uses: actions-rs/grcov@v0.1
-      - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3
-        with:
-          token: ${{ secrets.CODECOV_TOKEN }}
-          file: ${{ steps.coverage.outputs.report }}
-          yml: ./codecov.yml
-          fail_ci_if_error: true

.github/workflows/create-issue-dependencies.yml

@@ -1,6 +1,7 @@
 name: Create issue to upgrade dependencies
 on:
   schedule:
+    # Run the first of the month, every 3 month
     - cron: '0 0 1 */3 *'
   workflow_dispatch:
 
@@ -15,9 +16,13 @@ jobs:
         github_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         title: Upgrade dependencies
         body: |
-          We need to update the dependencies of the Meilisearch repository, and, if possible, the dependencies of all the core-team repositories that Meilisearch depends on (milli, charabia, heed...).
+          This issue is about updating Meilisearch dependencies:
+          - [ ] Cargo toml dependencies of Meilisearch; but also the main engine-team repositories that Meilisearch depends on (charabia, heed...)
+          - [ ] If new Rust versions have been released, update the Rust version in the Clippy job of this [GitHub Action file](./.github/workflows/rust.yml)
 
-          ⚠️ This issue should only be done at the beginning of the sprint!
+          ⚠️ To avoid last minute bugs, this issue should only be done at the beginning of the sprint!
+
+          The GitHub action dependencies are managed by [Dependabot](./.github/dependabot.yml)
         labels: |
           dependencies
           maintenance

.github/workflows/flaky.yml

@@ -1,15 +1,32 @@
 name: Look for flaky tests
 on:
+  workflow_dispatch:
   schedule:
-    - cron: "0 12 * * FRI" # every friday at 12:00PM
+    - cron: "0 12 * * FRI" # Every Friday at 12:00PM
 
 jobs:
   flaky:
-    runs-on: ubuntu-18.04
-
+    runs-on: ubuntu-latest
+    container:
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
     steps:
     - uses: actions/checkout@v3
+    - name: Install needed dependencies
+      run: |
+        apt-get update && apt-get install -y curl
+        apt-get install build-essential -y
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
     - name: Install cargo-flaky
       run: cargo install cargo-flaky
-    - name: Run cargo flaky 100 times
-      run: cargo flaky -i 100 --release
+    - name: Run cargo flaky in the dumps
+      run: cd dump; cargo flaky -i 100 --release
+    - name: Run cargo flaky in the index-scheduler
+      run: cd index-scheduler; cargo flaky -i 100 --release
+    - name: Run cargo flaky in the auth
+      run: cd meilisearch-auth; cargo flaky -i 100 --release
+    - name: Run cargo flaky in meilisearch
+      run: cd meilisearch; cargo flaky -i 100 --release

.github/workflows/latest-git-tag.yml

@@ -0,0 +1,29 @@
+# Create or update a latest git tag when releasing a stable version of Meilisearch
+name: Update latest git tag
+on:
+  workflow_dispatch:
+  release:
+    types: [released]
+
+jobs:
+  check-version:
+    name: Check the version validity
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Check release validity
+        if: github.event_name == 'release'
+        run: bash .github/scripts/check-release.sh
+
+  update-latest-tag:
+    runs-on: ubuntu-latest
+    needs: check-version
+    steps:
+      - uses: actions/checkout@v3
+      - uses: rickstaa/action-create-tag@v1
+        with:
+          tag: "latest"
+          message: "Latest stable release of Meilisearch"
+          # Move the tag if `latest` already exists
+          force_push_tag: true
+          github_token: ${{ secrets.MEILI_BOT_GH_PAT }}

.github/workflows/manual_benchmarks.yml

@@ -0,0 +1,77 @@
+name: Benchmarks
+
+on:
+  workflow_dispatch:
+    inputs:
+      dataset_name:
+        description: 'The name of the dataset used to benchmark (search_songs, search_wiki, search_geo or indexing)'
+        required: false
+        default: 'search_songs'
+
+env:
+  BENCH_NAME: ${{ github.event.inputs.dataset_name }}
+
+jobs:
+  benchmarks:
+    name: Run and upload benchmarks
+    runs-on: benchmarks
+    timeout-minutes: 4320 # 72h
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      # Set variables
+      - name: Set current branch name
+        shell: bash
+        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
+        id: current_branch
+      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
+        shell: bash
+        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
+        id: normalized_current_branch
+      - name: Set shorter commit SHA
+        shell: bash
+        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
+        id: commit_sha
+      - name: Set file basename with format "dataset_branch_commitSHA"
+        shell: bash
+        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
+        id: file
+
+      # Run benchmarks
+      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
+        run: |
+          cd benchmarks
+          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
+
+      # Generate critcmp files
+      - name: Install critcmp
+        uses: taiki-e/install-action@v2
+        with:
+          tool: critcmp
+      - name: Export cripcmp file
+        run: |
+          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
+
+      # Upload benchmarks
+      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
+        uses: BetaHuhn/do-spaces-action@v2
+        with:
+          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
+          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
+          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
+          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
+          source: ${{ steps.file.outputs.basename }}.json
+          out_dir: critcmp_results
+
+      # Helper
+      - name: 'README: compare with another benchmark'
+        run: |
+          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
+          echo 'How to compare this benchmark with another one?'
+          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
+          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/milestone-workflow.yml

@@ -3,8 +3,8 @@ name: Milestone's workflow
 # /!\ No git flow are handled here
 
 # For each Milestone created (not opened!), and if the release is NOT a patch release (only the patch changed)
-# - the roadmap issue is created, see https://github.com/meilisearch/core-team/blob/main/issue-templates/roadmap-issue.md
-# - the changelog issue is created, see https://github.com/meilisearch/core-team/blob/main/issue-templates/changelog-issue.md
+# - the roadmap issue is created, see https://github.com/meilisearch/engine-team/blob/main/issue-templates/roadmap-issue.md
+# - the changelog issue is created, see https://github.com/meilisearch/engine-team/blob/main/issue-templates/changelog-issue.md
 
 # For each Milestone closed
 # - the `release_version` label is created
@@ -31,8 +31,6 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       is-patch: ${{ steps.check-patch.outputs.is-patch }}
-    env:
-      MILESTONE_VERSION: ${{ github.event.milestone.title }}
     steps:
       - uses: actions/checkout@v3
       - name: Check if this release is a patch release only
@@ -41,10 +39,10 @@ jobs:
           echo version: $MILESTONE_VERSION
           if [[ $MILESTONE_VERSION =~ ^v[0-9]+\.[0-9]+\.0$ ]]; then
             echo 'This is NOT a patch release'
-            echo ::set-output name=is-patch::false
+            echo "is-patch=false" >> $GITHUB_OUTPUT
           elif [[ $MILESTONE_VERSION =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
             echo 'This is a patch release'
-            echo ::set-output name=is-patch::true
+            echo "is-patch=true" >> $GITHUB_OUTPUT
           else
             echo "Not a valid format of release, check the Milestone's title."
             echo 'Should be vX.Y.Z'
@@ -61,7 +59,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Download the issue template
-        run: curl -s https://raw.githubusercontent.com/meilisearch/core-team/main/issue-templates/roadmap-issue.md > $ISSUE_TEMPLATE
+        run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/roadmap-issue.md > $ISSUE_TEMPLATE
       - name: Replace all empty occurrences in the templates
         run: |
           # Replace all <<version>> occurrences
@@ -94,7 +92,7 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Download the issue template
-        run: curl -s https://raw.githubusercontent.com/meilisearch/core-team/main/issue-templates/changelog-issue.md > $ISSUE_TEMPLATE
+        run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/changelog-issue.md > $ISSUE_TEMPLATE
       - name: Replace all empty occurrences in the templates
         run: |
           # Replace all <<version>> occurrences

.github/workflows/publish-binaries.yml

@@ -1,4 +1,5 @@
 on:
+  workflow_dispatch:
   schedule:
     - cron: '0 2 * * *' # Every day at 2:00am
   release:
@@ -17,69 +18,93 @@ jobs:
       # If yes, it means we are publishing an official release.
       # If no, we are releasing a RC, so no need to check the version.
       - name: Check tag format
-        if: github.event_name != 'schedule'
+        if: github.event_name == 'release'
         id: check-tag-format
         run: |
           escaped_tag=$(printf "%q" ${{ github.ref_name }})
 
           if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo ::set-output name=stable::true
+            echo "stable=true" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=stable::false
+            echo "stable=false" >> $GITHUB_OUTPUT
           fi
       - name: Check release validity
-        if: github.event_name != 'schedule' && steps.check-tag-format.outputs.stable == 'true'
+        if: github.event_name == 'release' && steps.check-tag-format.outputs.stable == 'true'
         run: bash .github/scripts/check-release.sh
 
-  publish:
+  publish-linux:
+    name: Publish binary for Linux
+    runs-on: ubuntu-latest
+    needs: check-version
+    container:
+      # Use ubuntu-18.04 to compile with glibc 2.27
+      image: ubuntu:18.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install needed dependencies
+      run: |
+        apt-get update && apt-get install -y curl
+        apt-get install build-essential -y
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
+    - name: Build
+      run: cargo build --release --locked
+    # No need to upload binaries for dry run (cron)
+    - name: Upload binaries to release
+      if: github.event_name == 'release'
+      uses: svenstaro/upload-release-action@2.4.0
+      with:
+        repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
+        file: target/release/meilisearch
+        asset_name: meilisearch-linux-amd64
+        tag: ${{ github.ref }}
+
+  publish-macos-windows:
     name: Publish binary for ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     needs: check-version
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-18.04, macos-latest, windows-latest]
+        os: [macos-12, windows-2022]
         include:
-          - os: ubuntu-18.04
-            artifact_name: meilisearch
-            asset_name: meilisearch-linux-amd64
-          - os: macos-latest
+          - os: macos-12
             artifact_name: meilisearch
             asset_name: meilisearch-macos-amd64
-          - os: windows-latest
+          - os: windows-2022
             artifact_name: meilisearch.exe
             asset_name: meilisearch-windows-amd64.exe
-
     steps:
-    - uses: hecrj/setup-rust-action@master
-      with:
-        rust-version: stable
     - uses: actions/checkout@v3
+    - uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
     - name: Build
       run: cargo build --release --locked
     # No need to upload binaries for dry run (cron)
     - name: Upload binaries to release
-      if: github.event_name != 'schedule'
-      uses: svenstaro/upload-release-action@v1-release
+      if: github.event_name == 'release'
+      uses: svenstaro/upload-release-action@2.4.0
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/release/${{ matrix.artifact_name }}
         asset_name: ${{ matrix.asset_name }}
         tag: ${{ github.ref }}
-        
+
   publish-macos-apple-silicon:
     name: Publish binary for macOS silicon
     runs-on: ${{ matrix.os }}
     needs: check-version
-    continue-on-error: false
     strategy:
       fail-fast: false
       matrix:
         include:
-          - os: macos-latest
+          - os: macos-12
             target: aarch64-apple-darwin
             asset_name: meilisearch-macos-apple-silicon
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -97,8 +122,8 @@ jobs:
           args: --release --target ${{ matrix.target }}
       - name: Upload the binary to release
         # No need to upload binaries for dry run (cron)
-        if: github.event_name != 'schedule'
-        uses: svenstaro/upload-release-action@v1-release
+        if: github.event_name == 'release'
+        uses: svenstaro/upload-release-action@2.4.0
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/${{ matrix.target }}/release/meilisearch
@@ -109,7 +134,6 @@ jobs:
     name: Publish binary for aarch64
     runs-on: ${{ matrix.os }}
     needs: check-version
-    continue-on-error: false
     strategy:
       fail-fast: false
       matrix:
@@ -120,11 +144,9 @@ jobs:
             linker: gcc-aarch64-linux-gnu
             use-cross: true
             asset_name: meilisearch-linux-aarch64
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-
       - name: Installing Rust toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -132,16 +154,13 @@ jobs:
           profile: minimal
           target: ${{ matrix.target }}
           override: true
-
       - name: APT update
         run: |
           sudo apt update
-
       - name: Install target specific tools
         if: matrix.use-cross
         run: |
           sudo apt-get install -y ${{ matrix.linker }}
-
       - name: Configure target aarch64 GNU
         if: matrix.target == 'aarch64-unknown-linux-gnu'
         ## Environment variable is not passed using env:
@@ -153,22 +172,18 @@ jobs:
           echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
           echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
           echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
-          echo RUSTFLAGS="-Clink-arg=-fuse-ld=gold" >> $GITHUB_ENV
-
       - name: Cargo build
         uses: actions-rs/cargo@v1
         with:
           command: build
           use-cross: ${{ matrix.use-cross }}
           args: --release --target ${{ matrix.target }}
-
       - name: List target output files
         run: ls -lR ./target
-
       - name: Upload the binary to release
         # No need to upload binaries for dry run (cron)
-        if: github.event_name != 'schedule'
-        uses: svenstaro/upload-release-action@v1-release
+        if: github.event_name == 'release'
+        uses: svenstaro/upload-release-action@2.4.0
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/${{ matrix.target }}/release/meilisearch

.github/workflows/publish-deb-brew-pkg.yml

@@ -1,4 +1,4 @@
-name: Publish deb pkg to GitHub release & APT repository & Homebrew
+name: Publish to APT repository & Homebrew
 
 on:
   release:
@@ -15,19 +15,27 @@ jobs:
 
   debian:
     name: Publish debian packagge
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     needs: check-version
+    container:
+      # Use ubuntu-18.04 to compile with glibc 2.27
+      image: ubuntu:18.04
     steps:
-    - uses: hecrj/setup-rust-action@master
+    - name: Install needed dependencies
+      run: |
+        apt-get update && apt-get install -y curl
+        apt-get install build-essential -y
+    - uses: actions-rs/toolchain@v1
       with:
-        rust-version: stable
+        toolchain: stable
+        override: true
     - name: Install cargo-deb
       run: cargo install cargo-deb
     - uses: actions/checkout@v3
     - name: Build deb package
-      run: cargo deb -p meilisearch-http -o target/debian/meilisearch.deb
+      run: cargo deb -p meilisearch -o target/debian/meilisearch.deb
     - name: Upload debian pkg to release
-      uses: svenstaro/upload-release-action@v1-release
+      uses: svenstaro/upload-release-action@2.4.0
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/debian/meilisearch.deb
@@ -38,11 +46,11 @@ jobs:
 
   homebrew:
     name: Bump Homebrew formula
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     needs: check-version
     steps:
       - name: Create PR to Homebrew
-        uses: mislav/bump-homebrew-formula-action@v1
+        uses: mislav/bump-homebrew-formula-action@v2
         with:
           formula-name: meilisearch
         env:

.github/workflows/publish-docker-images.yml

@@ -1,10 +1,16 @@
 ---
 on:
-  schedule:
-    - cron: '0 4 * * *' # Every day at 4:00am
   push:
-    tags:
-      - '*'
+    # Will run for every tag pushed except `latest`
+    # When the `latest` git tag is created with this [CI](../latest-git-tag.yml)
+    # we don't need to create a Docker `latest` image again.
+    # The `latest` Docker image push is already done in this CI when releasing a stable version of Meilisearch.
+    tags-ignore:
+      - latest
+  # Both `schedule` and `workflow_dispatch` build the nightly tag
+  schedule:
+    - cron: '0 23 * * *' # Every day at 11:00pm
+  workflow_dispatch:
 
 name: Publish tagged images to Docker Hub
 
@@ -14,27 +20,43 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
-      # Check if the tag has the v<nmumber>.<number>.<number> format. If yes, it means we are publishing an official release.
+      # If we are running a cron or manual job ('schedule' or 'workflow_dispatch' event), it means we are publishing the `nightly` tag, so not considered stable.
+      # If we have pushed a tag, and the tag has the v<nmumber>.<number>.<number> format, it means we are publishing an official release, so considered stable.
       # In this situation, we need to set `output.stable` to create/update the following tags (additionally to the `vX.Y.Z` Docker tag):
       # - a `vX.Y` (without patch version) Docker tag
       # - a `latest` Docker tag
-      - name: Check tag format
-        if: github.event_name != 'schedule'
+      # For any other tag pushed, this is not considered stable.
+      - name: Define if stable and latest release
         id: check-tag-format
+        env:
+          # To avoid request limit with the .github/scripts/is-latest-release.sh script
+          GITHUB_PATH: ${{ secrets.MEILI_BOT_GH_PAT }}
         run: |
           escaped_tag=$(printf "%q" ${{ github.ref_name }})
+          echo "latest=false" >> $GITHUB_OUTPUT
 
-          if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
-            echo ::set-output name=stable::true
+          if [[ ${{ github.event_name }} != 'push' ]]; then
+            echo "stable=false" >> $GITHUB_OUTPUT
+          elif [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "stable=true" >> $GITHUB_OUTPUT
+            echo "latest=$(sh .github/scripts/is-latest-release.sh)" >> $GITHUB_OUTPUT
           else
-            echo ::set-output name=stable::false
+            echo "stable=false" >> $GITHUB_OUTPUT
           fi
 
-      # Check only the validity of the tag for official releases (not for pre-releases or other tags)
+      # Check only the validity of the tag for stable releases (not for pre-releases or other tags)
       - name: Check release validity
-        if: github.event_name != 'schedule' && steps.check-tag-format.outputs.stable == 'true'
+        if: steps.check-tag-format.outputs.stable == 'true'
         run: bash .github/scripts/check-release.sh
 
+      - name: Set build-args for Docker buildx
+        id: build-metadata
+        run: |
+          # Extract commit date
+          commit_date=$(git show -s --format=%cd --date=iso-strict ${{ github.sha }})
+
+          echo "date=$commit_date" >> $GITHUB_OUTPUT
+
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2
 
@@ -42,7 +64,6 @@ jobs:
         uses: docker/setup-buildx-action@v2
 
       - name: Login to Docker Hub
-        if: github.event_name != 'schedule'
         uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
@@ -53,25 +74,30 @@ jobs:
         uses: docker/metadata-action@v4
         with:
           images: getmeili/meilisearch
-          # The latest and `vX.Y` tags are only pushed for the official Meilisearch releases
-          # See https://github.com/docker/metadata-action#latest-tag
+          # Prevent `latest` to be updated for each new tag pushed.
+          # We need latest and `vX.Y` tags to only be pushed for the stable Meilisearch releases.
           flavor: latest=false
           tags: |
             type=ref,event=tag
+            type=raw,value=nightly,enable=${{ github.event_name != 'push' }}
             type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
-            type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
+            type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
 
       - name: Build and push
-        uses: docker/build-push-action@v3
+        uses: docker/build-push-action@v4
         with:
-          # We do not push tags for the cron jobs, this is only for test purposes
-          push: ${{ github.event_name != 'schedule' }}
+          push: true
           platforms: linux/amd64,linux/arm64
           tags: ${{ steps.meta.outputs.tags }}
+          build-args: |
+            COMMIT_SHA=${{ github.sha }}
+            COMMIT_DATE=${{ steps.build-metadata.outputs.date }}
+            GIT_TAG=${{ github.ref_name }}
 
       # /!\ Don't touch this without checking with Cloud team
       - name: Send CI information to Cloud team
-        if: github.event_name != 'schedule'
+        # Do not send if nightly build (i.e. 'schedule' or 'workflow_dispatch' event)
+        if: github.event_name == 'push'
         uses: peter-evans/repository-dispatch@v2
         with:
           token: ${{ secrets.MEILI_BOT_GH_PAT }}

.github/workflows/push_benchmarks_indexing.yml

@@ -0,0 +1,79 @@
+name: Benchmarks indexing (push)
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
+  BENCH_NAME: "indexing"
+
+jobs:
+  benchmarks:
+    name: Run and upload benchmarks
+    runs-on: benchmarks
+    timeout-minutes: 4320 # 72h
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      # Set variables
+      - name: Set current branch name
+        shell: bash
+        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
+        id: current_branch
+      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
+        shell: bash
+        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
+        id: normalized_current_branch
+      - name: Set shorter commit SHA
+        shell: bash
+        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
+        id: commit_sha
+      - name: Set file basename with format "dataset_branch_commitSHA"
+        shell: bash
+        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
+        id: file
+
+      # Run benchmarks
+      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
+        run: |
+          cd benchmarks
+          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
+
+      # Generate critcmp files
+      - name: Install critcmp
+        uses: taiki-e/install-action@v2
+        with:
+          tool: critcmp
+      - name: Export cripcmp file
+        run: |
+          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
+
+      # Upload benchmarks
+      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
+        uses: BetaHuhn/do-spaces-action@v2
+        with:
+          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
+          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
+          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
+          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
+          source: ${{ steps.file.outputs.basename }}.json
+          out_dir: critcmp_results
+
+      # Upload benchmarks to influxdb
+      - name: Upload ${{ steps.file.outputs.basename }}.json to influxDB
+        run: telegraf --config https://eu-central-1-1.aws.cloud2.influxdata.com/api/v2/telegrafs/08b52e34a370b000 --once --debug
+
+      # Helper
+      - name: 'README: compare with another benchmark'
+        run: |
+          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
+          echo 'How to compare this benchmark with another one?'
+          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
+          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/push_benchmarks_search_geo.yml

@@ -0,0 +1,78 @@
+name: Benchmarks search geo (push)
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  BENCH_NAME: "search_geo"
+  INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
+
+jobs:
+  benchmarks:
+    name: Run and upload benchmarks
+    runs-on: benchmarks
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      # Set variables
+      - name: Set current branch name
+        shell: bash
+        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
+        id: current_branch
+      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
+        shell: bash
+        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
+        id: normalized_current_branch
+      - name: Set shorter commit SHA
+        shell: bash
+        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
+        id: commit_sha
+      - name: Set file basename with format "dataset_branch_commitSHA"
+        shell: bash
+        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
+        id: file
+
+      # Run benchmarks
+      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
+        run: |
+          cd benchmarks
+          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
+
+      # Generate critcmp files
+      - name: Install critcmp
+        uses: taiki-e/install-action@v2
+        with:
+          tool: critcmp
+      - name: Export cripcmp file
+        run: |
+          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
+
+      # Upload benchmarks
+      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
+        uses: BetaHuhn/do-spaces-action@v2
+        with:
+          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
+          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
+          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
+          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
+          source: ${{ steps.file.outputs.basename }}.json
+          out_dir: critcmp_results
+
+      # Upload benchmarks to influxdb
+      - name: Upload ${{ steps.file.outputs.basename }}.json to influxDB
+        run: telegraf --config https://eu-central-1-1.aws.cloud2.influxdata.com/api/v2/telegrafs/08b52e34a370b000 --once --debug
+
+      # Helper
+      - name: 'README: compare with another benchmark'
+        run: |
+          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
+          echo 'How to compare this benchmark with another one?'
+          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
+          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/push_benchmarks_search_songs.yml

@@ -0,0 +1,78 @@
+name: Benchmarks search songs (push)
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  BENCH_NAME: "search_songs"
+  INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
+
+jobs:
+  benchmarks:
+    name: Run and upload benchmarks
+    runs-on: benchmarks
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      # Set variables
+      - name: Set current branch name
+        shell: bash
+        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
+        id: current_branch
+      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
+        shell: bash
+        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
+        id: normalized_current_branch
+      - name: Set shorter commit SHA
+        shell: bash
+        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
+        id: commit_sha
+      - name: Set file basename with format "dataset_branch_commitSHA"
+        shell: bash
+        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
+        id: file
+
+      # Run benchmarks
+      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
+        run: |
+          cd benchmarks
+          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
+
+      # Generate critcmp files
+      - name: Install critcmp
+        uses: taiki-e/install-action@v2
+        with:
+          tool: critcmp
+      - name: Export cripcmp file
+        run: |
+          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
+
+      # Upload benchmarks
+      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
+        uses: BetaHuhn/do-spaces-action@v2
+        with:
+          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
+          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
+          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
+          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
+          source: ${{ steps.file.outputs.basename }}.json
+          out_dir: critcmp_results
+
+      # Upload benchmarks to influxdb
+      - name: Upload ${{ steps.file.outputs.basename }}.json to influxDB
+        run: telegraf --config https://eu-central-1-1.aws.cloud2.influxdata.com/api/v2/telegrafs/08b52e34a370b000 --once --debug
+
+      # Helper
+      - name: 'README: compare with another benchmark'
+        run: |
+          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
+          echo 'How to compare this benchmark with another one?'
+          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
+          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/push_benchmarks_search_wiki.yml

@@ -0,0 +1,78 @@
+name: Benchmarks search wikipedia articles (push)
+
+on:
+  push:
+    branches:
+      - main
+
+env:
+  BENCH_NAME: "search_wiki"
+  INFLUX_TOKEN: ${{ secrets.INFLUX_TOKEN }}
+
+jobs:
+  benchmarks:
+    name: Run and upload benchmarks
+    runs-on: benchmarks
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      # Set variables
+      - name: Set current branch name
+        shell: bash
+        run: echo "name=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_OUTPUT
+        id: current_branch
+      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
+        shell: bash
+        run: echo "name=$(echo ${GITHUB_REF#refs/heads/} | tr '/' '_')" >> $GITHUB_OUTPUT
+        id: normalized_current_branch
+      - name: Set shorter commit SHA
+        shell: bash
+        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
+        id: commit_sha
+      - name: Set file basename with format "dataset_branch_commitSHA"
+        shell: bash
+        run: echo "basename=$(echo ${BENCH_NAME}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
+        id: file
+
+      # Run benchmarks
+      - name: Run benchmarks - Dataset ${BENCH_NAME} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
+        run: |
+          cd benchmarks
+          cargo bench --bench ${BENCH_NAME} -- --save-baseline ${{ steps.file.outputs.basename }}
+
+      # Generate critcmp files
+      - name: Install critcmp
+        uses: taiki-e/install-action@v2
+        with:
+          tool: critcmp
+      - name: Export cripcmp file
+        run: |
+          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
+
+      # Upload benchmarks
+      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
+        uses: BetaHuhn/do-spaces-action@v2
+        with:
+          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
+          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
+          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
+          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
+          source: ${{ steps.file.outputs.basename }}.json
+          out_dir: critcmp_results
+
+      # Upload benchmarks to influxdb
+      - name: Upload ${{ steps.file.outputs.basename }}.json to influxDB
+        run: telegraf --config https://eu-central-1-1.aws.cloud2.influxdata.com/api/v2/telegrafs/08b52e34a370b000 --once --debug
+
+      # Helper
+      - name: 'README: compare with another benchmark'
+        run: |
+          echo "${{ steps.file.outputs.basename }}.json has just been pushed."
+          echo 'How to compare this benchmark with another one?'
+          echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
+          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/rust.yml

@@ -2,6 +2,9 @@ name: Rust
 
 on:
   workflow_dispatch:
+  schedule:
+    # Everyday at 5:00am
+    - cron: '0 5 * * *'
   pull_request:
   push:
     # trying and staging branches are for Bors config
@@ -15,78 +18,128 @@ env:
   RUSTFLAGS: "-D warnings"
 
 jobs:
-  tests:
+  test-linux:
+    name: Tests on ubuntu-18.04
+    runs-on: ubuntu-latest
+    container:
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install needed dependencies
+      run: |
+        apt-get update && apt-get install -y curl
+        apt-get install build-essential -y
+    - name: Run test with Rust stable
+      if: github.event_name != 'schedule'
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
+    - name: Run test with Rust nightly
+      if: github.event_name == 'schedule'
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: nightly
+        override: true
+    # Disable cache due to disk space issues with Windows workers in CI
+    # - name: Cache dependencies
+    #   uses: Swatinem/rust-cache@v2.2.0
+    - name: Run cargo check without any default features
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --locked --release --no-default-features --all
+    - name: Run cargo test
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --locked --release --all
+
+  test-others:
     name: Tests on ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-18.04, macos-latest, windows-latest]
+        os: [macos-12, windows-2022]
     steps:
     - uses: actions/checkout@v3
-    - name: Cache dependencies
-      uses: Swatinem/rust-cache@v2.0.0
+#     - name: Cache dependencies
+#       uses: Swatinem/rust-cache@v2.2.0
     - name: Run cargo check without any default features
       uses: actions-rs/cargo@v1
       with:
         command: build
-        args: --locked --release --no-default-features
+        args: --locked --release --no-default-features --all
     - name: Run cargo test
       uses: actions-rs/cargo@v1
       with:
         command: test
-        args: --locked --release
+        args: --locked --release --all
 
   # We run tests in debug also, to make sure that the debug_assertions are hit
   test-debug:
     name: Run tests in debug
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
+    container:
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
     steps:
       - uses: actions/checkout@v3
+      - name: Install needed dependencies
+        run: |
+          apt-get update && apt-get install -y curl
+          apt-get install build-essential -y
       - uses: actions-rs/toolchain@v1
         with:
-          profile: minimal
           toolchain: stable
           override: true
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.0.0
+      # - name: Cache dependencies
+      #   uses: Swatinem/rust-cache@v2.2.0
       - name: Run tests in debug
         uses: actions-rs/cargo@v1
         with:
           command: test
-          args: --locked
+          args: --locked --all
 
   clippy:
     name: Run Clippy
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: 1.67.0
           override: true
           components: clippy
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.0.0
+      # - name: Cache dependencies
+      #   uses: Swatinem/rust-cache@v2.2.0
       - name: Run cargo clippy
         uses: actions-rs/cargo@v1
         with:
           command: clippy
-          args: --all-targets -- --deny warnings
+          # allow unlined_format_args https://github.com/rust-lang/rust-clippy/issues/10087
+          args: --all-targets -- --deny warnings --allow clippy::uninlined_format_args
 
   fmt:
     name: Run Rustfmt
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: stable
+          toolchain: nightly
           override: true
           components: rustfmt
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.0.0
+      # - name: Cache dependencies
+      #   uses: Swatinem/rust-cache@v2.2.0
       - name: Run cargo fmt
-        run: cargo fmt --all -- --check
+        # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
+        # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate
+        # we are going to create an empty file where rustfmt expects it.
+        run: |
+          echo -ne "\n" > benchmarks/benches/datasets_paths.rs
+          cargo fmt --all -- --check

.github/workflows/uffizzi-build.yml

@@ -0,0 +1,100 @@
+name: Uffizzi - Build PR Image
+on:
+  pull_request:
+    types: [opened,synchronize,reopened,closed]
+
+jobs:
+  build-meilisearch:
+    name: Build and push `meilisearch`
+    runs-on: ubuntu-latest
+    outputs:
+      tags: ${{ steps.meta.outputs.tags }}
+    if: ${{ github.event.action != 'closed' }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Generate UUID image name
+        id: uuid
+        run: echo "UUID_TAG=$(uuidgen)" >> $GITHUB_ENV
+
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: registry.uffizzi.com/${{ env.UUID_TAG }}
+          tags: | 
+            type=raw,value=60d
+
+      - name: Build Image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./
+          file: .github/uffizzi/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  render-compose-file:
+    name: Render Docker Compose File
+    # Pass output of this workflow to another triggered by `workflow_run` event.
+    runs-on: ubuntu-latest
+    needs: 
+      - build-meilisearch
+    outputs:
+      compose-file-cache-key:  ${{ env.COMPOSE_FILE_HASH }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Render Compose File
+        run: |
+          MEILISEARCH_IMAGE=$(echo ${{ needs.build-meilisearch.outputs.tags }})
+          export MEILISEARCH_IMAGE
+          # Render simple template from environment variables.
+          envsubst < .github/uffizzi/docker-compose.uffizzi.yml > docker-compose.rendered.yml
+          cat docker-compose.rendered.yml
+      - name: Upload Rendered Compose File as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec 
+          path: docker-compose.rendered.yml
+          retention-days: 2
+      - name: Serialize PR Event to File
+        run:  |
+          cat << EOF > event.json
+          ${{ toJSON(github.event) }} 
+          
+          EOF
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2
+
+  delete-preview:
+    name: Call for Preview Deletion
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'closed' }}
+    steps:
+      # If this PR is closing, we will not render a compose file nor pass it to the next workflow.
+      - name: Serialize PR Event to File
+        run:  |
+          cat << EOF > event.json
+          ${{ toJSON(github.event) }} 
+          
+          EOF
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2

.github/workflows/uffizzi-preview-deploy.yml

@@ -0,0 +1,103 @@
+name: Uffizzi - Deploy Preview
+
+on:
+  workflow_run:
+    workflows:
+      - "Uffizzi - Build PR Image"
+    types:
+      - completed
+
+jobs:
+  cache-compose-file:
+    name: Cache Compose File
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    outputs:
+      compose-file-cache-key: ${{ env.COMPOSE_FILE_HASH }}
+      pr-number: ${{ env.PR_NUMBER }}
+      expected-url: ${{ env.EXPECTED_URL }}
+    steps:
+      - name: 'Download artifacts'
+        # Fetch output (zip archive) from the workflow run that triggered this workflow.
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "preview-spec"
+            })[0];
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/preview-spec.zip`, Buffer.from(download.data));
+      
+      - name: 'Unzip artifact'
+        run: unzip preview-spec.zip
+      
+      - name: Read Event into ENV
+        run: |
+          echo 'EVENT_JSON<<EOF' >> $GITHUB_ENV
+          cat event.json >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      
+      - name: Hash Rendered Compose File
+        id: hash
+        # If the previous workflow was triggered by a PR close event, we will not have a compose file artifact.
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.rendered.yml | awk '{ print $1 }')" >> $GITHUB_ENV
+      
+      - name: Cache Rendered Compose File
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        uses: actions/cache@v3
+        with:
+          path: docker-compose.rendered.yml
+          key: ${{ env.COMPOSE_FILE_HASH }}
+
+      - name: Read PR Number From Event Object
+        id: pr
+        run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV
+      
+      - name: DEBUG - Print Job Outputs
+        if: ${{ runner.debug }}
+        run: |
+          echo "PR number: ${{ env.PR_NUMBER }}"
+          echo "Compose file hash: ${{ env.COMPOSE_FILE_HASH }}"
+          cat event.json
+
+      - name: Add expected URL env var
+        if: ${{ runner.debug }}
+        run: |
+          REPO=$(echo ${{ github.repository }} | sed 's/\./+/g')
+          echo "EXPECTED_URL=${{ inputs.server }}/github.com/$REPO/pull/${{ env.PR_NUMBER }}" >> $GITHUB_ENV
+
+  deploy-uffizzi-preview:
+    name: Use Remote Workflow to Preview on Uffizzi
+    needs:
+      - cache-compose-file
+    uses: UffizziCloud/preview-action/.github/workflows/reusable.yaml@desc
+    with:
+      # If this workflow was triggered by a PR close event, cache-key will be an empty string
+      # and this reusable workflow will delete the preview deployment.
+      compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }}
+      compose-file-cache-path: docker-compose.rendered.yml
+      server: https://app.uffizzi.com
+      pr-number: ${{ needs.cache-compose-file.outputs.pr-number }}
+      description: |
+        The meilisearch preview environment contains a web terminal from where you can run the
+        `meilisearch` command. You should be able to access this instance of meilisearch running in
+        the preview from the link Meilisearch Endpoint link given below.
+         
+        Web Terminal Endpoint : ${{ needs.cache-compose-file.outputs.expected-url }}
+        Meilisearch Endpoint : ${{ needs.cache-compose-file.outputs.expected-url }}/meilisearch
+    permissions:
+      contents: read
+      pull-requests: write
+      id-token: write

.github/workflows/update-cargo-toml-version.yml

@@ -13,10 +13,9 @@ env:
   GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
 
 jobs:
-
   update-version-cargo-toml:
     name: Update version in Cargo.toml files
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
@@ -30,7 +29,7 @@ jobs:
         run: |
           raw_new_version=$(echo $NEW_VERSION | cut -d 'v' -f 2)
           new_string="version = \"$raw_new_version\""
-          sd '^version = "\d+.\d+.\w+"$' "$new_string" */Cargo.toml
+          sd '^version = "\d+.\d+.\w+"$' "$new_string" Cargo.toml
       - name: Build Meilisearch to update Cargo.lock
         run: cargo build
       - name: Commit and push the changes to the ${{ env.NEW_BRANCH }} branch
@@ -44,4 +43,5 @@ jobs:
             --title "Update version for the next release ($NEW_VERSION) in Cargo.toml files" \
             --body '⚠️ This PR is automatically generated. Check the new version is the expected one before merging.' \
             --label 'skip changelog' \
-            --milestone $NEW_VERSION
+            --milestone $NEW_VERSION \
+            --base $GITHUB_REF_NAME

.gitignore

@@ -1,3 +1,5 @@
+.idea/
+.vscode/
 /target
 **/*.csv
 **/*.json_lines
@@ -8,9 +10,11 @@
 /snapshots
 /dumps
 
-
 # Snapshots
 ## ... large
 *.full.snap
 ##  ... unreviewed
 *.snap.new
+
+# Fuzzcheck data for the facet indexing fuzz test
+milli/fuzz/update::facet::incremental::fuzz::fuzz/

CONTRIBUTING.md

@@ -10,24 +10,12 @@ If Meilisearch does not offer optimized support for your language, please consid
 
 ## Table of Contents
 
-- [Hacktoberfest 2022](#hacktoberfest-2022)
 - [Assumptions](#assumptions)
 - [How to Contribute](#how-to-contribute)
 - [Development Workflow](#development-workflow)
 - [Git Guidelines](#git-guidelines)
 - [Release Process (for internal team only)](#release-process-for-internal-team-only)
 
-## Hacktoberfest 2022
-
-It's [Hacktoberfest month](https://hacktoberfest.com)! 🥳
-
-Thanks so much for participating with Meilisearch this year!
-1. We will follow the quality standards set by the organizers of Hacktoberfest (see detail on their [website](https://hacktoberfest.com/participation/#spam)). Our reviewers will not consider any PR that doesn’t match that standard.
-2. PRs reviews will take place from Monday to Thursday, during usual working hours, CEST time. If you submit outside of these hours, there’s no need to panic; we will get around to your contribution.
-3. There will be no issue assignment as we don’t want people to ask to be assigned specific issues and never return, discouraging the volunteer contributors from opening a PR to fix this issue. We take the liberty to choose the PR that best fixes the issue, so we encourage you to get to it as soon as possible and do your best!
-
-You can check out the longer, more complete guideline documentation [here](https://github.com/meilisearch/.github/blob/main/Hacktoberfest_2022_contributors_guidelines.md).
-
 ## Assumptions
 
 1. **You're familiar with [GitHub](https://github.com) and the [Pull Requests](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)(PR) workflow.**
@@ -64,6 +52,23 @@ cargo test
 
 This command will be triggered to each PR as a requirement for merging it.
 
+#### Snapshot-based tests
+
+We are using [insta](https://insta.rs) to perform snapshot-based testing.
+We recommend using the insta tooling (such as `cargo-insta`) to update the snapshots if they change following a PR.
+
+New tests should use insta where possible rather than manual `assert` statements.
+
+Furthermore, we provide some macros on top of insta, notably a way to use snapshot hashes instead of inline snapshots, saving a lot of space in the repository.
+
+To effectively debug snapshot-based hashes, we recommend you export the `MEILI_TEST_FULL_SNAPS` environment variable so that snapshot are fully created locally:
+
+```
+export MEILI_TEST_FULL_SNAPS=true # add this to your .bashrc, .zshrc, ...
+```
+
+#### Test troubleshooting
+
 If you get a "Too many open files" error you might want to increase the open file limit using this command:
 
 ```bash
@@ -109,7 +114,35 @@ _[Read more about this](https://github.com/meilisearch/integration-guides/blob/m
 
 ### How to Publish a new Release
 
-The full Meilisearch release process is described in [this guide](https://github.com/meilisearch/core-team/blob/main/resources/meilisearch-release.md). Please follow it carefully before doing any release.
+The full Meilisearch release process is described in [this guide](https://github.com/meilisearch/engine-team/blob/main/resources/meilisearch-release.md). Please follow it carefully before doing any release.
+
+### How to publish a prototype
+
+Depending on the developed feature, you might need to provide a prototyped version of Meilisearch to make it easier to test by the users.
+
+The prototype name must follow this convention: `prototype-X-Y` where
+- `X` is the feature name formatted in `kebab-case`. It should not end with a single number.
+- `Y` is the version of the prototype, starting from `0`.
+
+✅ Example: `prototype-auto-resize-0`. </br>
+❌ Bad example: `auto-resize-0`: lacks the `prototype` prefix. </br>
+❌ Bad example: `prototype-auto-resize`: lacks the version suffix. </br>
+❌ Bad example: `prototype-auto-resize-0-0`: feature name ends with a single number.
+
+Steps to create a prototype:
+
+1. In your terminal, go to the last commit of your branch (the one you want to provide as a prototype).
+2. Create a tag following the convention: `git tag prototype-X-Y`
+3. Run Meilisearch and check that its launch summary features a line: `Prototype: prototype-X-Y` (you may need to switch branches and back after tagging for this to work).
+3. Push the tag: `git push origin prototype-X-Y`
+4. Check the [Docker CI](https://github.com/meilisearch/meilisearch/actions/workflows/publish-docker-images.yml) is now running.
+
+🐳 Once the CI has finished to run (~1h30), a Docker image named `prototype-X-Y` will be available on [DockerHub](https://hub.docker.com/repository/docker/getmeili/meilisearch/general). People can use it with the following command: `docker run -p 7700:7700 -v $(pwd)/meili_data:/meili_data getmeili/meilisearch:prototype-X-Y`. <br>
+More information about [how to run Meilisearch with Docker](https://docs.meilisearch.com/learn/cookbooks/docker.html#download-meilisearch-with-docker).
+
+⚙️ However, no binaries will be created. If the users do not use Docker, they can go to the `prototype-X-Y` tag in the Meilisearch repository and compile from the source code.
+
+⚠️ When sharing a prototype with users, remind them to not use it in production. Prototypes are solely for test purposes.
 
 ### Release assets
 

Cargo.toml

@@ -1,7 +1,7 @@
 [workspace]
 resolver = "2"
 members = [
-    "meilisearch-http",
+    "meilisearch",
     "meilisearch-types",
     "meilisearch-auth",
     "meili-snap",
@@ -9,13 +9,50 @@ members = [
     "dump",
     "file-store",
     "permissive-json-pointer",
+    "milli",
+    "filter-parser",
+    "flatten-serde-json",
+    "json-depth-checker",
+    "benchmarks"
 ]
 
+[workspace.package]
+version = "1.0.0"
+authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
+description = "Meilisearch HTTP server"
+homepage = "https://meilisearch.com"
+readme = "README.md"
+edition = "2021"
+license = "MIT"
+
 [profile.release]
 codegen-units = 1
 
 [profile.dev.package.flate2]
 opt-level = 3
 
-[profile.dev.package.milli]
+[profile.dev.package.grenad]
+opt-level = 3
+[profile.dev.package.roaring]
+opt-level = 3
+
+[profile.dev.package.lindera-ipadic-builder]
+opt-level = 3
+[profile.dev.package.encoding]
+opt-level = 3
+[profile.dev.package.yada]
+opt-level = 3
+
+[profile.release.package.lindera-ipadic-builder]
+opt-level = 3
+[profile.release.package.encoding]
+opt-level = 3
+[profile.release.package.yada]
+opt-level = 3
+
+[profile.bench.package.lindera-ipadic-builder]
+opt-level = 3
+[profile.bench.package.encoding]
+opt-level = 3
+[profile.bench.package.yada]
 opt-level = 3

Dockerfile

@@ -1,31 +1,30 @@
 # Compile
-FROM    rust:alpine3.16 AS compiler
-
-RUN     apk add -q --update-cache --no-cache build-base openssl-dev
+FROM    rust:bullseye AS compiler
 
 WORKDIR /meilisearch
 
 ARG     COMMIT_SHA
 ARG     COMMIT_DATE
-ENV     COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE}
+ARG     GIT_TAG
+ENV     VERGEN_GIT_SHA=${COMMIT_SHA} VERGEN_GIT_COMMIT_TIMESTAMP=${COMMIT_DATE} VERGEN_GIT_SEMVER_LIGHTWEIGHT=${GIT_TAG}
 ENV     RUSTFLAGS="-C target-feature=-crt-static"
 
 COPY    . .
 RUN     set -eux; \
-        apkArch="$(apk --print-arch)"; \
-        if [ "$apkArch" = "aarch64" ]; then \
+        arch="$(dpkg --print-architecture)"; \
+        if [ "$arch" = "aarch64" ]; then \
             export JEMALLOC_SYS_WITH_LG_PAGE=16; \
         fi && \
         cargo build --release
 
 # Run
-FROM    alpine:3.16
+FROM    debian:11.6
 
 ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
 ENV     MEILI_SERVER_PROVIDER docker
 
-RUN     apk update --quiet \
-        && apk add -q --no-cache libgcc tini curl
+RUN     apt update -q \
+        && apt install -q -y tini
 
 # add meilisearch to the `/bin` so you can run it from anywhere and it's easy
 # to find.

README.md

@@ -9,11 +9,10 @@
   <a href="https://blog.meilisearch.com">Blog</a> |
   <a href="https://docs.meilisearch.com">Documentation</a> |
   <a href="https://docs.meilisearch.com/faq/">FAQ</a> |
-  <a href="https://slack.meilisearch.com">Slack</a>
+  <a href="https://discord.meilisearch.com">Discord</a>
 </h4>
 
 <p align="center">
-  <a href="https://github.com/meilisearch/meilisearch/actions"><img src="https://github.com/meilisearch/meilisearch/workflows/Cargo%20test/badge.svg" alt="Build Status"></a>
   <a href="https://deps.rs/repo/github/meilisearch/meilisearch"><img src="https://deps.rs/repo/github/meilisearch/meilisearch/status.svg" alt="Dependency status"></a>
   <a href="https://github.com/meilisearch/meilisearch/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
   <a href="https://app.bors.tech/repositories/26457"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
@@ -34,14 +33,6 @@ Meilisearch helps you shape a delightful search experience in a snap, offering f
 
 🔥 [**Try it!**](https://where2watch.meilisearch.com/) 🔥
 
-## 🎃 Hacktoberfest
-
-It’s Hacktoberfest 2022 @Meilisearch
-
-[Hacktoberfest](https://hacktoberfest.com/) is a celebration of the open-source community. This year, and for the third time in a row, Meilisearch is participating in this fantastic event.
-
-You’d like to contribute? Don’t hesitate to check out our [contributing guidelines](./CONTRIBUTING.md).
-
 ## ✨ Features
 
 - **Search-as-you-type:** find search results in less than 50 milliseconds
@@ -69,7 +60,7 @@ You may also want to check out [Meilisearch 101](https://docs.meilisearch.com/le
 
 ## ☁️ Meilisearch cloud
 
-Join the closed beta for Meilisearch cloud by filling out [this form](https://meilisearch.typeform.com/to/VI2cI2rv).
+Let us manage your infrastructure so you can focus on integrating a great search experience. Try [Meilisearch Cloud](https://meilisearch.com/pricing) today.
 
 ## 🧰 SDKs & integration tools
 
@@ -77,7 +68,7 @@ Install one of our SDKs in your project for seamless integration between Meilise
 
 Take a look at the complete [Meilisearch integration list](https://docs.meilisearch.com/learn/what_is_meilisearch/sdks.html).
 
-![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)
+[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://docs.meilisearch.com/learn/what_is_meilisearch/sdks.html)
 
 ## ⚙️ Advanced usage
 
@@ -105,7 +96,19 @@ Meilisearch is a search engine created by [Meili](https://www.welcometothejungle
 
 - For feature requests, please visit our [product repository](https://github.com/meilisearch/product/discussions)
 - Found a bug? Open an [issue](https://github.com/meilisearch/meilisearch/issues)!
-- Want to be part of our Slack community? [Join us!](https://slack.meilisearch.com/)
+- Want to be part of our Discord community? [Join us!](https://discord.gg/meilisearch)
 - For everything else, please check [this page listing some of the other places where you can find us](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html)
 
 Thank you for your support!
+
+## 👩‍💻 Contributing
+
+Meilisearch is, and will always be, open-source! If you want to contribute to the project, please take a look at [our contribution guidelines](CONTRIBUTING.md).
+
+## 📦 Versioning
+
+Meilisearch releases and their associated binaries are available [in this GitHub page](https://github.com/meilisearch/meilisearch/releases).
+
+The binaries are versioned following [SemVer conventions](https://semver.org/). To know more, read our [versioning policy](https://github.com/meilisearch/engine-team/blob/main/resources/versioning-policy.md).
+
+Differently from the binaries, crates in this repository are not currently available on [crates.io](https://crates.io/) and do not follow [SemVer conventions](https://semver.org).

assets/milli-logo.svg

@@ -0,0 +1,6 @@
+<svg width="277" height="236" viewBox="0 0 277 236" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path fill-rule="evenodd" clip-rule="evenodd" d="M213.085 190L242.907 86H276.196L246.375 190H213.085Z" fill="#494949"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M0 190L29.8215 86H63.1111L33.2896 190H0Z" fill="#494949"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M124.986 0L57.5772 235.083L60.7752 236H90.6038L158.276 0H124.986Z" fill="#494949"/>
+<path fill-rule="evenodd" clip-rule="evenodd" d="M195.273 0L127.601 236H160.891L228.563 0H195.273Z" fill="#494949"/>
+</svg>

benchmarks/.gitignore

@@ -0,0 +1 @@
+benches/datasets_paths.rs

benchmarks/Cargo.toml

@@ -0,0 +1,54 @@
+[package]
+name = "benchmarks"
+publish = false
+
+version.workspace = true
+authors.workspace = true
+description.workspace = true
+homepage.workspace = true
+readme.workspace = true
+edition.workspace = true
+license.workspace = true
+
+[dependencies]
+anyhow = "1.0.65"
+csv = "1.1.6"
+milli = { path = "../milli", default-features = false }
+mimalloc = { version = "0.1.29", default-features = false }
+serde_json = { version = "1.0.85", features = ["preserve_order"] }
+
+[dev-dependencies]
+criterion = { version = "0.4.0", features = ["html_reports"] }
+rand = "0.8.5"
+rand_chacha = "0.3.1"
+roaring = "0.10.1"
+
+[build-dependencies]
+anyhow = "1.0.65"
+bytes = "1.2.1"
+convert_case = "0.6.0"
+flate2 = "1.0.24"
+reqwest = { version = "0.11.12", features = ["blocking", "rustls-tls"], default-features = false }
+
+[features]
+default = ["milli/default"]
+
+[[bench]]
+name = "search_songs"
+harness = false
+
+[[bench]]
+name = "search_wiki"
+harness = false
+
+[[bench]]
+name = "search_geo"
+harness = false
+
+[[bench]]
+name = "indexing"
+harness = false
+
+[[bench]]
+name = "formatting"
+harness = false

benchmarks/README.md

@@ -0,0 +1,138 @@
+Benchmarks
+==========
+
+## TOC
+
+- [Run the benchmarks](#run-the-benchmarks)
+- [Comparison between benchmarks](#comparison-between-benchmarks)
+- [Datasets](#datasets)
+
+## Run the benchmarks
+
+### On our private server
+
+The Meili team has self-hosted his own GitHub runner to run benchmarks on our dedicated bare metal server.
+
+To trigger the benchmark workflow:
+- Go to the `Actions` tab of this repository.
+- Select the `Benchmarks` workflow on the left.
+- Click on `Run workflow` in the blue banner.
+- Select the branch on which you want to run the benchmarks and select the dataset you want (default: `songs`).
+- Finally, click on `Run workflow`.
+
+This GitHub workflow will run the benchmarks and push the `critcmp` report to a DigitalOcean Space (= S3).
+
+The name of the uploaded file is displayed in the workflow.
+
+_[More about critcmp](https://github.com/BurntSushi/critcmp)._
+
+💡 To compare the just-uploaded benchmark with another one, check out the [next section](#comparison-between-benchmarks).
+
+### On your machine
+
+To run all the benchmarks (~5h):
+
+```bash
+cargo bench
+```
+
+To run only the `search_songs` (~1h), `search_wiki` (~3h), `search_geo` (~20m) or `indexing` (~2h) benchmark:
+
+```bash
+cargo bench --bench <dataset name>
+```
+
+By default, the benchmarks will be downloaded and uncompressed automatically in the target directory.<br>
+If you don't want to download the datasets every time you update something on the code, you can specify a custom directory with the environment variable `MILLI_BENCH_DATASETS_PATH`:
+
+```bash
+mkdir ~/datasets
+MILLI_BENCH_DATASETS_PATH=~/datasets cargo bench --bench search_songs # the four datasets are downloaded
+touch build.rs
+MILLI_BENCH_DATASETS_PATH=~/datasets cargo bench --bench songs # the code is compiled again but the datasets are not downloaded
+```
+
+## Comparison between benchmarks
+
+The benchmark reports we push are generated with `critcmp`. Thus, we use `critcmp` to show the result of a benchmark, or compare results between multiple benchmarks.
+
+We provide a script to download and display the comparison report.
+
+Requirements:
+- `grep`
+- `curl`
+- [`critcmp`](https://github.com/BurntSushi/critcmp)
+
+List the available file in the DO Space:
+
+```bash
+./benchmarks/script/list.sh
+```
+```bash
+songs_main_09a4321.json
+songs_geosearch_24ec456.json
+search_songs_main_cb45a10b.json
+```
+
+Run the comparison script:
+
+```bash
+# we get the result of ONE benchmark, this give you an idea of how much time an operation took
+./benchmarks/scripts/compare.sh son songs_geosearch_24ec456.json
+# we compare two benchmarks
+./benchmarks/scripts/compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json
+# we compare three benchmarks
+./benchmarks/scripts/compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json search_songs_main_cb45a10b.json
+```
+
+## Datasets
+
+The benchmarks uses the following datasets:
+- `smol-songs`
+- `smol-wiki`
+- `movies`
+- `smol-all-countries`
+
+### Songs
+
+`smol-songs` is a subset of the [`songs.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/songs.csv.gz).
+
+It was generated with this command:
+
+```bash
+xsv sample --seed 42 1000000 songs.csv -o smol-songs.csv
+```
+
+_[Download the generated `smol-songs` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-songs.csv.gz)._
+
+### Wiki
+
+`smol-wiki` is a subset of the [`wikipedia-articles.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/wiki-articles.csv.gz).
+
+It was generated with the following command:
+
+```bash
+xsv sample --seed 42 500000 wiki-articles.csv -o smol-wiki-articles.csv
+```
+
+_[Download the `smol-wiki` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-wiki-articles.csv.gz)._
+
+### Movies
+
+`movies` is a really small dataset we uses as our example in the [getting started](https://docs.meilisearch.com/learn/getting_started/)
+
+_[Download the `movies` dataset](https://docs.meilisearch.com/movies.json)._
+
+
+### All Countries
+
+`smol-all-countries` is a subset of the [`all-countries.csv` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/all-countries.csv.gz)
+It has been converted to jsonlines and then edited so it matches our format for the `_geo` field.
+
+It was generated with the following command:
+```bash
+bat all-countries.csv.gz | gunzip | xsv sample --seed 42 1000000 | csv2json-lite | sd '"latitude":"(.*?)","longitude":"(.*?)"' '"_geo": { "lat": $1, "lng": $2 }' | sd '\[|\]|,$' '' | gzip > smol-all-countries.jsonl.gz
+```
+
+_[Download the `smol-all-countries` dataset](https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets/smol-all-countries.jsonl.gz)._
+

benchmarks/benches/formatting.rs

@@ -0,0 +1,67 @@
+use std::rc::Rc;
+
+use criterion::{criterion_group, criterion_main};
+use milli::tokenizer::TokenizerBuilder;
+use milli::{FormatOptions, MatcherBuilder, MatchingWord, MatchingWords};
+
+#[global_allocator]
+static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
+struct Conf<'a> {
+    name: &'a str,
+    text: &'a str,
+    matching_words: MatcherBuilder<'a, Vec<u8>>,
+}
+
+fn bench_formatting(c: &mut criterion::Criterion) {
+    #[rustfmt::skip]
+    let confs = &[
+    	Conf {
+    		name: "'the door d'",
+			text: r#"He used to do the door sounds in "Star Trek" with his mouth, phssst, phssst. The MD-11 passenger and cargo doors also tend to behave like electromagnetic apertures, because the doors do not have continuous electrical contact with the door frames around the door perimeter. But Theodor said that the doors don't work."#,
+			matching_words: MatcherBuilder::new(MatchingWords::new(vec![
+	            (vec![Rc::new(MatchingWord::new("t".to_string(), 0, false).unwrap()), Rc::new(MatchingWord::new("he".to_string(), 0, false).unwrap())], vec![0]),
+	            (vec![Rc::new(MatchingWord::new("the".to_string(), 0, false).unwrap())], vec![0]),
+	            (vec![Rc::new(MatchingWord::new("door".to_string(), 1, false).unwrap())], vec![1]),
+	            (vec![Rc::new(MatchingWord::new("do".to_string(), 0, false).unwrap()), Rc::new(MatchingWord::new("or".to_string(), 0, false).unwrap())], vec![0]),
+	            (vec![Rc::new(MatchingWord::new("thedoor".to_string(), 1, false).unwrap())], vec![0, 1]),
+	            (vec![Rc::new(MatchingWord::new("d".to_string(), 0, true).unwrap())], vec![2]),
+	            (vec![Rc::new(MatchingWord::new("thedoord".to_string(), 1, true).unwrap())], vec![0, 1, 2]),
+	            (vec![Rc::new(MatchingWord::new("doord".to_string(), 1, true).unwrap())], vec![1, 2]),
+        	]
+            ), TokenizerBuilder::default().build()),
+		},
+    ];
+
+    let format_options = &[
+        FormatOptions { highlight: false, crop: None },
+        FormatOptions { highlight: true, crop: None },
+        FormatOptions { highlight: false, crop: Some(10) },
+        FormatOptions { highlight: true, crop: Some(10) },
+        FormatOptions { highlight: false, crop: Some(20) },
+        FormatOptions { highlight: true, crop: Some(20) },
+    ];
+
+    for option in format_options {
+        let highlight = if option.highlight { "highlight" } else { "no-highlight" };
+
+        let name = match option.crop {
+            Some(size) => format!("{}-crop({})", highlight, size),
+            None => format!("{}-no-crop", highlight),
+        };
+
+        let mut group = c.benchmark_group(&name);
+        for conf in confs {
+            group.bench_function(conf.name, |b| {
+                b.iter(|| {
+                    let mut matcher = conf.matching_words.build(conf.text);
+                    matcher.format(*option);
+                })
+            });
+        }
+        group.finish();
+    }
+}
+
+criterion_group!(benches, bench_formatting);
+criterion_main!(benches);

benchmarks/benches/search_geo.rs

@@ -0,0 +1,122 @@
+mod datasets_paths;
+mod utils;
+
+use criterion::{criterion_group, criterion_main};
+use milli::update::Settings;
+use utils::Conf;
+
+#[global_allocator]
+static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
+fn base_conf(builder: &mut Settings) {
+    let displayed_fields =
+        ["geonameid", "name", "asciiname", "alternatenames", "_geo", "population"]
+            .iter()
+            .map(|s| s.to_string())
+            .collect();
+    builder.set_displayed_fields(displayed_fields);
+
+    let searchable_fields =
+        ["name", "alternatenames", "elevation"].iter().map(|s| s.to_string()).collect();
+    builder.set_searchable_fields(searchable_fields);
+
+    let filterable_fields =
+        ["_geo", "population", "elevation"].iter().map(|s| s.to_string()).collect();
+    builder.set_filterable_fields(filterable_fields);
+
+    let sortable_fields =
+        ["_geo", "population", "elevation"].iter().map(|s| s.to_string()).collect();
+    builder.set_sortable_fields(sortable_fields);
+}
+
+#[rustfmt::skip]
+const BASE_CONF: Conf = Conf {
+    dataset: datasets_paths::SMOL_ALL_COUNTRIES,
+    dataset_format: "jsonl",
+    queries: &[
+        "",
+    ],
+    configure: base_conf,
+    primary_key: Some("geonameid"),
+    ..Conf::BASE
+};
+
+fn bench_geo(c: &mut criterion::Criterion) {
+    #[rustfmt::skip]
+    let confs = &[
+        // A basic placeholder with no geo
+        utils::Conf {
+            group_name: "placeholder with no geo",
+            ..BASE_CONF
+        },
+        // Medium aglomeration: probably the most common usecase
+        utils::Conf {
+            group_name: "asc sort from Lille",
+            sort: Some(vec!["_geoPoint(50.62999333378238, 3.086269263384099):asc"]),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "desc sort from Lille",
+            sort: Some(vec!["_geoPoint(50.62999333378238, 3.086269263384099):desc"]),
+            ..BASE_CONF
+        },
+        // Big agglomeration: a lot of documents close to our point
+        utils::Conf {
+            group_name: "asc sort from Tokyo",
+            sort: Some(vec!["_geoPoint(35.749512532692144, 139.61664952543356):asc"]),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "desc sort from Tokyo",
+            sort: Some(vec!["_geoPoint(35.749512532692144, 139.61664952543356):desc"]),
+            ..BASE_CONF
+        },
+        // The furthest point from any civilization
+        utils::Conf {
+            group_name: "asc sort from Point Nemo",
+            sort: Some(vec!["_geoPoint(-48.87561645055408, -123.39275749319793):asc"]),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "desc sort from Point Nemo",
+            sort: Some(vec!["_geoPoint(-48.87561645055408, -123.39275749319793):desc"]),
+            ..BASE_CONF
+        },
+        // Filters
+        utils::Conf {
+            group_name: "filter of 100km from Lille",
+            filter: Some("_geoRadius(50.62999333378238, 3.086269263384099, 100000)"),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "filter of 1km from Lille",
+            filter: Some("_geoRadius(50.62999333378238, 3.086269263384099, 1000)"),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "filter of 100km from Tokyo",
+            filter: Some("_geoRadius(35.749512532692144, 139.61664952543356, 100000)"),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "filter of 1km from Tokyo",
+            filter: Some("_geoRadius(35.749512532692144, 139.61664952543356, 1000)"),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "filter of 100km from Point Nemo",
+            filter: Some("_geoRadius(-48.87561645055408, -123.39275749319793, 100000)"),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "filter of 1km from Point Nemo",
+            filter: Some("_geoRadius(-48.87561645055408, -123.39275749319793, 1000)"),
+            ..BASE_CONF
+        },
+    ];
+
+    utils::run_benches(c, confs);
+}
+
+criterion_group!(benches, bench_geo);
+criterion_main!(benches);

benchmarks/benches/search_songs.rs

@@ -0,0 +1,196 @@
+mod datasets_paths;
+mod utils;
+
+use criterion::{criterion_group, criterion_main};
+use milli::update::Settings;
+use utils::Conf;
+
+#[global_allocator]
+static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
+fn base_conf(builder: &mut Settings) {
+    let displayed_fields =
+        ["id", "title", "album", "artist", "genre", "country", "released", "duration"]
+            .iter()
+            .map(|s| s.to_string())
+            .collect();
+    builder.set_displayed_fields(displayed_fields);
+
+    let searchable_fields = ["title", "album", "artist"].iter().map(|s| s.to_string()).collect();
+    builder.set_searchable_fields(searchable_fields);
+
+    let faceted_fields = ["released-timestamp", "duration-float", "genre", "country", "artist"]
+        .iter()
+        .map(|s| s.to_string())
+        .collect();
+    builder.set_filterable_fields(faceted_fields);
+}
+
+#[rustfmt::skip]
+const BASE_CONF: Conf = Conf {
+    dataset: datasets_paths::SMOL_SONGS,
+    queries: &[
+        "john ",             // 9097
+        "david ",            // 4794
+        "charles ",          // 1957
+        "david bowie ",      // 1200
+        "michael jackson ",  // 600
+        "thelonious monk ",  // 303
+        "charles mingus ",   // 142
+        "marcus miller ",    // 60
+        "tamo ",             // 13
+        "Notstandskomitee ", // 4
+    ],
+    configure: base_conf,
+    primary_key: Some("id"),
+    ..Conf::BASE
+};
+
+fn bench_songs(c: &mut criterion::Criterion) {
+    let default_criterion: Vec<String> =
+        milli::default_criteria().iter().map(|criteria| criteria.to_string()).collect();
+    let default_criterion = default_criterion.iter().map(|s| s.as_str());
+    let asc_default: Vec<&str> =
+        std::iter::once("released-timestamp:asc").chain(default_criterion.clone()).collect();
+    let desc_default: Vec<&str> =
+        std::iter::once("released-timestamp:desc").chain(default_criterion.clone()).collect();
+
+    let basic_with_quote: Vec<String> = BASE_CONF
+        .queries
+        .iter()
+        .map(|s| {
+            s.trim().split(' ').map(|s| format!(r#""{}""#, s)).collect::<Vec<String>>().join(" ")
+        })
+        .collect();
+    let basic_with_quote: &[&str] =
+        &basic_with_quote.iter().map(|s| s.as_str()).collect::<Vec<&str>>();
+
+    #[rustfmt::skip]
+    let confs = &[
+        /* first we bench each criterion alone */
+        utils::Conf {
+            group_name: "proximity",
+            queries: &[
+                "black saint sinner lady ",
+                "les dangeureuses 1960 ",
+                "The Disneyland Sing-Along Chorus ",
+                "Under Great Northern Lights ",
+                "7000 Danses Un Jour Dans Notre Vie ",
+            ],
+            criterion: Some(&["proximity"]),
+            optional_words: false,
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "typo",
+            queries: &[
+                "mongus ",
+                "thelonius monk ",
+                "Disnaylande ",
+                "the white striper ",
+                "indochie ",
+                "indochien ",
+                "klub des loopers ",
+                "fear of the duck ",
+                "michel depech ",
+                "stromal ",
+                "dire straights ",
+                "Arethla Franklin ",
+            ],
+            criterion: Some(&["typo"]),
+            optional_words: false,
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "words",
+            queries: &[
+                "the black saint and the sinner lady and the good doggo ", // four words to pop
+                "les liaisons dangeureuses 1793 ",                         // one word to pop
+                "The Disneyland Children's Sing-Alone song ",              // two words to pop
+                "seven nation mummy ",                                     // one word to pop
+                "7000 Danses / Le Baiser / je me trompe de mots ",         // four words to pop
+                "Bring Your Daughter To The Slaughter but now this is not part of the title ", // nine words to pop
+                "whathavenotnsuchforth and a good amount of words to pop to match the first one ", // 13
+            ],
+            criterion: Some(&["words"]),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "asc",
+            criterion: Some(&["released-timestamp:desc"]),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "desc",
+            criterion: Some(&["released-timestamp:desc"]),
+            ..BASE_CONF
+        },
+
+        /* then we bench the asc and desc criterion on top of the default criterion */
+        utils::Conf {
+            group_name: "asc + default",
+            criterion: Some(&asc_default[..]),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "desc + default",
+            criterion: Some(&desc_default[..]),
+            ..BASE_CONF
+        },
+
+        /* we bench the filters with the default request */
+        utils::Conf {
+            group_name: "basic filter: <=",
+            filter: Some("released-timestamp <= 946728000"), // year 2000
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "basic filter: TO",
+            filter: Some("released-timestamp 946728000 TO 1262347200"), // year 2000 to 2010
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "big filter",
+            filter: Some("released-timestamp != 1262347200 AND (NOT (released-timestamp = 946728000)) AND (duration-float = 1 OR (duration-float 1.1 TO 1.5 AND released-timestamp > 315576000))"),
+            ..BASE_CONF
+        },
+
+        /* the we bench some global / normal search with all the default criterion in the default
+         * order */
+        utils::Conf {
+            group_name: "basic placeholder",
+            queries: &[""],
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "basic without quote",
+            queries: &BASE_CONF
+                .queries
+                .iter()
+                .map(|s| s.trim()) // we remove the space at the end of each request
+                .collect::<Vec<&str>>(),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "basic with quote",
+            queries: basic_with_quote,
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "prefix search",
+            queries: &[
+                "s", // 500k+ results
+                "a", //
+                "b", //
+                "i", //
+                "x", // only 7k results
+            ],
+            ..BASE_CONF
+        },
+    ];
+
+    utils::run_benches(c, confs);
+}
+
+criterion_group!(benches, bench_songs);
+criterion_main!(benches);

benchmarks/benches/search_wiki.rs

@@ -0,0 +1,129 @@
+mod datasets_paths;
+mod utils;
+
+use criterion::{criterion_group, criterion_main};
+use milli::update::Settings;
+use utils::Conf;
+
+#[global_allocator]
+static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
+fn base_conf(builder: &mut Settings) {
+    let displayed_fields = ["title", "body", "url"].iter().map(|s| s.to_string()).collect();
+    builder.set_displayed_fields(displayed_fields);
+
+    let searchable_fields = ["title", "body"].iter().map(|s| s.to_string()).collect();
+    builder.set_searchable_fields(searchable_fields);
+}
+
+#[rustfmt::skip]
+const BASE_CONF: Conf = Conf {
+    dataset: datasets_paths::SMOL_WIKI_ARTICLES,
+    queries: &[
+        "mingus ",        // 46 candidates
+        "miles davis ",   // 159
+        "rock and roll ", // 1007
+        "machine ",       // 3448
+        "spain ",         // 7002
+        "japan ",         // 10.593
+        "france ",        // 17.616
+        "film ",          // 24.959
+    ],
+    configure: base_conf,
+    ..Conf::BASE
+};
+
+fn bench_songs(c: &mut criterion::Criterion) {
+    let basic_with_quote: Vec<String> = BASE_CONF
+        .queries
+        .iter()
+        .map(|s| {
+            s.trim().split(' ').map(|s| format!(r#""{}""#, s)).collect::<Vec<String>>().join(" ")
+        })
+        .collect();
+    let basic_with_quote: &[&str] =
+        &basic_with_quote.iter().map(|s| s.as_str()).collect::<Vec<&str>>();
+
+    #[rustfmt::skip]
+    let confs = &[
+        /* first we bench each criterion alone */
+        utils::Conf {
+            group_name: "proximity",
+            queries: &[
+                "herald sings ",
+                "april paris ",
+                "tea two ",
+                "diesel engine ",
+            ],
+            criterion: Some(&["proximity"]),
+            optional_words: false,
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "typo",
+            queries: &[
+                "migrosoft ",
+                "linax ",
+                "Disnaylande ",
+                "phytogropher ",
+                "nympalidea ",
+                "aritmetric ",
+                "the fronce ",
+                "sisan ",
+            ],
+            criterion: Some(&["typo"]),
+            optional_words: false,
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "words",
+            queries: &[
+                "the black saint and the sinner lady and the good doggo ", // four words to pop, 27 results
+                "Kameya Tokujirō mingus monk ",                            // two words to pop, 55
+                "Ulrich Hensel meilisearch milli ",                        // two words to pop, 306
+                "Idaho Bellevue pizza ",                                   // one word to pop, 800
+                "Abraham machin ",                                         // one word to pop, 1141
+            ],
+            criterion: Some(&["words"]),
+            ..BASE_CONF
+        },
+        /* the we bench some global / normal search with all the default criterion in the default
+         * order */
+        utils::Conf {
+            group_name: "basic placeholder",
+            queries: &[""],
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "basic without quote",
+            queries: &BASE_CONF
+                .queries
+                .iter()
+                .map(|s| s.trim()) // we remove the space at the end of each request
+                .collect::<Vec<&str>>(),
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "basic with quote",
+            queries: basic_with_quote,
+            ..BASE_CONF
+        },
+        utils::Conf {
+            group_name: "prefix search",
+            queries: &[
+                "t", // 453k results
+                "c", // 405k
+                "g", // 318k
+                "j", // 227k
+                "q", // 71k
+                "x", // 17k
+            ],
+            ..BASE_CONF
+        },
+    ];
+
+    utils::run_benches(c, confs);
+}
+
+criterion_group!(benches, bench_songs);
+criterion_main!(benches);

benchmarks/benches/utils.rs

@@ -0,0 +1,256 @@
+#![allow(dead_code)]
+
+use std::fs::{create_dir_all, remove_dir_all, File};
+use std::io::{self, BufRead, BufReader, Cursor, Read, Seek};
+use std::num::ParseFloatError;
+use std::path::Path;
+use std::str::FromStr;
+
+use criterion::BenchmarkId;
+use milli::documents::{DocumentsBatchBuilder, DocumentsBatchReader};
+use milli::heed::EnvOpenOptions;
+use milli::update::{
+    IndexDocuments, IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Settings,
+};
+use milli::{Criterion, Filter, Index, Object, TermsMatchingStrategy};
+use serde_json::Value;
+
+pub struct Conf<'a> {
+    /// where we are going to create our database.mmdb directory
+    /// each benchmark will first try to delete it and then recreate it
+    pub database_name: &'a str,
+    /// the dataset to be used, it must be an uncompressed csv
+    pub dataset: &'a str,
+    /// The format of the dataset
+    pub dataset_format: &'a str,
+    pub group_name: &'a str,
+    pub queries: &'a [&'a str],
+    /// here you can change which criterion are used and in which order.
+    /// - if you specify something all the base configuration will be thrown out
+    /// - if you don't specify anything (None) the default configuration will be kept
+    pub criterion: Option<&'a [&'a str]>,
+    /// the last chance to configure your database as you want
+    pub configure: fn(&mut Settings),
+    pub filter: Option<&'a str>,
+    pub sort: Option<Vec<&'a str>>,
+    /// enable or disable the optional words on the query
+    pub optional_words: bool,
+    /// primary key, if there is None we'll auto-generate docids for every documents
+    pub primary_key: Option<&'a str>,
+}
+
+impl Conf<'_> {
+    pub const BASE: Self = Conf {
+        database_name: "benches.mmdb",
+        dataset_format: "csv",
+        dataset: "",
+        group_name: "",
+        queries: &[],
+        criterion: None,
+        configure: |_| (),
+        filter: None,
+        sort: None,
+        optional_words: true,
+        primary_key: None,
+    };
+}
+
+pub fn base_setup(conf: &Conf) -> Index {
+    match remove_dir_all(conf.database_name) {
+        Ok(_) => (),
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => (),
+        Err(e) => panic!("{}", e),
+    }
+    create_dir_all(conf.database_name).unwrap();
+
+    let mut options = EnvOpenOptions::new();
+    options.map_size(100 * 1024 * 1024 * 1024); // 100 GB
+    options.max_readers(10);
+    let index = Index::new(options, conf.database_name).unwrap();
+
+    let config = IndexerConfig::default();
+    let mut wtxn = index.write_txn().unwrap();
+    let mut builder = Settings::new(&mut wtxn, &index, &config);
+
+    if let Some(primary_key) = conf.primary_key {
+        builder.set_primary_key(primary_key.to_string());
+    }
+
+    if let Some(criterion) = conf.criterion {
+        builder.reset_filterable_fields();
+        builder.reset_criteria();
+        builder.reset_stop_words();
+
+        let criterion = criterion.iter().map(|s| Criterion::from_str(s).unwrap()).collect();
+        builder.set_criteria(criterion);
+    }
+
+    (conf.configure)(&mut builder);
+
+    builder.execute(|_| (), || false).unwrap();
+    wtxn.commit().unwrap();
+
+    let config = IndexerConfig::default();
+    let mut wtxn = index.write_txn().unwrap();
+    let indexing_config = IndexDocumentsConfig {
+        autogenerate_docids: conf.primary_key.is_none(),
+        update_method: IndexDocumentsMethod::ReplaceDocuments,
+        ..Default::default()
+    };
+    let builder =
+        IndexDocuments::new(&mut wtxn, &index, &config, indexing_config, |_| (), || false).unwrap();
+    let documents = documents_from(conf.dataset, conf.dataset_format);
+    let (builder, user_error) = builder.add_documents(documents).unwrap();
+    user_error.unwrap();
+    builder.execute().unwrap();
+    wtxn.commit().unwrap();
+
+    index
+}
+
+pub fn run_benches(c: &mut criterion::Criterion, confs: &[Conf]) {
+    for conf in confs {
+        let index = base_setup(conf);
+
+        let file_name = Path::new(conf.dataset).file_name().and_then(|f| f.to_str()).unwrap();
+        let name = format!("{}: {}", file_name, conf.group_name);
+        let mut group = c.benchmark_group(&name);
+
+        for &query in conf.queries {
+            group.bench_with_input(BenchmarkId::from_parameter(query), &query, |b, &query| {
+                b.iter(|| {
+                    let rtxn = index.read_txn().unwrap();
+                    let mut search = index.search(&rtxn);
+                    search.query(query).terms_matching_strategy(TermsMatchingStrategy::default());
+                    if let Some(filter) = conf.filter {
+                        let filter = Filter::from_str(filter).unwrap().unwrap();
+                        search.filter(filter);
+                    }
+                    if let Some(sort) = &conf.sort {
+                        let sort = sort.iter().map(|sort| sort.parse().unwrap()).collect();
+                        search.sort_criteria(sort);
+                    }
+                    let _ids = search.execute().unwrap();
+                });
+            });
+        }
+        group.finish();
+
+        index.prepare_for_closing().wait();
+    }
+}
+
+pub fn documents_from(filename: &str, filetype: &str) -> DocumentsBatchReader<impl BufRead + Seek> {
+    let reader = File::open(filename)
+        .unwrap_or_else(|_| panic!("could not find the dataset in: {}", filename));
+    let reader = BufReader::new(reader);
+    let documents = match filetype {
+        "csv" => documents_from_csv(reader).unwrap(),
+        "json" => documents_from_json(reader).unwrap(),
+        "jsonl" => documents_from_jsonl(reader).unwrap(),
+        otherwise => panic!("invalid update format {:?}", otherwise),
+    };
+    DocumentsBatchReader::from_reader(Cursor::new(documents)).unwrap()
+}
+
+fn documents_from_jsonl(reader: impl BufRead) -> anyhow::Result<Vec<u8>> {
+    let mut documents = DocumentsBatchBuilder::new(Vec::new());
+
+    for result in serde_json::Deserializer::from_reader(reader).into_iter::<Object>() {
+        let object = result?;
+        documents.append_json_object(&object)?;
+    }
+
+    documents.into_inner().map_err(Into::into)
+}
+
+fn documents_from_json(reader: impl BufRead) -> anyhow::Result<Vec<u8>> {
+    let mut documents = DocumentsBatchBuilder::new(Vec::new());
+
+    documents.append_json_array(reader)?;
+
+    documents.into_inner().map_err(Into::into)
+}
+
+fn documents_from_csv(reader: impl BufRead) -> anyhow::Result<Vec<u8>> {
+    let csv = csv::Reader::from_reader(reader);
+
+    let mut documents = DocumentsBatchBuilder::new(Vec::new());
+    documents.append_csv(csv)?;
+
+    documents.into_inner().map_err(Into::into)
+}
+
+enum AllowedType {
+    String,
+    Number,
+}
+
+fn parse_csv_header(header: &str) -> (String, AllowedType) {
+    // if there are several separators we only split on the last one.
+    match header.rsplit_once(':') {
+        Some((field_name, field_type)) => match field_type {
+            "string" => (field_name.to_string(), AllowedType::String),
+            "number" => (field_name.to_string(), AllowedType::Number),
+            // we may return an error in this case.
+            _otherwise => (header.to_string(), AllowedType::String),
+        },
+        None => (header.to_string(), AllowedType::String),
+    }
+}
+
+struct CSVDocumentDeserializer<R>
+where
+    R: Read,
+{
+    documents: csv::StringRecordsIntoIter<R>,
+    headers: Vec<(String, AllowedType)>,
+}
+
+impl<R: Read> CSVDocumentDeserializer<R> {
+    fn from_reader(reader: R) -> io::Result<Self> {
+        let mut records = csv::Reader::from_reader(reader);
+
+        let headers = records.headers()?.into_iter().map(parse_csv_header).collect();
+
+        Ok(Self { documents: records.into_records(), headers })
+    }
+}
+
+impl<R: Read> Iterator for CSVDocumentDeserializer<R> {
+    type Item = anyhow::Result<Object>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let csv_document = self.documents.next()?;
+
+        match csv_document {
+            Ok(csv_document) => {
+                let mut document = Object::new();
+
+                for ((field_name, field_type), value) in
+                    self.headers.iter().zip(csv_document.into_iter())
+                {
+                    let parsed_value: Result<Value, ParseFloatError> = match field_type {
+                        AllowedType::Number => {
+                            value.parse::<f64>().map(Value::from).map_err(Into::into)
+                        }
+                        AllowedType::String => Ok(Value::String(value.to_string())),
+                    };
+
+                    match parsed_value {
+                        Ok(value) => drop(document.insert(field_name.to_string(), value)),
+                        Err(_e) => {
+                            return Some(Err(anyhow::anyhow!(
+                                "Value '{}' is not a valid number",
+                                value
+                            )))
+                        }
+                    }
+                }
+
+                Some(Ok(document))
+            }
+            Err(e) => Some(Err(anyhow::anyhow!("Error parsing csv document: {}", e))),
+        }
+    }
+}

benchmarks/build.rs

@@ -0,0 +1,115 @@
+use std::fs::File;
+use std::io::{Cursor, Read, Seek, Write};
+use std::path::{Path, PathBuf};
+use std::{env, fs};
+
+use bytes::Bytes;
+use convert_case::{Case, Casing};
+use flate2::read::GzDecoder;
+use reqwest::IntoUrl;
+
+const BASE_URL: &str = "https://milli-benchmarks.fra1.digitaloceanspaces.com/datasets";
+
+const DATASET_SONGS: (&str, &str) = ("smol-songs", "csv");
+const DATASET_SONGS_1_2: (&str, &str) = ("smol-songs-1_2", "csv");
+const DATASET_SONGS_3_4: (&str, &str) = ("smol-songs-3_4", "csv");
+const DATASET_SONGS_4_4: (&str, &str) = ("smol-songs-4_4", "csv");
+const DATASET_WIKI: (&str, &str) = ("smol-wiki-articles", "csv");
+const DATASET_WIKI_1_2: (&str, &str) = ("smol-wiki-articles-1_2", "csv");
+const DATASET_WIKI_3_4: (&str, &str) = ("smol-wiki-articles-3_4", "csv");
+const DATASET_WIKI_4_4: (&str, &str) = ("smol-wiki-articles-4_4", "csv");
+const DATASET_MOVIES: (&str, &str) = ("movies", "json");
+const DATASET_MOVIES_1_2: (&str, &str) = ("movies-1_2", "json");
+const DATASET_MOVIES_3_4: (&str, &str) = ("movies-3_4", "json");
+const DATASET_MOVIES_4_4: (&str, &str) = ("movies-4_4", "json");
+const DATASET_NESTED_MOVIES: (&str, &str) = ("nested_movies", "json");
+const DATASET_GEO: (&str, &str) = ("smol-all-countries", "jsonl");
+
+const ALL_DATASETS: &[(&str, &str)] = &[
+    DATASET_SONGS,
+    DATASET_SONGS_1_2,
+    DATASET_SONGS_3_4,
+    DATASET_SONGS_4_4,
+    DATASET_WIKI,
+    DATASET_WIKI_1_2,
+    DATASET_WIKI_3_4,
+    DATASET_WIKI_4_4,
+    DATASET_MOVIES,
+    DATASET_MOVIES_1_2,
+    DATASET_MOVIES_3_4,
+    DATASET_MOVIES_4_4,
+    DATASET_NESTED_MOVIES,
+    DATASET_GEO,
+];
+
+/// The name of the environment variable used to select the path
+/// of the directory containing the datasets
+const BASE_DATASETS_PATH_KEY: &str = "MILLI_BENCH_DATASETS_PATH";
+
+fn main() -> anyhow::Result<()> {
+    let out_dir = PathBuf::from(env::var(BASE_DATASETS_PATH_KEY).unwrap_or(env::var("OUT_DIR")?));
+
+    let benches_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR")?).join("benches");
+    let mut manifest_paths_file = File::create(benches_dir.join("datasets_paths.rs"))?;
+    write!(
+        manifest_paths_file,
+        r#"//! This file is generated by the build script.
+//! Do not modify by hand, use the build.rs file.
+#![allow(dead_code)]
+"#
+    )?;
+    writeln!(manifest_paths_file)?;
+
+    for (dataset, extension) in ALL_DATASETS {
+        let out_path = out_dir.join(dataset);
+        let out_file = out_path.with_extension(extension);
+
+        writeln!(
+            &mut manifest_paths_file,
+            r#"pub const {}: &str = {:?};"#,
+            dataset.to_case(Case::ScreamingSnake),
+            out_file.display(),
+        )?;
+
+        if out_file.exists() {
+            eprintln!(
+                "The dataset {} already exists on the file system and will not be downloaded again",
+                out_path.display(),
+            );
+            continue;
+        }
+        let url = format!("{}/{}.{}.gz", BASE_URL, dataset, extension);
+        eprintln!("downloading: {}", url);
+        let bytes = retry(|| download_dataset(url.clone()), 10)?;
+        eprintln!("{} downloaded successfully", url);
+        eprintln!("uncompressing in {}", out_file.display());
+        uncompress_in_file(bytes, &out_file)?;
+    }
+
+    Ok(())
+}
+
+fn retry<Ok, Err>(fun: impl Fn() -> Result<Ok, Err>, times: usize) -> Result<Ok, Err> {
+    for _ in 0..times {
+        if let ok @ Ok(_) = fun() {
+            return ok;
+        }
+    }
+    fun()
+}
+
+fn download_dataset<U: IntoUrl>(url: U) -> anyhow::Result<Cursor<Bytes>> {
+    let bytes =
+        reqwest::blocking::Client::builder().timeout(None).build()?.get(url).send()?.bytes()?;
+    Ok(Cursor::new(bytes))
+}
+
+fn uncompress_in_file<R: Read + Seek, P: AsRef<Path>>(bytes: R, path: P) -> anyhow::Result<()> {
+    let path = path.as_ref();
+    let mut gz = GzDecoder::new(bytes);
+    let mut dataset = Vec::new();
+    gz.read_to_end(&mut dataset)?;
+
+    fs::write(path, dataset)?;
+    Ok(())
+}

benchmarks/scripts/compare.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+# Requirements:
+# - critcmp. See: https://github.com/BurntSushi/critcmp
+# - curl
+
+# Usage
+# $ bash compare.sh json_file1 json_file1
+# ex: bash compare.sh songs_main_09a4321.json songs_geosearch_24ec456.json
+
+# Checking that critcmp is installed
+command -v critcmp > /dev/null 2>&1
+if [[ "$?" -ne 0 ]]; then
+    echo 'You must install critcmp to make this script work.'
+    echo 'See: https://github.com/BurntSushi/critcmp'
+    echo '  $ cargo install critcmp'
+    exit 1
+fi
+
+s3_url='https://milli-benchmarks.fra1.digitaloceanspaces.com/critcmp_results'
+
+for file in $@
+do
+    file_s3_url="$s3_url/$file"
+    file_local_path="/tmp/$file"
+
+    if [[ ! -f $file_local_path ]]; then
+        curl $file_s3_url --output $file_local_path --silent
+        if [[ "$?" -ne 0 ]]; then
+            echo 'curl command failed.'
+            exit 1
+        fi
+    fi
+done
+
+path_list=$(echo " $@" | sed 's/ / \/tmp\//g')
+
+critcmp $path_list

benchmarks/scripts/list.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+# Requirements:
+# - curl
+# - grep
+
+res=$(curl -s https://milli-benchmarks.fra1.digitaloceanspaces.com | grep -o '<Key>[^<]\+' | cut -c 5- | grep critcmp_results/ | cut -c 18-)
+
+for pattern in "$@"
+do
+	res=$(echo "$res" | grep $pattern)
+done
+
+echo "$res"

benchmarks/src/lib.rs

@@ -0,0 +1,5 @@
+//! This library is only used to isolate the benchmarks
+//! from the original milli library.
+//!
+//! It does not include interesting functions for milli library
+//! users only for milli contributors.

bors.toml

@@ -1,7 +1,7 @@
 status = [
     'Tests on ubuntu-18.04',
-    'Tests on macos-latest',
-    'Tests on windows-latest',
+    'Tests on macos-12',
+    'Tests on windows-2022',
     'Run Clippy',
     'Run Rustfmt',
     'Run tests in debug',

config.toml

@@ -28,17 +28,9 @@ http_payload_size_limit = "100 MB"
 
 log_level = "INFO"
 # Defines how much detail should be present in Meilisearch's logs.
-# Meilisearch currently supports five log levels, listed in order of increasing verbosity: `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`
+# Meilisearch currently supports six log levels, listed in order of increasing verbosity:  `OFF`, `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`
 # https://docs.meilisearch.com/learn/configuration/instance_options.html#log-level
 
-max_index_size = "100 GiB"
-# Sets the maximum size of the index.
-# https://docs.meilisearch.com/learn/configuration/instance_options.html#max-index-size
-
-max_task_db_size = "100 GiB"
-# Sets the maximum size of the task database.
-# https://docs.meilisearch.com/learn/configuration/instance_options.html#max-task-db-size
-
 # max_indexing_memory = "2 GiB"
 # Sets the maximum amount of RAM Meilisearch can use when indexing.
 # https://docs.meilisearch.com/learn/configuration/instance_options.html#max-indexing-memory
@@ -47,18 +39,13 @@ max_task_db_size = "100 GiB"
 # Sets the maximum number of threads Meilisearch can use during indexing.
 # https://docs.meilisearch.com/learn/configuration/instance_options.html#max-indexing-threads
 
-disable_auto_batching = false
-# Deactivates auto-batching when provided.
-# https://docs.meilisearch.com/learn/configuration/instance_options.html#disable-auto-batching
-
-
 #############
 ### DUMPS ###
 #############
 
-dumps_dir = "dumps/"
+dump_dir = "dumps/"
 # Sets the directory where Meilisearch will create dump files.
-# https://docs.meilisearch.com/learn/configuration/instance_options.html#dumps-destination
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#dump-directory
 
 # import_dump = "./path/to/my/file.dump"
 # Imports the dump file located at the specified path. Path must point to a .dump file.
@@ -78,17 +65,15 @@ ignore_dump_if_db_exists = false
 #################
 
 schedule_snapshot = false
-# Activates scheduled snapshots when provided.
+# Enables scheduled snapshots when true, disable when false (the default).
+# If the value is given as an integer, then enables the scheduled snapshot with the passed value as the interval
+# between each snapshot, in seconds.
 # https://docs.meilisearch.com/learn/configuration/instance_options.html#schedule-snapshot-creation
 
 snapshot_dir = "snapshots/"
 # Sets the directory where Meilisearch will store snapshots.
 # https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-destination
 
-snapshot_interval_sec = 86400
-# Defines the interval between each snapshot. Value must be given in seconds.
-# https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-interval
-
 # import_snapshot = "./path/to/my/snapshot"
 # Launches Meilisearch after importing a previously-generated snapshot at the given filepath.
 # https://docs.meilisearch.com/learn/configuration/instance_options.html#import-snapshot

download-latest.sh

@@ -1,5 +1,8 @@
 #!/bin/sh
 
+# This script can optionally use a GitHub token to increase your request limit (for example, if using this script in a CI).
+# To use a GitHub token, pass it through the GITHUB_PAT environment variable.
+
 # GLOBALS
 
 # Colors
@@ -10,9 +13,6 @@ DEFAULT='\033[0m'
 # Project name
 PNAME='meilisearch'
 
-# Version regexp i.e. v[number].[number].[number]
-GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$'
-
 # GitHub API address
 GITHUB_API='https://api.github.com/repos/meilisearch/meilisearch/releases'
 # GitHub Release address
@@ -20,126 +20,26 @@ GITHUB_REL='https://github.com/meilisearch/meilisearch/releases/download/'
 
 # FUNCTIONS
 
-# semverParseInto and semverLT from: https://github.com/cloudflare/semver_bash/blob/master/semver.sh
-# usage: semverParseInto version major minor patch special
-# version: the string version
-# major, minor, patch, special: will be assigned by the function
-semverParseInto() {
-    local RE='[^0-9]*\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)\([0-9A-Za-z-]*\)'
-    # MAJOR
-    eval $2=`echo $1 | sed -e "s#$RE#\1#"`
-    # MINOR
-    eval $3=`echo $1 | sed -e "s#$RE#\2#"`
-    # PATCH
-    eval $4=`echo $1 | sed -e "s#$RE#\3#"`
-    # SPECIAL
-    eval $5=`echo $1 | sed -e "s#$RE#\4#"`
-}
-
-# usage: semverLT version1 version2
-semverLT() {
-    local MAJOR_A=0
-    local MINOR_A=0
-    local PATCH_A=0
-    local SPECIAL_A=0
-
-    local MAJOR_B=0
-    local MINOR_B=0
-    local PATCH_B=0
-    local SPECIAL_B=0
-
-    semverParseInto $1 MAJOR_A MINOR_A PATCH_A SPECIAL_A
-    semverParseInto $2 MAJOR_B MINOR_B PATCH_B SPECIAL_B
-
-    if [ $MAJOR_A -lt $MAJOR_B ]; then
-        return 0
-    fi
-    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -lt $MINOR_B ]; then
-        return 0
-    fi
-    if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
-        return 0
-    fi
-    if [ "_$SPECIAL_A"  == '_' ] && [ "_$SPECIAL_B"  == '_' ] ; then
-        return 1
-    fi
-    if [ "_$SPECIAL_A"  == '_' ] && [ "_$SPECIAL_B"  != '_' ] ; then
-        return 1
-    fi
-    if [ "_$SPECIAL_A"  != '_' ] && [ "_$SPECIAL_B"  == '_' ] ; then
-        return 0
-    fi
-    if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
-        return 0
-    fi
-
-    return 1
-}
-
-# Get a token from: https://github.com/settings/tokens to increase rate limit (from 60 to 5000),
-# make sure the token scope is set to 'public_repo'.
-# Create GITHUB_PAT environment variable once you acquired the token to start using it.
-# Returns the tag of the latest stable release (in terms of semver and not of release date).
+# Gets the version of the latest stable version of Meilisearch by setting the $latest variable.
+# Returns 0 in case of success, 1 otherwise.
 get_latest() {
     # temp_file is needed because the grep would start before the download is over
     temp_file=$(mktemp -q /tmp/$PNAME.XXXXXXXXX)
+    latest_release="$GITHUB_API/latest"
+
     if [ $? -ne 0 ]; then
-        echo "$0: Can't create temp file, bye bye.."
+        echo "$0: Can't create temp file."
+        fetch_release_failure_usage
         exit 1
     fi
 
     if [ -z "$GITHUB_PAT" ]; then
-        curl -s $GITHUB_API > "$temp_file" || return 1
+        curl -s "$latest_release" > "$temp_file" || return 1
     else
-        curl -H "Authorization: token $GITHUB_PAT" -s $GITHUB_API > "$temp_file" || return 1
+        curl -H "Authorization: token $GITHUB_PAT" -s "$latest_release" > "$temp_file" || return 1
     fi
 
-    releases=$(cat "$temp_file" | \
-        grep -E '"tag_name":|"draft":|"prerelease":' \
-        | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
-        # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
-        # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
-
-    i=0
-    latest=''
-    current_tag=''
-    for release_info in $releases; do
-        # Checking tag_name
-        if [ $i -eq 0 ]; then
-            # If it's not an alpha or beta release
-            if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then
-                current_tag=$release_info
-            else
-                current_tag=''
-            fi
-            i=1
-        # Checking draft boolean
-        elif [ $i -eq 1 ]; then
-            if [ "$release_info" = 'true' ]; then
-                current_tag=''
-            fi
-            i=2
-        # Checking prerelease boolean
-        elif [ $i -eq 2 ]; then
-            if [ "$release_info" = 'true' ]; then
-                current_tag=''
-            fi
-            i=0
-            # If the current_tag is valid
-            if [ "$current_tag" != '' ]; then
-                # If there is no latest yes
-                if [ "$latest" = '' ]; then
-                    latest="$current_tag"
-                else
-                    # Comparing latest and the current tag
-                    semverLT $current_tag $latest
-                    if [ $? -eq 1 ]; then
-                        latest="$current_tag"
-                    fi
-                fi
-            fi
-        fi
-    done
+    latest="$(cat "$temp_file" | grep '"tag_name":' | cut -d ':' -f2 | tr -d '"' | tr -d ',' | tr -d ' ')"
 
     rm -f "$temp_file"
     return 0
@@ -174,9 +74,9 @@ get_archi() {
         archi='amd64'
         ;;
     'arm64')
-        # MacOS M1
+        # macOS M1/M2
         if [ $os = 'macos' ]; then
-            archi='amd64'
+            archi='apple-silicon'
         else
             archi='aarch64'
         fi
@@ -210,12 +110,13 @@ fetch_release_failure_usage() {
     echo ''
     printf "$RED%s\n$DEFAULT" 'ERROR: Impossible to get the latest stable version of Meilisearch.'
     echo 'Please let us know about this issue: https://github.com/meilisearch/meilisearch/issues/new/choose'
+    echo ''
+    echo 'In the meantime, you can manually download the appropriate binary from the GitHub release assets here: https://github.com/meilisearch/meilisearch/releases/latest'
 }
 
 fill_release_variables() {
     # Fill $latest variable.
     if ! get_latest; then
-        # TO CHANGE.
         fetch_release_failure_usage
         exit 1
     fi

dump/Cargo.toml

@@ -1,7 +1,14 @@
 [package]
 name = "dump"
-version = "0.30.0"
-edition = "2021"
+publish = false
+
+version.workspace = true
+authors.workspace = true
+description.workspace = true
+edition.workspace = true
+homepage.workspace = true
+readme.workspace = true
+license.workspace = true
 
 [dependencies]
 anyhow = "1.0.65"

dump/src/error.rs

@@ -3,8 +3,6 @@ use thiserror::Error;
 
 #[derive(Debug, Error)]
 pub enum Error {
-    #[error("The version 1 of the dumps is not supported anymore. You can re-export your dump from a version between 0.21 and 0.24, or start fresh from a version 0.25 onwards.")]
-    DumpV1Unsupported,
     #[error("Bad index name.")]
     BadIndexName,
     #[error("Malformed task.")]
@@ -21,14 +19,14 @@ pub enum Error {
 impl ErrorCode for Error {
     fn error_code(&self) -> Code {
         match self {
-            // Are these three really Internal errors?
-            // TODO look at that later.
-            Error::Io(_) => Code::Internal,
+            Error::Io(e) => e.error_code(),
+
+            // These errors either happen when creating a dump and don't need any error code,
+            // or come from an internal bad deserialization.
             Error::Serde(_) => Code::Internal,
             Error::Uuid(_) => Code::Internal,
 
             // all these errors should never be raised when creating a dump, thus no error code should be associated.
-            Error::DumpV1Unsupported => Code::Internal,
             Error::BadIndexName => Code::Internal,
             Error::MalformedTask => Code::Internal,
         }

dump/src/lib.rs

@@ -23,7 +23,7 @@ const CURRENT_DUMP_VERSION: Version = Version::V6;
 
 type Result<T> = std::result::Result<T, Error>;
 
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct Metadata {
     pub dump_version: Version,
@@ -32,7 +32,7 @@ pub struct Metadata {
     pub dump_date: OffsetDateTime,
 }
 
-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct IndexMetadata {
     pub uid: String,
@@ -43,7 +43,7 @@ pub struct IndexMetadata {
     pub updated_at: OffsetDateTime,
 }
 
-#[derive(Debug, PartialEq, Eq, Deserialize, Serialize)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Deserialize, Serialize)]
 pub enum Version {
     V1,
     V2,
@@ -87,7 +87,7 @@ pub struct TaskDump {
     pub finished_at: Option<OffsetDateTime>,
 }
 
-// A `Kind` specific version made for the dump. If modified you may break the dump.
+// A `Kind` specific version made for the dump. If modified you may break the dump.
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum KindDump {
@@ -125,7 +125,6 @@ pub enum KindDump {
         tasks: RoaringBitmap,
     },
     DumpCreation {
-        dump_uid: String,
         keys: Vec<Key>,
         instance_uid: Option<InstanceUid>,
     },
@@ -188,8 +187,8 @@ impl From<KindWithContent> for KindDump {
             KindWithContent::TaskDeletion { query, tasks } => {
                 KindDump::TasksDeletion { query, tasks }
             }
-            KindWithContent::DumpCreation { dump_uid, keys, instance_uid } => {
-                KindDump::DumpCreation { dump_uid, keys, instance_uid }
+            KindWithContent::DumpCreation { keys, instance_uid } => {
+                KindDump::DumpCreation { keys, instance_uid }
             }
             KindWithContent::SnapshotCreation => KindDump::SnapshotCreation,
         }
@@ -199,17 +198,16 @@ impl From<KindWithContent> for KindDump {
 #[cfg(test)]
 pub(crate) mod test {
     use std::fs::File;
-    use std::io::{Seek, SeekFrom};
+    use std::io::Seek;
     use std::str::FromStr;
 
     use big_s::S;
     use maplit::btreeset;
-    use meilisearch_types::index_uid::IndexUid;
+    use meilisearch_types::index_uid_pattern::IndexUidPattern;
     use meilisearch_types::keys::{Action, Key};
     use meilisearch_types::milli::update::Setting;
     use meilisearch_types::milli::{self};
     use meilisearch_types::settings::{Checked, Settings};
-    use meilisearch_types::star_or::StarOr;
     use meilisearch_types::tasks::{Details, Status};
     use serde_json::{json, Map, Value};
     use time::macros::datetime;
@@ -342,7 +340,7 @@ pub(crate) mod test {
                 name: Some(S("doggos_key")),
                 uid: Uuid::from_str("9f8a34da-b6b2-42f0-939b-dbd4c3448655").unwrap(),
                 actions: vec![Action::DocumentsAll],
-                indexes: vec![StarOr::Other(IndexUid::from_str("doggos").unwrap())],
+                indexes: vec![IndexUidPattern::from_str("doggos").unwrap()],
                 expires_at: Some(datetime!(4130-03-14 12:21 UTC)),
                 created_at: datetime!(1960-11-15 0:00 UTC),
                 updated_at: datetime!(2022-11-10 0:00 UTC),
@@ -352,7 +350,7 @@ pub(crate) mod test {
                 name: Some(S("master_key")),
                 uid: Uuid::from_str("4622f717-1c00-47bb-a494-39d76a49b591").unwrap(),
                 actions: vec![Action::All],
-                indexes: vec![StarOr::Star],
+                indexes: vec![IndexUidPattern::all()],
                 expires_at: None,
                 created_at: datetime!(0000-01-01 00:01 UTC),
                 updated_at: datetime!(1964-05-04 17:25 UTC),
@@ -411,13 +409,12 @@ pub(crate) mod test {
         // create the dump
         let mut file = tempfile::tempfile().unwrap();
         dump.persist_to(&mut file).unwrap();
-        file.seek(SeekFrom::Start(0)).unwrap();
+        file.rewind().unwrap();
 
         file
     }
 
     #[test]
-    #[ignore]
     fn test_creating_and_read_dump() {
         let mut file = create_test_dump();
         let mut dump = DumpReader::open(&mut file).unwrap();

dump/src/reader/compat/mod.rs

@@ -1,3 +1,4 @@
+pub mod v1_to_v2;
 pub mod v2_to_v3;
 pub mod v3_to_v4;
 pub mod v4_to_v5;

dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-3.snap

@@ -0,0 +1,38 @@
+---
+source: dump/src/reader/compat/v1_to_v2.rs
+expression: products.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "typo",
+    "words",
+    "proximity",
+    "attribute",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {
+    "android": [
+      "phone",
+      "smartphone"
+    ],
+    "iphone": [
+      "phone",
+      "smartphone"
+    ],
+    "phone": [
+      "android",
+      "iphone",
+      "smartphone"
+    ]
+  },
+  "distinctAttribute": null
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-6.snap

@@ -0,0 +1,31 @@
+---
+source: dump/src/reader/compat/v1_to_v2.rs
+expression: movies.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [
+    "genres",
+    "id"
+  ],
+  "sortableAttributes": [
+    "genres",
+    "id"
+  ],
+  "rankingRules": [
+    "typo",
+    "words",
+    "proximity",
+    "attribute",
+    "exactness",
+    "release_date:asc"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v1_to_v2__test__compat_v1_v2-9.snap

@@ -0,0 +1,24 @@
+---
+source: dump/src/reader/compat/v1_to_v2.rs
+expression: spells.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "typo",
+    "words",
+    "proximity",
+    "attribute",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-11.snap

@@ -0,0 +1,23 @@
+---
+source: dump/src/reader/compat/v2_to_v3.rs
+expression: movies2.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-14.snap

@@ -0,0 +1,23 @@
+---
+source: dump/src/reader/compat/v2_to_v3.rs
+expression: spells.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-5.snap

@@ -0,0 +1,37 @@
+---
+source: dump/src/reader/compat/v2_to_v3.rs
+expression: products.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {
+    "android": [
+      "phone",
+      "smartphone"
+    ],
+    "iphone": [
+      "phone",
+      "smartphone"
+    ],
+    "phone": [
+      "android",
+      "iphone",
+      "smartphone"
+    ]
+  },
+  "distinctAttribute": null
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v2_to_v3__test__compat_v2_v3-8.snap

@@ -0,0 +1,24 @@
+---
+source: dump/src/reader/compat/v2_to_v3.rs
+expression: movies.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "exactness",
+    "release_date:asc"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-12.snap

@@ -0,0 +1,25 @@
+---
+source: dump/src/reader/compat/v3_to_v4.rs
+expression: movies2.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-15.snap

@@ -0,0 +1,25 @@
+---
+source: dump/src/reader/compat/v3_to_v4.rs
+expression: spells.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-6.snap

@@ -0,0 +1,39 @@
+---
+source: dump/src/reader/compat/v3_to_v4.rs
+expression: products.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {
+    "android": [
+      "phone",
+      "smartphone"
+    ],
+    "iphone": [
+      "phone",
+      "smartphone"
+    ],
+    "phone": [
+      "android",
+      "iphone",
+      "smartphone"
+    ]
+  },
+  "distinctAttribute": null
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v3_to_v4__test__compat_v3_v4-9.snap

@@ -0,0 +1,31 @@
+---
+source: dump/src/reader/compat/v3_to_v4.rs
+expression: movies.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [
+    "genres",
+    "id"
+  ],
+  "sortableAttributes": [
+    "release_date"
+  ],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness",
+    "release_date:asc"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-12.snap

@@ -0,0 +1,56 @@
+---
+source: dump/src/reader/compat/v4_to_v5.rs
+expression: spells.settings().unwrap()
+---
+{
+  "displayedAttributes": "Reset",
+  "searchableAttributes": "Reset",
+  "filterableAttributes": {
+    "Set": []
+  },
+  "sortableAttributes": {
+    "Set": []
+  },
+  "rankingRules": {
+    "Set": [
+      "words",
+      "typo",
+      "proximity",
+      "attribute",
+      "sort",
+      "exactness"
+    ]
+  },
+  "stopWords": {
+    "Set": []
+  },
+  "synonyms": {
+    "Set": {}
+  },
+  "distinctAttribute": "Reset",
+  "typoTolerance": {
+    "Set": {
+      "enabled": {
+        "Set": true
+      },
+      "minWordSizeForTypos": {
+        "Set": {
+          "oneTypo": {
+            "Set": 5
+          },
+          "twoTypos": {
+            "Set": 9
+          }
+        }
+      },
+      "disableOnWords": {
+        "Set": []
+      },
+      "disableOnAttributes": {
+        "Set": []
+      }
+    }
+  },
+  "faceting": "NotSet",
+  "pagination": "NotSet"
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-6.snap

@@ -0,0 +1,70 @@
+---
+source: dump/src/reader/compat/v4_to_v5.rs
+expression: products.settings().unwrap()
+---
+{
+  "displayedAttributes": "Reset",
+  "searchableAttributes": "Reset",
+  "filterableAttributes": {
+    "Set": []
+  },
+  "sortableAttributes": {
+    "Set": []
+  },
+  "rankingRules": {
+    "Set": [
+      "words",
+      "typo",
+      "proximity",
+      "attribute",
+      "sort",
+      "exactness"
+    ]
+  },
+  "stopWords": {
+    "Set": []
+  },
+  "synonyms": {
+    "Set": {
+      "android": [
+        "phone",
+        "smartphone"
+      ],
+      "iphone": [
+        "phone",
+        "smartphone"
+      ],
+      "phone": [
+        "android",
+        "iphone",
+        "smartphone"
+      ]
+    }
+  },
+  "distinctAttribute": "Reset",
+  "typoTolerance": {
+    "Set": {
+      "enabled": {
+        "Set": true
+      },
+      "minWordSizeForTypos": {
+        "Set": {
+          "oneTypo": {
+            "Set": 5
+          },
+          "twoTypos": {
+            "Set": 9
+          }
+        }
+      },
+      "disableOnWords": {
+        "Set": []
+      },
+      "disableOnAttributes": {
+        "Set": []
+      }
+    }
+  },
+  "faceting": "NotSet",
+  "pagination": "NotSet"
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v4_to_v5__test__compat_v4_v5-9.snap

@@ -0,0 +1,62 @@
+---
+source: dump/src/reader/compat/v4_to_v5.rs
+expression: movies.settings().unwrap()
+---
+{
+  "displayedAttributes": "Reset",
+  "searchableAttributes": "Reset",
+  "filterableAttributes": {
+    "Set": [
+      "genres",
+      "id"
+    ]
+  },
+  "sortableAttributes": {
+    "Set": [
+      "release_date"
+    ]
+  },
+  "rankingRules": {
+    "Set": [
+      "words",
+      "typo",
+      "proximity",
+      "attribute",
+      "sort",
+      "exactness",
+      "release_date:asc"
+    ]
+  },
+  "stopWords": {
+    "Set": []
+  },
+  "synonyms": {
+    "Set": {}
+  },
+  "distinctAttribute": "Reset",
+  "typoTolerance": {
+    "Set": {
+      "enabled": {
+        "Set": true
+      },
+      "minWordSizeForTypos": {
+        "Set": {
+          "oneTypo": {
+            "Set": 5
+          },
+          "twoTypos": {
+            "Set": 9
+          }
+        }
+      },
+      "disableOnWords": {
+        "Set": []
+      },
+      "disableOnAttributes": {
+        "Set": []
+      }
+    }
+  },
+  "faceting": "NotSet",
+  "pagination": "NotSet"
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-12.snap

@@ -0,0 +1,40 @@
+---
+source: dump/src/reader/compat/v5_to_v6.rs
+expression: spells.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null,
+  "typoTolerance": {
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9
+    },
+    "disableOnWords": [],
+    "disableOnAttributes": []
+  },
+  "faceting": {
+    "maxValuesPerFacet": 100
+  },
+  "pagination": {
+    "maxTotalHits": 1000
+  }
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-6.snap

@@ -0,0 +1,54 @@
+---
+source: dump/src/reader/compat/v5_to_v6.rs
+expression: products.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {
+    "android": [
+      "phone",
+      "smartphone"
+    ],
+    "iphone": [
+      "phone",
+      "smartphone"
+    ],
+    "phone": [
+      "android",
+      "iphone",
+      "smartphone"
+    ]
+  },
+  "distinctAttribute": null,
+  "typoTolerance": {
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9
+    },
+    "disableOnWords": [],
+    "disableOnAttributes": []
+  },
+  "faceting": {
+    "maxValuesPerFacet": 100
+  },
+  "pagination": {
+    "maxTotalHits": 1000
+  }
+}

dump/src/reader/compat/snapshots/dump__reader__compat__v5_to_v6__test__compat_v5_v6-9.snap

@@ -0,0 +1,46 @@
+---
+source: dump/src/reader/compat/v5_to_v6.rs
+expression: movies.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [
+    "genres",
+    "id"
+  ],
+  "sortableAttributes": [
+    "release_date"
+  ],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness",
+    "release_date:asc"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null,
+  "typoTolerance": {
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9
+    },
+    "disableOnWords": [],
+    "disableOnAttributes": []
+  },
+  "faceting": {
+    "maxValuesPerFacet": 100
+  },
+  "pagination": {
+    "maxTotalHits": 1000
+  }
+}

dump/src/reader/compat/v1_to_v2.rs

@@ -0,0 +1,410 @@
+use std::str::FromStr;
+
+use super::v2_to_v3::CompatV2ToV3;
+use crate::reader::{v1, v2, Document};
+use crate::Result;
+
+pub struct CompatV1ToV2 {
+    pub from: v1::V1Reader,
+}
+
+impl CompatV1ToV2 {
+    pub fn new(v1: v1::V1Reader) -> Self {
+        Self { from: v1 }
+    }
+
+    pub fn to_v3(self) -> CompatV2ToV3 {
+        CompatV2ToV3::Compat(self)
+    }
+
+    pub fn version(&self) -> crate::Version {
+        self.from.version()
+    }
+
+    pub fn date(&self) -> Option<time::OffsetDateTime> {
+        self.from.date()
+    }
+
+    pub fn index_uuid(&self) -> Vec<v2::meta::IndexUuid> {
+        self.from
+            .index_uuid()
+            .into_iter()
+            .enumerate()
+            // we use the index of the index 😬 as UUID for the index, so that we can link the v2::Task to their index
+            .map(|(index, index_uuid)| v2::meta::IndexUuid {
+                uid: index_uuid.uid,
+                uuid: uuid::Uuid::from_u128(index as u128),
+            })
+            .collect()
+    }
+
+    pub fn indexes(&self) -> Result<impl Iterator<Item = Result<CompatIndexV1ToV2>> + '_> {
+        Ok(self.from.indexes()?.map(|index_reader| Ok(CompatIndexV1ToV2 { from: index_reader? })))
+    }
+
+    pub fn tasks(
+        &mut self,
+    ) -> Box<dyn Iterator<Item = Result<(v2::Task, Option<v2::UpdateFile>)>> + '_> {
+        // Convert an error here to an iterator yielding the error
+        let indexes = match self.from.indexes() {
+            Ok(indexes) => indexes,
+            Err(err) => return Box::new(std::iter::once(Err(err))),
+        };
+        let it = indexes.enumerate().flat_map(
+            move |(index, index_reader)| -> Box<dyn Iterator<Item = _>> {
+                let index_reader = match index_reader {
+                    Ok(index_reader) => index_reader,
+                    Err(err) => return Box::new(std::iter::once(Err(err))),
+                };
+                Box::new(
+                    index_reader
+                        .tasks()
+                        // Filter out the UpdateStatus::Customs variant that is not supported in v2
+                        // and enqueued tasks, that don't contain the necessary update file in v1
+                        .filter_map(move |task| -> Option<_> {
+                            let task = match task {
+                                Ok(task) => task,
+                                Err(err) => return Some(Err(err)),
+                            };
+                            Some(Ok((
+                                v2::Task {
+                                    uuid: uuid::Uuid::from_u128(index as u128),
+                                    update: Option::from(task)?,
+                                },
+                                None,
+                            )))
+                        }),
+                )
+            },
+        );
+        Box::new(it)
+    }
+}
+
+pub struct CompatIndexV1ToV2 {
+    pub from: v1::V1IndexReader,
+}
+
+impl CompatIndexV1ToV2 {
+    pub fn metadata(&self) -> &crate::IndexMetadata {
+        self.from.metadata()
+    }
+
+    pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Document>> + '_>> {
+        self.from.documents().map(|it| Box::new(it) as Box<dyn Iterator<Item = _>>)
+    }
+
+    pub fn settings(&mut self) -> Result<v2::settings::Settings<v2::settings::Checked>> {
+        Ok(v2::settings::Settings::<v2::settings::Unchecked>::from(self.from.settings()?).check())
+    }
+}
+
+impl From<v1::settings::Settings> for v2::Settings<v2::Unchecked> {
+    fn from(source: v1::settings::Settings) -> Self {
+        Self {
+            displayed_attributes: option_to_setting(source.displayed_attributes)
+                .map(|displayed| displayed.into_iter().collect()),
+            searchable_attributes: option_to_setting(source.searchable_attributes),
+            filterable_attributes: option_to_setting(source.attributes_for_faceting.clone())
+                .map(|filterable| filterable.into_iter().collect()),
+            sortable_attributes: option_to_setting(source.attributes_for_faceting)
+                .map(|sortable| sortable.into_iter().collect()),
+            ranking_rules: option_to_setting(source.ranking_rules).map(|ranking_rules| {
+                ranking_rules
+                    .into_iter()
+                    .filter_map(|ranking_rule| {
+                        match v1::settings::RankingRule::from_str(&ranking_rule) {
+                            Ok(ranking_rule) => {
+                                let criterion: Option<v2::settings::Criterion> =
+                                    ranking_rule.into();
+                                criterion.as_ref().map(ToString::to_string)
+                            }
+                            Err(()) => {
+                                log::warn!(
+                                    "Could not import the following ranking rule: `{}`.",
+                                    ranking_rule
+                                );
+                                None
+                            }
+                        }
+                    })
+                    .collect()
+            }),
+            stop_words: option_to_setting(source.stop_words),
+            synonyms: option_to_setting(source.synonyms),
+            distinct_attribute: option_to_setting(source.distinct_attribute),
+            _kind: std::marker::PhantomData,
+        }
+    }
+}
+
+fn option_to_setting<T>(opt: Option<Option<T>>) -> v2::Setting<T> {
+    match opt {
+        Some(Some(t)) => v2::Setting::Set(t),
+        None => v2::Setting::NotSet,
+        Some(None) => v2::Setting::Reset,
+    }
+}
+
+impl From<v1::update::UpdateStatus> for Option<v2::updates::UpdateStatus> {
+    fn from(source: v1::update::UpdateStatus) -> Self {
+        use v1::update::UpdateStatus as UpdateStatusV1;
+        use v2::updates::UpdateStatus as UpdateStatusV2;
+        Some(match source {
+            UpdateStatusV1::Enqueued { content } => {
+                log::warn!(
+                    "Cannot import task {} (importing enqueued tasks from v1 dumps is unsupported)",
+                    content.update_id
+                );
+                log::warn!("Task will be skipped in the queue of imported tasks.");
+
+                return None;
+            }
+            UpdateStatusV1::Failed { content } => UpdateStatusV2::Failed(v2::updates::Failed {
+                from: v2::updates::Processing {
+                    from: v2::updates::Enqueued {
+                        update_id: content.update_id,
+                        meta: Option::from(content.update_type)?,
+                        enqueued_at: content.enqueued_at,
+                        content: None,
+                    },
+                    started_processing_at: content.processed_at
+                        - std::time::Duration::from_secs_f64(content.duration),
+                },
+                error: v2::ResponseError {
+                    // error code is ignored by serialization, and so always default in deserialized v2 dumps
+                    // that's a good thing, because we don't have them in v1 dump 😅
+                    code: http::StatusCode::default(),
+                    message: content.error.unwrap_or_default(),
+                    // error codes are unchanged between v1 and v2
+                    error_code: content.error_code.unwrap_or_default(),
+                    // error types are unchanged between v1 and v2
+                    error_type: content.error_type.unwrap_or_default(),
+                    // error links are unchanged between v1 and v2
+                    error_link: content.error_link.unwrap_or_default(),
+                },
+                failed_at: content.processed_at,
+            }),
+            UpdateStatusV1::Processed { content } => {
+                UpdateStatusV2::Processed(v2::updates::Processed {
+                    success: match &content.update_type {
+                        v1::update::UpdateType::ClearAll => {
+                            v2::updates::UpdateResult::DocumentDeletion { deleted: u64::MAX }
+                        }
+                        v1::update::UpdateType::Customs => v2::updates::UpdateResult::Other,
+                        v1::update::UpdateType::DocumentsAddition { number } => {
+                            v2::updates::UpdateResult::DocumentsAddition(
+                                v2::updates::DocumentAdditionResult { nb_documents: *number },
+                            )
+                        }
+                        v1::update::UpdateType::DocumentsPartial { number } => {
+                            v2::updates::UpdateResult::DocumentsAddition(
+                                v2::updates::DocumentAdditionResult { nb_documents: *number },
+                            )
+                        }
+                        v1::update::UpdateType::DocumentsDeletion { number } => {
+                            v2::updates::UpdateResult::DocumentDeletion { deleted: *number as u64 }
+                        }
+                        v1::update::UpdateType::Settings { .. } => v2::updates::UpdateResult::Other,
+                    },
+                    processed_at: content.processed_at,
+                    from: v2::updates::Processing {
+                        from: v2::updates::Enqueued {
+                            update_id: content.update_id,
+                            meta: Option::from(content.update_type)?,
+                            enqueued_at: content.enqueued_at,
+                            content: None,
+                        },
+                        started_processing_at: content.processed_at
+                            - std::time::Duration::from_secs_f64(content.duration),
+                    },
+                })
+            }
+        })
+    }
+}
+
+impl From<v1::update::UpdateType> for Option<v2::updates::UpdateMeta> {
+    fn from(source: v1::update::UpdateType) -> Self {
+        Some(match source {
+            v1::update::UpdateType::ClearAll => v2::updates::UpdateMeta::ClearDocuments,
+            v1::update::UpdateType::Customs => {
+                log::warn!("Ignoring task with type 'Customs' that is no longer supported");
+                return None;
+            }
+            v1::update::UpdateType::DocumentsAddition { .. } => {
+                v2::updates::UpdateMeta::DocumentsAddition {
+                    method: v2::updates::IndexDocumentsMethod::ReplaceDocuments,
+                    format: v2::updates::UpdateFormat::Json,
+                    primary_key: None,
+                }
+            }
+            v1::update::UpdateType::DocumentsPartial { .. } => {
+                v2::updates::UpdateMeta::DocumentsAddition {
+                    method: v2::updates::IndexDocumentsMethod::UpdateDocuments,
+                    format: v2::updates::UpdateFormat::Json,
+                    primary_key: None,
+                }
+            }
+            v1::update::UpdateType::DocumentsDeletion { .. } => {
+                v2::updates::UpdateMeta::DeleteDocuments { ids: vec![] }
+            }
+            v1::update::UpdateType::Settings { settings } => {
+                v2::updates::UpdateMeta::Settings((*settings).into())
+            }
+        })
+    }
+}
+
+impl From<v1::settings::SettingsUpdate> for v2::Settings<v2::Unchecked> {
+    fn from(source: v1::settings::SettingsUpdate) -> Self {
+        let ranking_rules = v2::Setting::from(source.ranking_rules);
+
+        // go from the concrete types of v1 (RankingRule) to the concrete type of v2 (Criterion),
+        // and then back to string as this is what the settings manipulate
+        let ranking_rules = ranking_rules.map(|ranking_rules| {
+            ranking_rules
+                .into_iter()
+                // filter out the WordsPosition ranking rule that exists in v1 but not v2
+                .filter_map(Option::<v2::settings::Criterion>::from)
+                .map(|criterion| criterion.to_string())
+                .collect()
+        });
+
+        Self {
+            displayed_attributes: v2::Setting::from(source.displayed_attributes)
+                .map(|displayed_attributes| displayed_attributes.into_iter().collect()),
+            searchable_attributes: source.searchable_attributes.into(),
+            filterable_attributes: v2::Setting::from(source.attributes_for_faceting.clone())
+                .map(|attributes_for_faceting| attributes_for_faceting.into_iter().collect()),
+            sortable_attributes: v2::Setting::from(source.attributes_for_faceting)
+                .map(|attributes_for_faceting| attributes_for_faceting.into_iter().collect()),
+            ranking_rules,
+            stop_words: source.stop_words.into(),
+            synonyms: source.synonyms.into(),
+            distinct_attribute: source.distinct_attribute.into(),
+            _kind: std::marker::PhantomData,
+        }
+    }
+}
+
+impl From<v1::settings::RankingRule> for Option<v2::settings::Criterion> {
+    fn from(source: v1::settings::RankingRule) -> Self {
+        match source {
+            v1::settings::RankingRule::Typo => Some(v2::settings::Criterion::Typo),
+            v1::settings::RankingRule::Words => Some(v2::settings::Criterion::Words),
+            v1::settings::RankingRule::Proximity => Some(v2::settings::Criterion::Proximity),
+            v1::settings::RankingRule::Attribute => Some(v2::settings::Criterion::Attribute),
+            v1::settings::RankingRule::WordsPosition => {
+                log::warn!("Removing the 'WordsPosition' ranking rule that is no longer supported, please check the resulting ranking rules of your indexes");
+                None
+            }
+            v1::settings::RankingRule::Exactness => Some(v2::settings::Criterion::Exactness),
+            v1::settings::RankingRule::Asc(field_name) => {
+                Some(v2::settings::Criterion::Asc(field_name))
+            }
+            v1::settings::RankingRule::Desc(field_name) => {
+                Some(v2::settings::Criterion::Desc(field_name))
+            }
+        }
+    }
+}
+
+impl<T> From<v1::settings::UpdateState<T>> for v2::Setting<T> {
+    fn from(source: v1::settings::UpdateState<T>) -> Self {
+        match source {
+            v1::settings::UpdateState::Update(new_value) => v2::Setting::Set(new_value),
+            v1::settings::UpdateState::Clear => v2::Setting::Reset,
+            v1::settings::UpdateState::Nothing => v2::Setting::NotSet,
+        }
+    }
+}
+
+#[cfg(test)]
+pub(crate) mod test {
+    use std::fs::File;
+    use std::io::BufReader;
+
+    use flate2::bufread::GzDecoder;
+    use meili_snap::insta;
+    use tempfile::TempDir;
+
+    use super::*;
+
+    #[test]
+    fn compat_v1_v2() {
+        let dump = File::open("tests/assets/v1.dump").unwrap();
+        let dir = TempDir::new().unwrap();
+        let mut dump = BufReader::new(dump);
+        let gz = GzDecoder::new(&mut dump);
+        let mut archive = tar::Archive::new(gz);
+        archive.unpack(dir.path()).unwrap();
+
+        let mut dump = v1::V1Reader::open(dir).unwrap().to_v2();
+
+        // top level infos
+        assert_eq!(dump.date(), None);
+
+        // tasks
+        let tasks = dump.tasks().collect::<Result<Vec<_>>>().unwrap();
+        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"2298010973ee98cf4670787314176a3a");
+        assert_eq!(update_files.len(), 9);
+        assert!(update_files[..].iter().all(|u| u.is_none())); // no update file in dumps v1
+
+        // indexes
+        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        // the index are not ordered in any way by default
+        indexes.sort_by_key(|index| index.metadata().uid.to_string());
+
+        let mut products = indexes.pop().unwrap();
+        let mut movies = indexes.pop().unwrap();
+        let mut spells = indexes.pop().unwrap();
+        assert!(indexes.is_empty());
+
+        // products
+        insta::assert_json_snapshot!(products.metadata(), @r###"
+        {
+          "uid": "products",
+          "primaryKey": "sku",
+          "createdAt": "2022-10-02T13:23:39.976870431Z",
+          "updatedAt": "2022-10-02T13:27:54.353262482Z"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(products.settings().unwrap());
+        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(documents.len(), 10);
+        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
+
+        // movies
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        {
+          "uid": "movies",
+          "primaryKey": "id",
+          "createdAt": "2022-10-02T13:15:29.477512777Z",
+          "updatedAt": "2022-10-02T13:21:12.671204856Z"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(movies.settings().unwrap());
+        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(documents.len(), 10);
+        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5");
+
+        // spells
+        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        {
+          "uid": "dnd_spells",
+          "primaryKey": "index",
+          "createdAt": "2022-10-02T13:38:26.358882984Z",
+          "updatedAt": "2022-10-02T13:38:26.385609433Z"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(spells.settings().unwrap());
+        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(documents.len(), 10);
+        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"aa24c0cfc733d66c396237ad44263bed");
+    }
+}

dump/src/reader/compat/v2_to_v3.rs

@@ -4,22 +4,28 @@ use std::str::FromStr;
 use time::OffsetDateTime;
 use uuid::Uuid;
 
+use super::v1_to_v2::{CompatIndexV1ToV2, CompatV1ToV2};
 use super::v3_to_v4::CompatV3ToV4;
 use crate::reader::{v2, v3, Document};
 use crate::Result;
 
-pub struct CompatV2ToV3 {
-    pub from: v2::V2Reader,
+pub enum CompatV2ToV3 {
+    V2(v2::V2Reader),
+    Compat(CompatV1ToV2),
 }
 
 impl CompatV2ToV3 {
     pub fn new(v2: v2::V2Reader) -> CompatV2ToV3 {
-        CompatV2ToV3 { from: v2 }
+        CompatV2ToV3::V2(v2)
     }
 
     pub fn index_uuid(&self) -> Vec<v3::meta::IndexUuid> {
-        self.from
-            .index_uuid()
+        let v2_uuids = match self {
+            CompatV2ToV3::V2(from) => from.index_uuid(),
+            CompatV2ToV3::Compat(compat) => compat.index_uuid(),
+        };
+        v2_uuids
+            .into_iter()
             .into_iter()
             .map(|index| v3::meta::IndexUuid { uid: index.uid, uuid: index.uuid })
             .collect()
@@ -30,11 +36,17 @@ impl CompatV2ToV3 {
     }
 
     pub fn version(&self) -> crate::Version {
-        self.from.version()
+        match self {
+            CompatV2ToV3::V2(from) => from.version(),
+            CompatV2ToV3::Compat(compat) => compat.version(),
+        }
     }
 
     pub fn date(&self) -> Option<time::OffsetDateTime> {
-        self.from.date()
+        match self {
+            CompatV2ToV3::V2(from) => from.date(),
+            CompatV2ToV3::Compat(compat) => compat.date(),
+        }
     }
 
     pub fn instance_uid(&self) -> Result<Option<uuid::Uuid>> {
@@ -42,10 +54,18 @@ impl CompatV2ToV3 {
     }
 
     pub fn indexes(&self) -> Result<impl Iterator<Item = Result<CompatIndexV2ToV3>> + '_> {
-        Ok(self.from.indexes()?.map(|index_reader| -> Result<_> {
-            let compat = CompatIndexV2ToV3::new(index_reader?);
-            Ok(compat)
-        }))
+        Ok(match self {
+            CompatV2ToV3::V2(from) => Box::new(from.indexes()?.map(|index_reader| -> Result<_> {
+                let compat = CompatIndexV2ToV3::new(index_reader?);
+                Ok(compat)
+            }))
+                as Box<dyn Iterator<Item = Result<CompatIndexV2ToV3>> + '_>,
+            CompatV2ToV3::Compat(compat) => Box::new(compat.indexes()?.map(|index_reader| {
+                let compat = CompatIndexV2ToV3::Compat(Box::new(index_reader?));
+                Ok(compat)
+            }))
+                as Box<dyn Iterator<Item = Result<CompatIndexV2ToV3>> + '_>,
+        })
     }
 
     pub fn tasks(
@@ -54,11 +74,13 @@ impl CompatV2ToV3 {
         dyn Iterator<Item = Result<(v3::Task, Option<Box<dyn Iterator<Item = Result<Document>>>>)>>
             + '_,
     > {
-        let _indexes = self.from.index_uuid.clone();
+        let tasks = match self {
+            CompatV2ToV3::V2(from) => from.tasks(),
+            CompatV2ToV3::Compat(compat) => compat.tasks(),
+        };
 
         Box::new(
-            self.from
-                .tasks()
+            tasks
                 .map(move |task| {
                     task.map(|(task, content_file)| {
                         let task = v3::Task { uuid: task.uuid, update: task.update.into() };
@@ -76,27 +98,38 @@ impl CompatV2ToV3 {
     }
 }
 
-pub struct CompatIndexV2ToV3 {
-    from: v2::V2IndexReader,
+pub enum CompatIndexV2ToV3 {
+    V2(v2::V2IndexReader),
+    Compat(Box<CompatIndexV1ToV2>),
 }
 
 impl CompatIndexV2ToV3 {
     pub fn new(v2: v2::V2IndexReader) -> CompatIndexV2ToV3 {
-        CompatIndexV2ToV3 { from: v2 }
+        CompatIndexV2ToV3::V2(v2)
     }
 
     pub fn metadata(&self) -> &crate::IndexMetadata {
-        self.from.metadata()
+        match self {
+            CompatIndexV2ToV3::V2(from) => from.metadata(),
+            CompatIndexV2ToV3::Compat(compat) => compat.metadata(),
+        }
     }
 
     pub fn documents(&mut self) -> Result<Box<dyn Iterator<Item = Result<Document>> + '_>> {
-        self.from
-            .documents()
-            .map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>)
+        match self {
+            CompatIndexV2ToV3::V2(from) => from
+                .documents()
+                .map(|iter| Box::new(iter) as Box<dyn Iterator<Item = Result<Document>> + '_>),
+            CompatIndexV2ToV3::Compat(compat) => compat.documents(),
+        }
     }
 
     pub fn settings(&mut self) -> Result<v3::Settings<v3::Checked>> {
-        Ok(v3::Settings::<v3::Unchecked>::from(self.from.settings()?).check())
+        let settings = match self {
+            CompatIndexV2ToV3::V2(from) => from.settings()?,
+            CompatIndexV2ToV3::Compat(compat) => compat.settings()?,
+        };
+        Ok(v3::Settings::<v3::Unchecked>::from(settings).check())
     }
 }
 
@@ -328,28 +361,29 @@ impl From<String> for v3::Code {
     }
 }
 
-fn option_to_setting<T>(opt: Option<Option<T>>) -> v3::Setting<T> {
-    match opt {
-        Some(Some(t)) => v3::Setting::Set(t),
-        None => v3::Setting::NotSet,
-        Some(None) => v3::Setting::Reset,
+impl<A> From<v2::Setting<A>> for v3::Setting<A> {
+    fn from(setting: v2::Setting<A>) -> Self {
+        match setting {
+            v2::settings::Setting::Set(a) => v3::settings::Setting::Set(a),
+            v2::settings::Setting::Reset => v3::settings::Setting::Reset,
+            v2::settings::Setting::NotSet => v3::settings::Setting::NotSet,
+        }
     }
 }
 
 impl<T> From<v2::Settings<T>> for v3::Settings<v3::Unchecked> {
     fn from(settings: v2::Settings<T>) -> Self {
         v3::Settings {
-            displayed_attributes: option_to_setting(settings.displayed_attributes),
-            searchable_attributes: option_to_setting(settings.searchable_attributes),
-            filterable_attributes: option_to_setting(settings.filterable_attributes)
-                .map(|f| f.into_iter().collect()),
-            sortable_attributes: v3::Setting::NotSet,
-            ranking_rules: option_to_setting(settings.ranking_rules).map(|criteria| {
+            displayed_attributes: settings.displayed_attributes.into(),
+            searchable_attributes: settings.searchable_attributes.into(),
+            filterable_attributes: settings.filterable_attributes.into(),
+            sortable_attributes: settings.sortable_attributes.into(),
+            ranking_rules: v3::Setting::from(settings.ranking_rules).map(|criteria| {
                 criteria.into_iter().map(|criterion| patch_ranking_rules(&criterion)).collect()
             }),
-            stop_words: option_to_setting(settings.stop_words),
-            synonyms: option_to_setting(settings.synonyms),
-            distinct_attribute: option_to_setting(settings.distinct_attribute),
+            stop_words: settings.stop_words.into(),
+            synonyms: settings.synonyms.into(),
+            distinct_attribute: settings.distinct_attribute.into(),
             _kind: std::marker::PhantomData,
         }
     }
@@ -361,6 +395,7 @@ fn patch_ranking_rules(ranking_rule: &str) -> String {
         Ok(v2::settings::Criterion::Typo) => String::from("typo"),
         Ok(v2::settings::Criterion::Proximity) => String::from("proximity"),
         Ok(v2::settings::Criterion::Attribute) => String::from("attribute"),
+        Ok(v2::settings::Criterion::Sort) => String::from("sort"),
         Ok(v2::settings::Criterion::Exactness) => String::from("exactness"),
         Ok(v2::settings::Criterion::Asc(name)) => format!("{name}:asc"),
         Ok(v2::settings::Criterion::Desc(name)) => format!("{name}:desc"),
@@ -381,7 +416,6 @@ pub(crate) mod test {
     use super::*;
 
     #[test]
-    #[ignore]
     fn compat_v2_v3() {
         let dump = File::open("tests/assets/v2.dump").unwrap();
         let dir = TempDir::new().unwrap();
@@ -427,7 +461,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"54b3d7a0d96de35427d867fa17164a99");
+        insta::assert_json_snapshot!(products.settings().unwrap());
         let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
@@ -442,7 +476,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"ae7c5ade2243a553152dab2f354e9095");
+        insta::assert_json_snapshot!(movies.settings().unwrap());
         let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 110);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
@@ -457,7 +491,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @"1be82b894556d23953af557b6a328a58");
+        insta::assert_json_snapshot!(movies2.settings().unwrap());
         let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 0);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
@@ -472,7 +506,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"1be82b894556d23953af557b6a328a58");
+        insta::assert_json_snapshot!(spells.settings().unwrap());
         let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");

dump/src/reader/compat/v3_to_v4.rs

@@ -347,7 +347,6 @@ pub(crate) mod test {
     use super::*;
 
     #[test]
-    #[ignore]
     fn compat_v3_v4() {
         let dump = File::open("tests/assets/v3.dump").unwrap();
         let dir = TempDir::new().unwrap();
@@ -397,7 +396,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"d3402aff19b90acea9e9a07c466690aa");
+        insta::assert_json_snapshot!(products.settings().unwrap());
         let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
@@ -412,7 +411,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"687aaab250f01b55d57bc69aa313b581");
+        insta::assert_json_snapshot!(movies.settings().unwrap());
         let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 110);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
@@ -427,7 +426,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @"cd9fedbd7e3492831a94da62c90013ea");
+        insta::assert_json_snapshot!(movies2.settings().unwrap());
         let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 0);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
@@ -442,7 +441,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"cd9fedbd7e3492831a94da62c90013ea");
+        insta::assert_json_snapshot!(spells.settings().unwrap());
         let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");

dump/src/reader/compat/v4_to_v5.rs

@@ -383,7 +383,6 @@ pub(crate) mod test {
     use super::*;
 
     #[test]
-    #[ignore]
     fn compat_v4_v5() {
         let dump = File::open("tests/assets/v4.dump").unwrap();
         let dir = TempDir::new().unwrap();
@@ -430,7 +429,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"26947283836ee4cdf0974f82efcc5332");
+        insta::assert_json_snapshot!(products.settings().unwrap());
         let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
@@ -445,7 +444,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"156871410d17e23803d0c90ddc6a66cb");
+        insta::assert_json_snapshot!(movies.settings().unwrap());
         let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 110);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab");
@@ -460,7 +459,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"69c9916142612cf4a2da9b9ed9455e9e");
+        insta::assert_json_snapshot!(spells.settings().unwrap());
         let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");

dump/src/reader/compat/v5_to_v6.rs

@@ -1,3 +1,5 @@
+use std::str::FromStr;
+
 use super::v4_to_v5::{CompatIndexV4ToV5, CompatV4ToV5};
 use crate::reader::{v5, v6, Document, UpdateFile};
 use crate::Result;
@@ -119,11 +121,10 @@ impl CompatV5ToV6 {
                             allow_index_creation,
                             settings: Box::new(settings.into()),
                         },
-                        v5::tasks::TaskContent::Dump { uid } => v6::Kind::DumpCreation {
-                            dump_uid: uid,
-                            keys: keys.clone(),
-                            instance_uid,
-                        },
+                        v5::tasks::TaskContent::Dump { uid: _ } => {
+                            // in v6 we compute the dump_uid from the started_at processing time
+                            v6::Kind::DumpCreation { keys: keys.clone(), instance_uid }
+                        }
                     },
                     canceled_by: None,
                     details: task_view.details.map(|details| match details {
@@ -143,13 +144,15 @@ impl CompatV5ToV6 {
                             received_document_ids,
                             deleted_documents,
                         } => v6::Details::DocumentDeletion {
-                            matched_documents: received_document_ids,
+                            provided_ids: received_document_ids,
                             deleted_documents,
                         },
                         v5::Details::ClearAll { deleted_documents } => {
                             v6::Details::ClearAll { deleted_documents }
                         }
-                        v5::Details::Dump { dump_uid } => v6::Details::Dump { dump_uid },
+                        v5::Details::Dump { dump_uid } => {
+                            v6::Details::Dump { dump_uid: Some(dump_uid) }
+                        }
                     }),
                     error: task_view.error.map(|e| e.into()),
                     enqueued_at: task_view.enqueued_at,
@@ -178,10 +181,8 @@ impl CompatV5ToV6 {
                     .indexes
                     .into_iter()
                     .map(|index| match index {
-                        v5::StarOr::Star => v6::StarOr::Star,
-                        v5::StarOr::Other(uid) => {
-                            v6::StarOr::Other(v6::IndexUid::new_unchecked(uid.as_str()))
-                        }
+                        v5::StarOr::Star => v6::IndexUidPattern::all(),
+                        v5::StarOr::Other(uid) => v6::IndexUidPattern::new_unchecked(uid.as_str()),
                     })
                     .collect(),
                 expires_at: key.expires_at,
@@ -253,51 +254,50 @@ impl<T> From<v5::Setting<T>> for v6::Setting<T> {
 impl From<v5::ResponseError> for v6::ResponseError {
     fn from(error: v5::ResponseError) -> Self {
         let code = match error.error_code.as_ref() {
-            "index_creation_failed" => v6::Code::CreateIndex,
+            "index_creation_failed" => v6::Code::IndexCreationFailed,
             "index_already_exists" => v6::Code::IndexAlreadyExists,
             "index_not_found" => v6::Code::IndexNotFound,
             "invalid_index_uid" => v6::Code::InvalidIndexUid,
-            "invalid_min_word_length_for_typo" => v6::Code::InvalidMinWordLengthForTypo,
+            "invalid_min_word_length_for_typo" => v6::Code::InvalidSettingsTypoTolerance,
             "invalid_state" => v6::Code::InvalidState,
-            "primary_key_inference_failed" => v6::Code::MissingPrimaryKey,
-            "index_primary_key_already_exists" => v6::Code::PrimaryKeyAlreadyPresent,
+            "primary_key_inference_failed" => v6::Code::IndexPrimaryKeyNoCandidateFound,
+            "index_primary_key_already_exists" => v6::Code::IndexPrimaryKeyAlreadyExists,
             "max_fields_limit_exceeded" => v6::Code::MaxFieldsLimitExceeded,
             "missing_document_id" => v6::Code::MissingDocumentId,
             "invalid_document_id" => v6::Code::InvalidDocumentId,
-            "invalid_filter" => v6::Code::Filter,
-            "invalid_sort" => v6::Code::Sort,
+            "invalid_filter" => v6::Code::InvalidSettingsFilterableAttributes,
+            "invalid_sort" => v6::Code::InvalidSettingsSortableAttributes,
             "bad_parameter" => v6::Code::BadParameter,
             "bad_request" => v6::Code::BadRequest,
             "database_size_limit_reached" => v6::Code::DatabaseSizeLimitReached,
             "document_not_found" => v6::Code::DocumentNotFound,
             "internal" => v6::Code::Internal,
-            "invalid_geo_field" => v6::Code::InvalidGeoField,
-            "invalid_ranking_rule" => v6::Code::InvalidRankingRule,
-            "invalid_store_file" => v6::Code::InvalidStore,
-            "invalid_api_key" => v6::Code::InvalidToken,
+            "invalid_geo_field" => v6::Code::InvalidDocumentGeoField,
+            "invalid_ranking_rule" => v6::Code::InvalidSettingsRankingRules,
+            "invalid_store_file" => v6::Code::InvalidStoreFile,
+            "invalid_api_key" => v6::Code::InvalidApiKey,
             "missing_authorization_header" => v6::Code::MissingAuthorizationHeader,
             "no_space_left_on_device" => v6::Code::NoSpaceLeftOnDevice,
             "dump_not_found" => v6::Code::DumpNotFound,
             "task_not_found" => v6::Code::TaskNotFound,
             "payload_too_large" => v6::Code::PayloadTooLarge,
-            "unretrievable_document" => v6::Code::RetrieveDocument,
-            "search_error" => v6::Code::SearchDocuments,
+            "unretrievable_document" => v6::Code::UnretrievableDocument,
             "unsupported_media_type" => v6::Code::UnsupportedMediaType,
-            "dump_already_processing" => v6::Code::DumpAlreadyInProgress,
+            "dump_already_processing" => v6::Code::DumpAlreadyProcessing,
             "dump_process_failed" => v6::Code::DumpProcessFailed,
             "invalid_content_type" => v6::Code::InvalidContentType,
             "missing_content_type" => v6::Code::MissingContentType,
             "malformed_payload" => v6::Code::MalformedPayload,
             "missing_payload" => v6::Code::MissingPayload,
             "api_key_not_found" => v6::Code::ApiKeyNotFound,
-            "missing_parameter" => v6::Code::MissingParameter,
+            "missing_parameter" => v6::Code::BadRequest,
             "invalid_api_key_actions" => v6::Code::InvalidApiKeyActions,
             "invalid_api_key_indexes" => v6::Code::InvalidApiKeyIndexes,
             "invalid_api_key_expires_at" => v6::Code::InvalidApiKeyExpiresAt,
             "invalid_api_key_description" => v6::Code::InvalidApiKeyDescription,
             "invalid_api_key_name" => v6::Code::InvalidApiKeyName,
             "invalid_api_key_uid" => v6::Code::InvalidApiKeyUid,
-            "immutable_field" => v6::Code::ImmutableField,
+            "immutable_field" => v6::Code::BadRequest,
             "api_key_already_exists" => v6::Code::ApiKeyAlreadyExists,
             other => {
                 log::warn!("Unknown error code {}", other);
@@ -315,7 +315,26 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
             searchable_attributes: settings.searchable_attributes.into(),
             filterable_attributes: settings.filterable_attributes.into(),
             sortable_attributes: settings.sortable_attributes.into(),
-            ranking_rules: settings.ranking_rules.into(),
+            ranking_rules: {
+                match settings.ranking_rules {
+                    v5::settings::Setting::Set(ranking_rules) => {
+                        let mut new_ranking_rules = vec![];
+                        for rule in ranking_rules {
+                            match v6::RankingRuleView::from_str(&rule) {
+                                Ok(new_rule) => {
+                                    new_ranking_rules.push(new_rule);
+                                }
+                                Err(_) => {
+                                    log::warn!("Error while importing settings. The ranking rule `{rule}` does not exist anymore.")
+                                }
+                            }
+                        }
+                        v6::Setting::Set(new_ranking_rules)
+                    }
+                    v5::settings::Setting::Reset => v6::Setting::Reset,
+                    v5::settings::Setting::NotSet => v6::Setting::NotSet,
+                }
+            },
             stop_words: settings.stop_words.into(),
             synonyms: settings.synonyms.into(),
             distinct_attribute: settings.distinct_attribute.into(),
@@ -401,7 +420,6 @@ pub(crate) mod test {
     use super::*;
 
     #[test]
-    #[ignore]
     fn compat_v5_v6() {
         let dump = File::open("tests/assets/v5.dump").unwrap();
         let dir = TempDir::new().unwrap();
@@ -419,7 +437,7 @@ pub(crate) mod test {
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"42d4200cf6d92a6449989ca48cd8e28a");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"41f91d3a94911b2735ec41b07540df5c");
         assert_eq!(update_files.len(), 22);
         assert!(update_files[0].is_none()); // the dump creation
         assert!(update_files[1].is_some()); // the enqueued document addition
@@ -449,7 +467,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"8e5cadabf74aebe1160bf51c3d489efe");
+        insta::assert_json_snapshot!(products.settings().unwrap());
         let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
@@ -464,7 +482,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"4894ac1e74b9e1069ed5ee262b7a1aca");
+        insta::assert_json_snapshot!(movies.settings().unwrap());
         let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 200);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"e962baafd2fbae4cdd14e876053b0c5a");
@@ -479,7 +497,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"054dbf08a79e08bb9becba6f5d090f13");
+        insta::assert_json_snapshot!(spells.settings().unwrap());
         let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");

dump/src/reader/error.rs

@@ -1,42 +0,0 @@
-use meilisearch_auth::error::AuthControllerError;
-use meilisearch_types::error::{Code, ErrorCode};
-use meilisearch_types::internal_error;
-
-use crate::{index_resolver::error::IndexResolverError, tasks::error::TaskError};
-
-pub type Result<T> = std::result::Result<T, DumpError>;
-
-#[derive(thiserror::Error, Debug)]
-pub enum DumpError {
-    #[error("An internal error has occurred. `{0}`.")]
-    Internal(Box<dyn std::error::Error + Send + Sync + 'static>),
-    #[error("{0}")]
-    IndexResolver(Box<IndexResolverError>),
-}
-
-internal_error!(
-    DumpError: milli::heed::Error,
-    std::io::Error,
-    tokio::task::JoinError,
-    tokio::sync::oneshot::error::RecvError,
-    serde_json::error::Error,
-    tempfile::PersistError,
-    fs_extra::error::Error,
-    AuthControllerError,
-    TaskError
-);
-
-impl From<IndexResolverError> for DumpError {
-    fn from(e: IndexResolverError) -> Self {
-        Self::IndexResolver(Box::new(e))
-    }
-}
-
-impl ErrorCode for DumpError {
-    fn error_code(&self) -> Code {
-        match self {
-            DumpError::Internal(_) => Code::Internal,
-            DumpError::IndexResolver(e) => e.error_code(),
-        }
-    }
-}

dump/src/reader/mod.rs

@@ -9,11 +9,11 @@ use self::compat::v4_to_v5::CompatV4ToV5;
 use self::compat::v5_to_v6::{CompatIndexV5ToV6, CompatV5ToV6};
 use self::v5::V5Reader;
 use self::v6::{V6IndexReader, V6Reader};
-use crate::{Error, Result, Version};
+use crate::{Result, Version};
 
 mod compat;
 
-// pub(self) mod v1;
+pub(self) mod v1;
 pub(self) mod v2;
 pub(self) mod v3;
 pub(self) mod v4;
@@ -45,8 +45,9 @@ impl DumpReader {
         let MetadataVersion { dump_version } = serde_json::from_reader(&mut meta_file)?;
 
         match dump_version {
-            // Version::V1 => Ok(Box::new(v1::Reader::open(path)?)),
-            Version::V1 => Err(Error::DumpV1Unsupported),
+            Version::V1 => {
+                Ok(v1::V1Reader::open(path)?.to_v2().to_v3().to_v4().to_v5().to_v6().into())
+            }
             Version::V2 => Ok(v2::V2Reader::open(path)?.to_v3().to_v4().to_v5().to_v6().into()),
             Version::V3 => Ok(v3::V3Reader::open(path)?.to_v4().to_v5().to_v6().into()),
             Version::V4 => Ok(v4::V4Reader::open(path)?.to_v5().to_v6().into()),
@@ -189,7 +190,6 @@ pub(crate) mod test {
     use super::*;
 
     #[test]
-    #[ignore]
     fn import_dump_v5() {
         let dump = File::open("tests/assets/v5.dump").unwrap();
         let mut dump = DumpReader::open(dump).unwrap();
@@ -201,7 +201,7 @@ pub(crate) mod test {
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"42d4200cf6d92a6449989ca48cd8e28a");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"41f91d3a94911b2735ec41b07540df5c");
         assert_eq!(update_files.len(), 22);
         assert!(update_files[0].is_none()); // the dump creation
         assert!(update_files[1].is_some()); // the enqueued document addition
@@ -222,53 +222,52 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(products.metadata(), @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2022-10-04T15:51:35.939396731Z",
+          "updatedAt": "2022-10-04T15:55:01.897325373Z"
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"8e5cadabf74aebe1160bf51c3d489efe");
+        insta::assert_json_snapshot!(products.settings().unwrap());
         let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2022-10-04T15:51:35.291992167Z",
+          "updatedAt": "2022-10-04T15:55:10.33561842Z"
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"4894ac1e74b9e1069ed5ee262b7a1aca");
+        insta::assert_json_snapshot!(movies.settings().unwrap());
         let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 200);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"e962baafd2fbae4cdd14e876053b0c5a");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(spells.metadata(), @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2022-10-04T15:51:37.381094632Z",
+          "updatedAt": "2022-10-04T15:55:02.394503431Z"
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"054dbf08a79e08bb9becba6f5d090f13");
+        insta::assert_json_snapshot!(spells.settings().unwrap());
         let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
     }
 
     #[test]
-    #[ignore]
     fn import_dump_v4() {
         let dump = File::open("tests/assets/v4.dump").unwrap();
         let mut dump = DumpReader::open(dump).unwrap();
@@ -280,7 +279,7 @@ pub(crate) mod test {
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"491e244a80a19fe2a900b809d310c24a");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"c2445ddd1785528b80f2ba534d3bd00c");
         assert_eq!(update_files.len(), 10);
         assert!(update_files[0].is_some()); // the enqueued document addition
         assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
@@ -300,53 +299,52 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(products.metadata(), @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2022-10-06T12:53:39.360187055Z",
+          "updatedAt": "2022-10-06T12:53:40.603035979Z"
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"1f9da51a4518166fb440def5437eafdb");
+        insta::assert_json_snapshot!(products.settings().unwrap());
         let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2022-10-06T12:53:38.710611568Z",
+          "updatedAt": "2022-10-06T12:53:49.785862546Z"
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"488816aba82c1bd65f1609630055c611");
+        insta::assert_json_snapshot!(movies.settings().unwrap());
         let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 110);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"786022a66ecb992c8a2a60fee070a5ab");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        insta::assert_json_snapshot!(spells.metadata(), @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "[now]",
-          "updatedAt": "[now]"
+          "createdAt": "2022-10-06T12:53:40.831649057Z",
+          "updatedAt": "2022-10-06T12:53:41.116036186Z"
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"7b4f66dad597dc651650f35fe34be27f");
+        insta::assert_json_snapshot!(spells.settings().unwrap());
         let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
     }
 
     #[test]
-    #[ignore]
     fn import_dump_v3() {
         let dump = File::open("tests/assets/v3.dump").unwrap();
         let mut dump = DumpReader::open(dump).unwrap();
@@ -358,7 +356,7 @@ pub(crate) mod test {
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"7cacce2e21702be696b866808c726946");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"cd12efd308fe3ed226356a727ab42ed3");
         assert_eq!(update_files.len(), 10);
         assert!(update_files[0].is_some()); // the enqueued document addition
         assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
@@ -388,7 +386,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"855f3165dec609b919171ff83f82b364");
+        insta::assert_json_snapshot!(products.settings().unwrap());
         let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
@@ -403,7 +401,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"43e0bf1746c3ea1d64c1e10ea544c190");
+        insta::assert_json_snapshot!(movies.settings().unwrap());
         let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 110);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
@@ -418,7 +416,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @"5fd06a5038f49311600379d43412b655");
+        insta::assert_json_snapshot!(movies2.settings().unwrap());
         let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 0);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
@@ -433,14 +431,13 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"5fd06a5038f49311600379d43412b655");
+        insta::assert_json_snapshot!(spells.settings().unwrap());
         let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
     }
 
     #[test]
-    #[ignore]
     fn import_dump_v2() {
         let dump = File::open("tests/assets/v2.dump").unwrap();
         let mut dump = DumpReader::open(dump).unwrap();
@@ -452,7 +449,7 @@ pub(crate) mod test {
         // tasks
         let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
         let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"6cabec4e252b74c8f3a2c8517622e85f");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"bc616290adfe7d09a624cf6065ca9069");
         assert_eq!(update_files.len(), 9);
         assert!(update_files[0].is_some()); // the enqueued document addition
         assert!(update_files[1..].iter().all(|u| u.is_none())); // everything already processed
@@ -482,7 +479,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", products.settings()), @"b15b71f56dd082d8e8ec5182e688bf36");
+        insta::assert_json_snapshot!(products.settings().unwrap());
         let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
@@ -497,7 +494,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", movies.settings()), @"5389153ddf5527fa79c54b6a6e9c21f6");
+        insta::assert_json_snapshot!(movies.settings().unwrap());
         let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 110);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d153b5a81d8b3cdcbe1dec270b574022");
@@ -512,7 +509,7 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", movies2.settings()), @"8aebab01301d266acf3e18dd449c008f");
+        insta::assert_json_snapshot!(movies2.settings().unwrap());
         let documents = movies2.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 0);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
@@ -527,9 +524,162 @@ pub(crate) mod test {
         }
         "###);
 
-        meili_snap::snapshot_hash!(format!("{:#?}", spells.settings()), @"8aebab01301d266acf3e18dd449c008f");
+        insta::assert_json_snapshot!(spells.settings().unwrap());
         let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
     }
+
+    #[test]
+    fn import_dump_v2_from_meilisearch_v0_22_0_issue_3435() {
+        let dump = File::open("tests/assets/v2-v0.22.0.dump").unwrap();
+        let mut dump = DumpReader::open(dump).unwrap();
+
+        // top level infos
+        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-01-30 16:26:09.247261 +00:00:00");
+        assert_eq!(dump.instance_uid().unwrap(), None);
+
+        // tasks
+        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"2db37756d8af1fb7623436b76e8956a6");
+        assert_eq!(update_files.len(), 8);
+        assert!(update_files[0..].iter().all(|u| u.is_none())); // everything already processed
+
+        // keys
+        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"d751713988987e9331980363e24189ce");
+
+        // indexes
+        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        // the index are not ordered in any way by default
+        indexes.sort_by_key(|index| index.metadata().uid.to_string());
+
+        let mut products = indexes.pop().unwrap();
+        let mut movies = indexes.pop().unwrap();
+        let mut spells = indexes.pop().unwrap();
+        assert!(indexes.is_empty());
+
+        // products
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        {
+          "uid": "products",
+          "primaryKey": "sku",
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(products.settings().unwrap());
+        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(documents.len(), 10);
+        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
+
+        // movies
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        {
+          "uid": "movies",
+          "primaryKey": "id",
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(movies.settings().unwrap());
+        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(documents.len(), 10);
+        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"0227598af846e574139ee0b80e03a720");
+
+        // spells
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
+        {
+          "uid": "dnd_spells",
+          "primaryKey": "index",
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(spells.settings().unwrap());
+        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(documents.len(), 10);
+        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
+    }
+
+    #[test]
+    fn import_dump_v1() {
+        let dump = File::open("tests/assets/v1.dump").unwrap();
+        let mut dump = DumpReader::open(dump).unwrap();
+
+        // top level infos
+        assert_eq!(dump.date(), None);
+        assert_eq!(dump.instance_uid().unwrap(), None);
+
+        // tasks
+        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"8df6eab075a44b3c1af6b726f9fd9a43");
+        assert_eq!(update_files.len(), 9);
+        assert!(update_files[..].iter().all(|u| u.is_none())); // no update file in dump v1
+
+        // keys
+        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot!(meili_snap::json_string!(keys), @"[]");
+        meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"d751713988987e9331980363e24189ce");
+
+        // indexes
+        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        // the index are not ordered in any way by default
+        indexes.sort_by_key(|index| index.metadata().uid.to_string());
+
+        let mut products = indexes.pop().unwrap();
+        let mut movies = indexes.pop().unwrap();
+        let mut spells = indexes.pop().unwrap();
+        assert!(indexes.is_empty());
+
+        // products
+        insta::assert_json_snapshot!(products.metadata(), @r###"
+        {
+          "uid": "products",
+          "primaryKey": "sku",
+          "createdAt": "2022-10-02T13:23:39.976870431Z",
+          "updatedAt": "2022-10-02T13:27:54.353262482Z"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(products.settings().unwrap());
+        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(documents.len(), 10);
+        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
+
+        // movies
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        {
+          "uid": "movies",
+          "primaryKey": "id",
+          "createdAt": "2022-10-02T13:15:29.477512777Z",
+          "updatedAt": "2022-10-02T13:21:12.671204856Z"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(movies.settings().unwrap());
+        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(documents.len(), 10);
+        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5");
+
+        // spells
+        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        {
+          "uid": "dnd_spells",
+          "primaryKey": "index",
+          "createdAt": "2022-10-02T13:38:26.358882984Z",
+          "updatedAt": "2022-10-02T13:38:26.385609433Z"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(spells.settings().unwrap());
+        let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(documents.len(), 10);
+        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"aa24c0cfc733d66c396237ad44263bed");
+    }
 }

dump/src/reader/snapshots/dump__reader__test__import_dump_v1-11.snap

@@ -0,0 +1,24 @@
+---
+source: dump/src/reader/mod.rs
+expression: spells.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "typo",
+    "words",
+    "proximity",
+    "attribute",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v1-5.snap

@@ -0,0 +1,38 @@
+---
+source: dump/src/reader/mod.rs
+expression: products.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "typo",
+    "words",
+    "proximity",
+    "attribute",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {
+    "android": [
+      "phone",
+      "smartphone"
+    ],
+    "iphone": [
+      "phone",
+      "smartphone"
+    ],
+    "phone": [
+      "android",
+      "iphone",
+      "smartphone"
+    ]
+  },
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v1-8.snap

@@ -0,0 +1,31 @@
+---
+source: dump/src/reader/mod.rs
+expression: movies.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [
+    "genres",
+    "id"
+  ],
+  "sortableAttributes": [
+    "genres",
+    "id"
+  ],
+  "rankingRules": [
+    "typo",
+    "words",
+    "proximity",
+    "attribute",
+    "exactness",
+    "release_date:asc"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v2-11.snap

@@ -0,0 +1,23 @@
+---
+source: dump/src/reader/mod.rs
+expression: movies2.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v2-14.snap

@@ -0,0 +1,23 @@
+---
+source: dump/src/reader/mod.rs
+expression: spells.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v2-5.snap

@@ -0,0 +1,37 @@
+---
+source: dump/src/reader/mod.rs
+expression: products.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {
+    "android": [
+      "phone",
+      "smartphone"
+    ],
+    "iphone": [
+      "phone",
+      "smartphone"
+    ],
+    "phone": [
+      "android",
+      "iphone",
+      "smartphone"
+    ]
+  },
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v2-8.snap

@@ -0,0 +1,24 @@
+---
+source: dump/src/reader/mod.rs
+expression: movies.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "exactness",
+    "release_date:asc"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-11.snap

@@ -0,0 +1,25 @@
+---
+source: dump/src/reader/mod.rs
+expression: spells.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-5.snap

@@ -0,0 +1,39 @@
+---
+source: dump/src/reader/mod.rs
+expression: products.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {
+    "android": [
+      "phone",
+      "smartphone"
+    ],
+    "iphone": [
+      "phone",
+      "smartphone"
+    ],
+    "phone": [
+      "android",
+      "iphone",
+      "smartphone"
+    ]
+  },
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v2_from_meilisearch_v0_22_0_issue_3435-8.snap

@@ -0,0 +1,30 @@
+---
+source: dump/src/reader/mod.rs
+expression: movies.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [
+    "genres",
+    "id"
+  ],
+  "sortableAttributes": [
+    "release_date"
+  ],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "exactness",
+    "release_date:asc"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v3-11.snap

@@ -0,0 +1,25 @@
+---
+source: dump/src/reader/mod.rs
+expression: movies2.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v3-14.snap

@@ -0,0 +1,25 @@
+---
+source: dump/src/reader/mod.rs
+expression: spells.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v3-5.snap

@@ -0,0 +1,39 @@
+---
+source: dump/src/reader/mod.rs
+expression: products.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {
+    "android": [
+      "phone",
+      "smartphone"
+    ],
+    "iphone": [
+      "phone",
+      "smartphone"
+    ],
+    "phone": [
+      "android",
+      "iphone",
+      "smartphone"
+    ]
+  },
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v3-8.snap

@@ -0,0 +1,31 @@
+---
+source: dump/src/reader/mod.rs
+expression: movies.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [
+    "genres",
+    "id"
+  ],
+  "sortableAttributes": [
+    "release_date"
+  ],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness",
+    "release_date:asc"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v4-12.snap

@@ -0,0 +1,34 @@
+---
+source: dump/src/reader/mod.rs
+expression: spells.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null,
+  "typoTolerance": {
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9
+    },
+    "disableOnWords": [],
+    "disableOnAttributes": []
+  }
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v4-6.snap

@@ -0,0 +1,48 @@
+---
+source: dump/src/reader/mod.rs
+expression: products.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {
+    "android": [
+      "phone",
+      "smartphone"
+    ],
+    "iphone": [
+      "phone",
+      "smartphone"
+    ],
+    "phone": [
+      "android",
+      "iphone",
+      "smartphone"
+    ]
+  },
+  "distinctAttribute": null,
+  "typoTolerance": {
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9
+    },
+    "disableOnWords": [],
+    "disableOnAttributes": []
+  }
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v4-9.snap

@@ -0,0 +1,40 @@
+---
+source: dump/src/reader/mod.rs
+expression: movies.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [
+    "genres",
+    "id"
+  ],
+  "sortableAttributes": [
+    "release_date"
+  ],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness",
+    "release_date:asc"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null,
+  "typoTolerance": {
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9
+    },
+    "disableOnWords": [],
+    "disableOnAttributes": []
+  }
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v5-12.snap

@@ -0,0 +1,40 @@
+---
+source: dump/src/reader/mod.rs
+expression: spells.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null,
+  "typoTolerance": {
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9
+    },
+    "disableOnWords": [],
+    "disableOnAttributes": []
+  },
+  "faceting": {
+    "maxValuesPerFacet": 100
+  },
+  "pagination": {
+    "maxTotalHits": 1000
+  }
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v5-6.snap

@@ -0,0 +1,54 @@
+---
+source: dump/src/reader/mod.rs
+expression: products.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [],
+  "sortableAttributes": [],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness"
+  ],
+  "stopWords": [],
+  "synonyms": {
+    "android": [
+      "phone",
+      "smartphone"
+    ],
+    "iphone": [
+      "phone",
+      "smartphone"
+    ],
+    "phone": [
+      "android",
+      "iphone",
+      "smartphone"
+    ]
+  },
+  "distinctAttribute": null,
+  "typoTolerance": {
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9
+    },
+    "disableOnWords": [],
+    "disableOnAttributes": []
+  },
+  "faceting": {
+    "maxValuesPerFacet": 100
+  },
+  "pagination": {
+    "maxTotalHits": 1000
+  }
+}

dump/src/reader/snapshots/dump__reader__test__import_dump_v5-9.snap

@@ -0,0 +1,46 @@
+---
+source: dump/src/reader/mod.rs
+expression: movies.settings().unwrap()
+---
+{
+  "displayedAttributes": [
+    "*"
+  ],
+  "searchableAttributes": [
+    "*"
+  ],
+  "filterableAttributes": [
+    "genres",
+    "id"
+  ],
+  "sortableAttributes": [
+    "release_date"
+  ],
+  "rankingRules": [
+    "words",
+    "typo",
+    "proximity",
+    "attribute",
+    "sort",
+    "exactness",
+    "release_date:asc"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "distinctAttribute": null,
+  "typoTolerance": {
+    "enabled": true,
+    "minWordSizeForTypos": {
+      "oneTypo": 5,
+      "twoTypos": 9
+    },
+    "disableOnWords": [],
+    "disableOnAttributes": []
+  },
+  "faceting": {
+    "maxValuesPerFacet": 100
+  },
+  "pagination": {
+    "maxTotalHits": 1000
+  }
+}

dump/src/reader/v1/mod.rs

@@ -1,173 +1,262 @@
-use std::{
-    convert::Infallible,
-    fs::{self, File},
-    io::{BufRead, BufReader},
-    path::Path,
-};
+use std::fs::{self, File};
+use std::io::{BufRead, BufReader};
+use std::path::{Path, PathBuf};
 
+use serde::Deserialize;
 use tempfile::TempDir;
 use time::OffsetDateTime;
 
-use self::update::UpdateStatus;
-
-use super::{DumpReader, IndexReader};
-use crate::{Error, Result, Version};
+use super::compat::v1_to_v2::CompatV1ToV2;
+use super::Document;
+use crate::{IndexMetadata, Result, Version};
 
 pub mod settings;
 pub mod update;
-pub mod v1;
 
 pub struct V1Reader {
-    dump: TempDir,
-    metadata: v1::Metadata,
-    indexes: Vec<V1IndexReader>,
+    pub dump: TempDir,
+    pub db_version: String,
+    pub dump_version: crate::Version,
+    indexes: Vec<V1Index>,
 }
 
-struct V1IndexReader {
-    name: String,
+pub struct IndexUuid {
+    pub name: String,
+    pub uid: String,
+}
+pub type Task = self::update::UpdateStatus;
+
+struct V1Index {
+    metadata: IndexMetadataV1,
+    path: PathBuf,
+}
+
+impl V1Index {
+    pub fn new(path: PathBuf, metadata: Index) -> Self {
+        Self { metadata: metadata.into(), path }
+    }
+
+    pub fn open(&self) -> Result<V1IndexReader> {
+        V1IndexReader::new(&self.path, self.metadata.clone())
+    }
+
+    pub fn metadata(&self) -> &IndexMetadata {
+        &self.metadata.metadata
+    }
+}
+
+pub struct V1IndexReader {
+    metadata: IndexMetadataV1,
     documents: BufReader<File>,
     settings: BufReader<File>,
     updates: BufReader<File>,
-
-    current_update: Option<UpdateStatus>,
 }
 
 impl V1IndexReader {
-    pub fn new(name: String, path: &Path) -> Result<Self> {
-        let mut ret = V1IndexReader {
-            name,
+    pub fn new(path: &Path, metadata: IndexMetadataV1) -> Result<Self> {
+        Ok(V1IndexReader {
+            metadata,
             documents: BufReader::new(File::open(path.join("documents.jsonl"))?),
             settings: BufReader::new(File::open(path.join("settings.json"))?),
             updates: BufReader::new(File::open(path.join("updates.jsonl"))?),
-            current_update: None,
-        };
-        ret.next_update();
-
-        Ok(ret)
+        })
     }
 
-    pub fn next_update(&mut self) -> Result<Option<UpdateStatus>> {
-        let current_update = if let Some(line) = self.updates.lines().next() {
-            Some(serde_json::from_str(&line?)?)
-        } else {
-            None
-        };
+    pub fn metadata(&self) -> &IndexMetadata {
+        &self.metadata.metadata
+    }
 
-        Ok(std::mem::replace(&mut self.current_update, current_update))
+    pub fn documents(&mut self) -> Result<impl Iterator<Item = Result<Document>> + '_> {
+        Ok((&mut self.documents)
+            .lines()
+            .map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }))
+    }
+
+    pub fn settings(&mut self) -> Result<self::settings::Settings> {
+        Ok(serde_json::from_reader(&mut self.settings)?)
+    }
+
+    pub fn tasks(self) -> impl Iterator<Item = Result<Task>> {
+        self.updates.lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) })
     }
 }
 
 impl V1Reader {
     pub fn open(dump: TempDir) -> Result<Self> {
-        let mut meta_file = fs::read(dump.path().join("metadata.json"))?;
-        let metadata = serde_json::from_reader(&*meta_file)?;
+        let meta_file = fs::read(dump.path().join("metadata.json"))?;
+        let metadata: Metadata = serde_json::from_reader(&*meta_file)?;
 
         let mut indexes = Vec::new();
 
-        let entries = fs::read_dir(dump.path())?;
-        for entry in entries {
-            let entry = entry?;
-            if entry.file_type()?.is_dir() {
-                indexes.push(V1IndexReader::new(
-                    entry
-                        .file_name()
-                        .to_str()
-                        .ok_or(Error::BadIndexName)?
-                        .to_string(),
-                    &entry.path(),
-                )?);
-            }
+        for index in metadata.indexes.into_iter() {
+            let index_path = dump.path().join(&index.uid);
+            indexes.push(V1Index::new(index_path, index));
         }
 
         Ok(V1Reader {
             dump,
-            metadata,
             indexes,
+            db_version: metadata.db_version,
+            dump_version: metadata.dump_version,
         })
     }
 
-    fn next_update(&mut self) -> Result<Option<UpdateStatus>> {
-        if let Some((idx, _)) = self
-            .indexes
+    pub fn to_v2(self) -> CompatV1ToV2 {
+        CompatV1ToV2 { from: self }
+    }
+
+    pub fn index_uuid(&self) -> Vec<IndexUuid> {
+        self.indexes
             .iter()
-            .map(|index| index.current_update)
-            .enumerate()
-            .filter_map(|(idx, update)| update.map(|u| (idx, u)))
-            .min_by_key(|(_, update)| update.enqueued_at())
-        {
-            self.indexes[idx].next_update()
-        } else {
-            Ok(None)
+            .map(|index| IndexUuid {
+                name: index.metadata.name.to_owned(),
+                uid: index.metadata().uid.to_owned(),
+            })
+            .collect()
+    }
+
+    pub fn version(&self) -> Version {
+        Version::V1
+    }
+
+    pub fn date(&self) -> Option<OffsetDateTime> {
+        None
+    }
+
+    pub fn indexes(&self) -> Result<impl Iterator<Item = Result<V1IndexReader>> + '_> {
+        Ok(self.indexes.iter().map(|index| index.open()))
+    }
+}
+
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Index {
+    pub name: String,
+    pub uid: String,
+    #[serde(with = "time::serde::rfc3339")]
+    created_at: OffsetDateTime,
+    #[serde(with = "time::serde::rfc3339")]
+    updated_at: OffsetDateTime,
+    pub primary_key: Option<String>,
+}
+
+#[derive(Clone)]
+pub struct IndexMetadataV1 {
+    pub name: String,
+    pub metadata: crate::IndexMetadata,
+}
+
+impl From<Index> for IndexMetadataV1 {
+    fn from(index: Index) -> Self {
+        IndexMetadataV1 {
+            name: index.name,
+            metadata: crate::IndexMetadata {
+                uid: index.uid,
+                primary_key: index.primary_key,
+                created_at: index.created_at,
+                updated_at: index.updated_at,
+            },
         }
     }
 }
 
-impl IndexReader for &V1IndexReader {
-    type Document = serde_json::Map<String, serde_json::Value>;
-    type Settings = settings::Settings;
-
-    fn name(&self) -> &str {
-        todo!()
-    }
-
-    fn documents(&self) -> Result<Box<dyn Iterator<Item = Result<Self::Document>>>> {
-        todo!()
-    }
-
-    fn settings(&self) -> Result<Self::Settings> {
-        todo!()
-    }
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct Metadata {
+    pub indexes: Vec<Index>,
+    pub db_version: String,
+    pub dump_version: crate::Version,
 }
 
-impl DumpReader for V1Reader {
-    type Document = serde_json::Map<String, serde_json::Value>;
-    type Settings = settings::Settings;
+#[cfg(test)]
+pub(crate) mod test {
+    use std::fs::File;
+    use std::io::BufReader;
 
-    type Task = update::UpdateStatus;
-    type UpdateFile = Infallible;
+    use flate2::bufread::GzDecoder;
+    use meili_snap::insta;
+    use tempfile::TempDir;
 
-    type Key = Infallible;
+    use super::*;
 
-    fn date(&self) -> Option<OffsetDateTime> {
-        None
-    }
+    #[test]
+    fn read_dump_v1() {
+        let dump = File::open("tests/assets/v1.dump").unwrap();
+        let dir = TempDir::new().unwrap();
+        let mut dump = BufReader::new(dump);
+        let gz = GzDecoder::new(&mut dump);
+        let mut archive = tar::Archive::new(gz);
+        archive.unpack(dir.path()).unwrap();
 
-    fn version(&self) -> Version {
-        Version::V1
-    }
+        let dump = V1Reader::open(dir).unwrap();
 
-    fn indexes(
-        &self,
-    ) -> Result<
-        Box<
-            dyn Iterator<
-                Item = Result<
-                    Box<
-                        dyn super::IndexReader<
-                            Document = Self::Document,
-                            Settings = Self::Settings,
-                        >,
-                    >,
-                >,
-            >,
-        >,
-    > {
-        Ok(Box::new(self.indexes.iter().map(|index| {
-            let index = Box::new(index)
-                as Box<dyn IndexReader<Document = Self::Document, Settings = Self::Settings>>;
-            Ok(index)
-        })))
-    }
+        // top level infos
+        assert_eq!(dump.date(), None);
 
-    fn tasks(&self) -> Box<dyn Iterator<Item = Result<(Self::Task, Option<Self::UpdateFile>)>>> {
-        Box::new(std::iter::from_fn(|| {
-            self.next_update()
-                .transpose()
-                .map(|result| result.map(|task| (task, None)))
-        }))
-    }
+        // indexes
+        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
 
-    fn keys(&self) -> Box<dyn Iterator<Item = Result<Self::Key>>> {
-        Box::new(std::iter::empty())
+        let mut products = indexes.pop().unwrap();
+        let mut movies = indexes.pop().unwrap();
+        let mut dnd_spells = indexes.pop().unwrap();
+
+        assert!(indexes.is_empty());
+
+        // products
+        insta::assert_json_snapshot!(products.metadata(), @r###"
+        {
+          "uid": "products",
+          "primaryKey": "sku",
+          "createdAt": "2022-10-02T13:23:39.976870431Z",
+          "updatedAt": "2022-10-02T13:27:54.353262482Z"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(products.settings().unwrap());
+        let documents = products.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(documents.len(), 10);
+        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b01c8371aea4c7171af0d4d846a2bdca");
+
+        // products tasks
+        let tasks = products.tasks().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"91de507f206ad21964584021932ba7a7");
+
+        // movies
+        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        {
+          "uid": "movies",
+          "primaryKey": "id",
+          "createdAt": "2022-10-02T13:15:29.477512777Z",
+          "updatedAt": "2022-10-02T13:21:12.671204856Z"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(movies.settings().unwrap());
+        let documents = movies.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(documents.len(), 10);
+        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"b63dbed5bbc059f3e32bc471ae699bf5");
+
+        // movies tasks
+        let tasks = movies.tasks().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"55eef4de2bef7e84c5ce0bee47488f56");
+
+        // spells
+        insta::assert_json_snapshot!(dnd_spells.metadata(), @r###"
+        {
+          "uid": "dnd_spells",
+          "primaryKey": "index",
+          "createdAt": "2022-10-02T13:38:26.358882984Z",
+          "updatedAt": "2022-10-02T13:38:26.385609433Z"
+        }
+        "###);
+
+        insta::assert_json_snapshot!(dnd_spells.settings().unwrap());
+        let documents = dnd_spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
+        assert_eq!(documents.len(), 10);
+        meili_snap::snapshot_hash!(format!("{:#?}", documents), @"aa24c0cfc733d66c396237ad44263bed");
+
+        // spells tasks
+        let tasks = dnd_spells.tasks().collect::<Result<Vec<_>>>().unwrap();
+        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"836dd7d64d5ad20ad901c44b1b161a4c");
     }
 }

dump/src/reader/v1/settings.rs

@@ -1,6 +1,9 @@
 use std::collections::{BTreeMap, BTreeSet};
 use std::result::Result as StdResult;
+use std::str::FromStr;
 
+use once_cell::sync::Lazy;
+use regex::Regex;
 use serde::{Deserialize, Deserializer, Serialize};
 
 #[derive(Default, Clone, Serialize, Deserialize, Debug)]
@@ -53,6 +56,34 @@ pub enum RankingRule {
     Desc(String),
 }
 
+static ASC_DESC_REGEX: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r#"(asc|desc)\(([\w_-]+)\)"#).unwrap());
+
+impl FromStr for RankingRule {
+    type Err = ();
+
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match s {
+            "typo" => Self::Typo,
+            "words" => Self::Words,
+            "proximity" => Self::Proximity,
+            "attribute" => Self::Attribute,
+            "wordsPosition" => Self::WordsPosition,
+            "exactness" => Self::Exactness,
+            text => {
+                let caps = ASC_DESC_REGEX.captures(text).ok_or(())?;
+                let order = caps.get(1).unwrap().as_str();
+                let field_name = caps.get(2).unwrap().as_str();
+                match order {
+                    "asc" => Self::Asc(field_name.to_string()),
+                    "desc" => Self::Desc(field_name.to_string()),
+                    _ => return Err(()),
+                }
+            }
+        })
+    }
+}
+
 // Any value that is present is considered Some value, including null.
 fn deserialize_some<'de, T, D>(deserializer: D) -> StdResult<Option<T>, D::Error>
 where

dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-10.snap

@@ -0,0 +1,24 @@
+---
+source: dump/src/reader/v1/mod.rs
+expression: dnd_spells.settings().unwrap()
+---
+{
+  "rankingRules": [
+    "typo",
+    "words",
+    "proximity",
+    "attribute",
+    "wordsPosition",
+    "exactness"
+  ],
+  "distinctAttribute": null,
+  "searchableAttributes": [
+    "*"
+  ],
+  "displayedAttributes": [
+    "*"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "attributesForFaceting": []
+}

dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-2.snap

@@ -0,0 +1,38 @@
+---
+source: dump/src/reader/v1/mod.rs
+expression: products.settings().unwrap()
+---
+{
+  "rankingRules": [
+    "typo",
+    "words",
+    "proximity",
+    "attribute",
+    "wordsPosition",
+    "exactness"
+  ],
+  "distinctAttribute": null,
+  "searchableAttributes": [
+    "*"
+  ],
+  "displayedAttributes": [
+    "*"
+  ],
+  "stopWords": [],
+  "synonyms": {
+    "android": [
+      "phone",
+      "smartphone"
+    ],
+    "iphone": [
+      "phone",
+      "smartphone"
+    ],
+    "phone": [
+      "android",
+      "iphone",
+      "smartphone"
+    ]
+  },
+  "attributesForFaceting": []
+}

dump/src/reader/v1/snapshots/dump__reader__v1__test__read_dump_v1-6.snap

@@ -0,0 +1,28 @@
+---
+source: dump/src/reader/v1/mod.rs
+expression: movies.settings().unwrap()
+---
+{
+  "rankingRules": [
+    "typo",
+    "words",
+    "proximity",
+    "attribute",
+    "wordsPosition",
+    "exactness",
+    "asc(release_date)"
+  ],
+  "distinctAttribute": null,
+  "searchableAttributes": [
+    "*"
+  ],
+  "displayedAttributes": [
+    "*"
+  ],
+  "stopWords": [],
+  "synonyms": {},
+  "attributesForFaceting": [
+    "id",
+    "genres"
+  ]
+}

dump/src/reader/v1/update.rs

@@ -1,54 +1,8 @@
 use serde::{Deserialize, Serialize};
-use serde_json::Value;
 use time::OffsetDateTime;
 
 use super::settings::SettingsUpdate;
 
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub struct Update {
-    data: UpdateData,
-    #[serde(with = "time::serde::rfc3339")]
-    enqueued_at: OffsetDateTime,
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum UpdateData {
-    ClearAll,
-    Customs(Vec<u8>),
-    // (primary key, documents)
-    DocumentsAddition {
-        primary_key: Option<String>,
-        documents: Vec<serde_json::Map<String, Value>>,
-    },
-    DocumentsPartial {
-        primary_key: Option<String>,
-        documents: Vec<serde_json::Map<String, Value>>,
-    },
-    DocumentsDeletion(Vec<String>),
-    Settings(Box<SettingsUpdate>),
-}
-
-impl UpdateData {
-    pub fn update_type(&self) -> UpdateType {
-        match self {
-            UpdateData::ClearAll => UpdateType::ClearAll,
-            UpdateData::Customs(_) => UpdateType::Customs,
-            UpdateData::DocumentsAddition { documents, .. } => UpdateType::DocumentsAddition {
-                number: documents.len(),
-            },
-            UpdateData::DocumentsPartial { documents, .. } => UpdateType::DocumentsPartial {
-                number: documents.len(),
-            },
-            UpdateData::DocumentsDeletion(deletion) => UpdateType::DocumentsDeletion {
-                number: deletion.len(),
-            },
-            UpdateData::Settings(update) => UpdateType::Settings {
-                settings: update.clone(),
-            },
-        }
-    }
-}
-
 #[derive(Debug, Clone, Serialize, Deserialize)]
 #[serde(tag = "name")]
 pub enum UpdateType {