Update Dockerfile

Co-authored-by: Markus Machatschek <markus.machatschek@hey.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -23,8 +23,7 @@ A clear and concise description of what you expected to happen.
 **Screenshots**
 If applicable, add screenshots to help explain your problem.
 
-**Meilisearch version:**
-[e.g. v0.20.0]
+**Meilisearch version:** [e.g. v0.20.0]
 
 **Additional context**
 Additional information that may be relevant to the issue.

.github/ISSUE_TEMPLATE/sprint_issue.md

@@ -1,32 +0,0 @@
----
-name: New sprint issue
-about: ⚠️ Should only be used by the engine team ⚠️
-title: ''
-labels: ''
-assignees: ''
-
----
-
-Related product team resources: [PRD]() (_internal only_)
-Related product discussion:
-Related spec: WIP
-
-## Motivation
-
-<!---Copy/paste the information in PRD or briefly detail the product motivation. Ask product team if any hesitation.-->
-
-## Usage
-
-<!---Link to the public part of the PRD, or to the related product discussion for experimental features-->
-
-## TODO
-
-<!---Feel free to adapt this list with more technical/product steps-->
-
-- [ ] Release a prototype
-- [ ] If prototype validated, merge changes into `main`
-- [ ] Update the spec
-
-## Impacted teams
-
-<!---Ping the related teams. Ask for the engine manager if any hesitation-->

.github/scripts/check-release.sh

@@ -1,41 +1,24 @@
-#!/usr/bin/env bash
-set -eu -o pipefail
+#!/bin/bash
 
-check_tag() {
-    local expected=$1
-    local actual=$2
-    local filename=$3
-
-    if [[ $actual != $expected ]]; then
-        echo >&2 "Error: the current tag does not match the version in $filename: found $actual, expected $expected"
-        return 1
-    fi
+# check_tag $current_tag $file_tag $file_name
+function check_tag {
+  if [[ "$1" != "$2" ]]; then
+      echo "Error: the current tag does not match the version in Cargo.toml: found $2 - expected $1"
+      ret=1
+  fi
 }
 
-read_version() {
-    grep '^version = ' | cut -d \" -f 2
-}
-
-if [[ -z "${GITHUB_REF:-}" ]]; then
-    echo >&2 "Error: GITHUB_REF is not set"
-    exit 1
-fi
-
-if [[ ! "$GITHUB_REF" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+(-[a-z0-9]+)?$ ]]; then
-    echo >&2 "Error: GITHUB_REF is not a valid tag: $GITHUB_REF"
-    exit 1
-fi
-
-current_tag=${GITHUB_REF#refs/tags/v}
 ret=0
+current_tag=${GITHUB_REF#'refs/tags/v'}
 
-toml_tag="$(cat Cargo.toml | read_version)"
-check_tag "$current_tag" "$toml_tag" Cargo.toml || ret=1
+file_tag="$(grep '^version = ' Cargo.toml | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')"
+check_tag $current_tag $file_tag
 
-lock_tag=$(grep -A 1 '^name = "meilisearch-auth"' Cargo.lock | read_version)
-check_tag "$current_tag" "$lock_tag" Cargo.lock || ret=1
+lock_file='Cargo.lock'
+lock_tag=$(grep -A 1 'name = "meilisearch-auth"' $lock_file | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')
+check_tag $current_tag $lock_tag $lock_file
 
-if (( ret == 0 )); then
-    echo 'OK'
+if [[ "$ret" -eq 0 ]] ; then
+  echo 'OK'
 fi
 exit $ret

.github/uffizzi/Dockerfile

@@ -0,0 +1,48 @@
+# Compile
+FROM    rust:alpine3.16 AS compiler
+
+RUN     apk add -q --update-cache --no-cache build-base openssl-dev
+
+WORKDIR /meilisearch
+
+ARG     COMMIT_SHA
+ARG     COMMIT_DATE
+ARG     GIT_TAG
+ENV     COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE} VERGEN_GIT_SEMVER_LIGHTWEIGHT=${GIT_TAG}
+ENV     RUSTFLAGS="-C target-feature=-crt-static"
+
+COPY    . .
+RUN     set -eux; \
+        apkArch="$(apk --print-arch)"; \
+        if [ "$apkArch" = "aarch64" ]; then \
+            export JEMALLOC_SYS_WITH_LG_PAGE=16; \
+        fi && \
+        cargo build --release
+
+# Run
+FROM    uffizzi/ttyd:alpine
+
+ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+ENV     MEILI_SERVER_PROVIDER docker
+ENV     MEILI_NO_ANALYTICS true
+
+RUN     apk update --quiet \
+        && apk add -q --no-cache libgcc tini curl
+
+# add meilisearch to the `/bin` so you can run it from anywhere and it's easy
+# to find.
+COPY    --from=compiler /meilisearch/target/release/meilisearch /bin/meilisearch
+# To stay compatible with the older version of the container (pre v0.27.0) we're
+# going to symlink the meilisearch binary in the path to `/meilisearch`
+RUN     ln -s /bin/meilisearch /meilisearch
+
+# This directory should hold all the data related to meilisearch so we're going
+# to move our PWD in there.
+# We don't want to put the meilisearch binary
+WORKDIR /meili_data
+
+
+EXPOSE  7700/tcp
+
+ENTRYPOINT ["tini", "--"]
+CMD     ["ttyd", "/bin/zsh"]

.github/uffizzi/docker-compose.uffizzi.yml

@@ -0,0 +1,26 @@
+version: "3"
+
+x-uffizzi:
+  ingress:
+    service: nginx
+    port: 8081
+
+services:
+  meilisearch:
+    image: "${MEILISEARCH_IMAGE}"
+    restart: unless-stopped
+    ports:
+    - "7681:7681"
+    - "7700:7700"
+    deploy:
+      resources:
+        limits:
+          memory: 500M
+  
+  nginx:
+    image: nginx:alpine
+    restart: unless-stopped
+    ports:
+    - "8081:8081"
+    volumes:
+    - ./.github/uffizzi/nginx:/etc/nginx

.github/uffizzi/nginx/nginx.conf

@@ -0,0 +1,28 @@
+
+events {
+  worker_connections  4096;  ## Default: 1024
+}
+
+http {
+    map $http_upgrade $connection_upgrade {
+        default upgrade;
+        '' close;
+    }
+
+    server {
+        listen 8081;
+
+        location / {
+            proxy_pass http://localhost:7681;
+            proxy_http_version 1.1;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection $connection_upgrade;
+        }
+
+        location /meilisearch/ {
+            # rewrite /meilisearch/(.*) /$1  break;
+            proxy_pass http://localhost:7700/;
+        }
+    }
+}
+

.github/workflows/benchmarks-pr.yml

@@ -1,97 +0,0 @@
-name: Benchmarks (PR)
-on: issue_comment
-permissions:
-  issues: write
-
-env:
-  GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-
-jobs:
-  run-benchmarks-on-comment:
-    if: startsWith(github.event.comment.body, '/benchmark')
-    name: Run and upload benchmarks
-    runs-on: benchmarks
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      - name: Check for Command
-        id: command
-        uses: xt0rted/slash-command-action@v2
-        with:
-          command: benchmark
-          reaction-type: "eyes"
-          repo-token: ${{ env.GH_TOKEN }}
-
-      - uses: xt0rted/pull-request-comment-branch@v2
-        id: comment-branch
-        with:
-          repo_token: ${{ env.GH_TOKEN }}
-
-      - uses: actions/checkout@v3
-        if: success()
-        with:
-          fetch-depth: 0 # fetch full history to be able to get main commit sha
-          ref: ${{ steps.comment-branch.outputs.head_ref }}
-
-      # Set variables
-      - name: Set current branch name
-        shell: bash
-        run: echo "name=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT
-        id: current_branch
-      - name: Set normalized current branch name # Replace `/` by `_` in branch name to avoid issues when pushing to S3
-        shell: bash
-        run: echo "name=$(git rev-parse --abbrev-ref HEAD | tr '/' '_')" >> $GITHUB_OUTPUT
-        id: normalized_current_branch
-      - name: Set shorter commit SHA
-        shell: bash
-        run: echo "short=$(echo $GITHUB_SHA | cut -c1-8)" >> $GITHUB_OUTPUT
-        id: commit_sha
-      - name: Set file basename with format "dataset_branch_commitSHA"
-        shell: bash
-        run: echo "basename=$(echo ${{ steps.command.outputs.command-arguments }}_${{ steps.normalized_current_branch.outputs.name }}_${{ steps.commit_sha.outputs.short }})" >> $GITHUB_OUTPUT
-        id: file
-
-      # Run benchmarks
-      - name: Run benchmarks - Dataset ${{ steps.command.outputs.command-arguments }} - Branch ${{ steps.current_branch.outputs.name }} - Commit ${{ steps.commit_sha.outputs.short }}
-        run: |
-          cd benchmarks
-          cargo bench --bench ${{ steps.command.outputs.command-arguments }} -- --save-baseline ${{ steps.file.outputs.basename }}
-
-      # Generate critcmp files
-      - name: Install critcmp
-        uses: taiki-e/install-action@v2
-        with:
-          tool: critcmp
-      - name: Export cripcmp file
-        run: |
-          critcmp --export ${{ steps.file.outputs.basename }} > ${{ steps.file.outputs.basename }}.json
-
-      # Upload benchmarks
-      - name: Upload ${{ steps.file.outputs.basename }}.json to DO Spaces # DigitalOcean Spaces = S3
-        uses: BetaHuhn/do-spaces-action@v2
-        with:
-          access_key: ${{ secrets.DO_SPACES_ACCESS_KEY }}
-          secret_key: ${{ secrets.DO_SPACES_SECRET_KEY }}
-          space_name: ${{ secrets.DO_SPACES_SPACE_NAME }}
-          space_region: ${{ secrets.DO_SPACES_SPACE_REGION }}
-          source: ${{ steps.file.outputs.basename }}.json
-          out_dir: critcmp_results
-
-      # Compute the diff of the benchmarks and send a message on the GitHub PR
-      - name: Compute and send a message in the PR
-        env:
-          GITHUB_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-        run: |
-          set -x
-          export base_ref=$(git merge-base origin/main ${{ steps.comment-branch.outputs.head_ref }} | head -c8)
-          export base_filename=$(echo ${{ steps.command.outputs.command-arguments }}_main_${base_ref}.json)
-          echo 'Here are your benchmarks diff 👊' >> body.txt
-          echo '```' >> body.txt
-          ./benchmarks/scripts/compare.sh $base_filename ${{ steps.file.outputs.basename }}.json >> body.txt
-          echo '```' >> body.txt
-          gh pr comment ${{ steps.current_branch.outputs.name }} --body-file body.txt

.github/workflows/create-issue-dependencies.yml

@@ -0,0 +1,28 @@
+name: Create issue to upgrade dependencies
+on:
+  schedule:
+    # Run the first of the month, every 3 month
+    - cron: '0 0 1 */3 *'
+  workflow_dispatch:
+
+jobs:
+  create-issue:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Create an issue
+      uses: actions-ecosystem/action-create-issue@v1
+      with:
+        github_token: ${{ secrets.MEILI_BOT_GH_PAT }}
+        title: Upgrade dependencies
+        body: |
+          This issue is about updating Meilisearch dependencies:
+          - [ ] Cargo toml dependencies of Meilisearch; but also the main engine-team repositories that Meilisearch depends on (charabia, heed...)
+          - [ ] If new Rust versions have been released, update the Rust version in the Clippy job of this [GitHub Action file](./.github/workflows/rust.yml)
+
+          ⚠️ To avoid last minute bugs, this issue should only be done at the beginning of the sprint!
+
+          The GitHub action dependencies are managed by [Dependabot](./.github/dependabot.yml)
+        labels: |
+          dependencies
+          maintenance

.github/workflows/dependency-issue.yml

@@ -1,24 +0,0 @@
-name: Create issue to upgrade dependencies
-
-on:
-  schedule:
-    # Run the first of the month, every 6 month
-    - cron: '0 0 1 */6 *'
-  workflow_dispatch:
-
-jobs:
-  create-issue:
-    runs-on: ubuntu-latest
-    env:
-      ISSUE_TEMPLATE: issue-template.md
-      GH_TOKEN: ${{ secrets.MEILI_BOT_GH_PAT }}
-    steps:
-    - uses: actions/checkout@v3
-    - name: Download the issue template
-      run: curl -s https://raw.githubusercontent.com/meilisearch/engine-team/main/issue-templates/dependency-issue.md > $ISSUE_TEMPLATE
-    - name: Create issue
-      run: |
-        gh issue create \
-          --title 'Upgrade dependencies' \
-          --label 'dependencies,maintenance' \
-          --body-file $ISSUE_TEMPLATE

.github/workflows/fuzzer-indexing.yml

@@ -1,24 +0,0 @@
-name: Run the indexing fuzzer
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  fuzz:
-    name: Setup the action
-    runs-on: ubuntu-latest
-    timeout-minutes: 4320 # 72h
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: stable
-          override: true
-
-      # Run benchmarks
-      - name: Run the fuzzer
-        run: |
-          cargo run --release --bin fuzz-indexing

.github/workflows/manual_benchmarks.yml

@@ -1,4 +1,4 @@
-name: Benchmarks (manual)
+name: Benchmarks
 
 on:
   workflow_dispatch:
@@ -74,4 +74,4 @@ jobs:
           echo "${{ steps.file.outputs.basename }}.json has just been pushed."
           echo 'How to compare this benchmark with another one?'
           echo '  - Check the available files with: ./benchmarks/scripts/list.sh'
-          echo "  - Run the following command: ./benchmaks/scripts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"
+          echo "  - Run the following command: ./benchmaks/scipts/compare.sh <file-to-compare-with> ${{ steps.file.outputs.basename }}.json"

.github/workflows/publish-binaries.yml

@@ -1,5 +1,3 @@
-name: Publish binaries to GitHub release
-
 on:
   workflow_dispatch:
   schedule:
@@ -7,6 +5,8 @@ on:
   release:
     types: [published]
 
+name: Publish binaries to release
+
 jobs:
   check-version:
     name: Check the version validity
@@ -54,7 +54,7 @@ jobs:
     # No need to upload binaries for dry run (cron)
     - name: Upload binaries to release
       if: github.event_name == 'release'
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.4.0
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/release/meilisearch
@@ -87,7 +87,7 @@ jobs:
     # No need to upload binaries for dry run (cron)
     - name: Upload binaries to release
       if: github.event_name == 'release'
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.4.0
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/release/${{ matrix.artifact_name }}
@@ -96,12 +96,14 @@ jobs:
 
   publish-macos-apple-silicon:
     name: Publish binary for macOS silicon
-    runs-on: macos-12
+    runs-on: ${{ matrix.os }}
     needs: check-version
     strategy:
+      fail-fast: false
       matrix:
         include:
-          - target: aarch64-apple-darwin
+          - os: macos-12
+            target: aarch64-apple-darwin
             asset_name: meilisearch-macos-apple-silicon
     steps:
       - name: Checkout repository
@@ -121,7 +123,7 @@ jobs:
       - name: Upload the binary to release
         # No need to upload binaries for dry run (cron)
         if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.4.0
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/${{ matrix.target }}/release/meilisearch
@@ -130,29 +132,21 @@ jobs:
 
   publish-aarch64:
     name: Publish binary for aarch64
-    runs-on: ubuntu-latest
+    runs-on: ${{ matrix.os }}
     needs: check-version
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27
-      image: ubuntu:18.04
     strategy:
+      fail-fast: false
       matrix:
         include:
-          - target: aarch64-unknown-linux-gnu
+          - build: aarch64
+            os: ubuntu-18.04
+            target: aarch64-unknown-linux-gnu
+            linker: gcc-aarch64-linux-gnu
+            use-cross: true
             asset_name: meilisearch-linux-aarch64
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update -y && apt upgrade -y
-          apt-get install -y curl build-essential gcc-aarch64-linux-gnu
-      - name: Set up Docker for cross compilation
-        run: |
-          apt-get install -y curl apt-transport-https ca-certificates software-properties-common
-          curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
-          add-apt-repository "deb [arch=$(dpkg --print-architecture)] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable"
-          apt-get update -y && apt-get install -y docker-ce
       - name: Installing Rust toolchain
         uses: actions-rs/toolchain@v1
         with:
@@ -160,7 +154,15 @@ jobs:
           profile: minimal
           target: ${{ matrix.target }}
           override: true
+      - name: APT update
+        run: |
+          sudo apt update
+      - name: Install target specific tools
+        if: matrix.use-cross
+        run: |
+          sudo apt-get install -y ${{ matrix.linker }}
       - name: Configure target aarch64 GNU
+        if: matrix.target == 'aarch64-unknown-linux-gnu'
         ## Environment variable is not passed using env:
         ## LD gold won't work with MUSL
         # env:
@@ -174,16 +176,14 @@ jobs:
         uses: actions-rs/cargo@v1
         with:
           command: build
-          use-cross: true
+          use-cross: ${{ matrix.use-cross }}
           args: --release --target ${{ matrix.target }}
-        env:
-          CROSS_DOCKER_IN_DOCKER: true
       - name: List target output files
         run: ls -lR ./target
       - name: Upload the binary to release
         # No need to upload binaries for dry run (cron)
         if: github.event_name == 'release'
-        uses: svenstaro/upload-release-action@2.7.0
+        uses: svenstaro/upload-release-action@2.4.0
         with:
           repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
           file: target/${{ matrix.target }}/release/meilisearch

.github/workflows/publish-deb-brew-pkg.yml

@@ -1,4 +1,4 @@
-name: Publish to APT & Homebrew
+name: Publish to APT repository & Homebrew
 
 on:
   release:
@@ -35,7 +35,7 @@ jobs:
     - name: Build deb package
       run: cargo deb -p meilisearch -o target/debian/meilisearch.deb
     - name: Upload debian pkg to release
-      uses: svenstaro/upload-release-action@2.7.0
+      uses: svenstaro/upload-release-action@2.4.0
       with:
         repo_token: ${{ secrets.MEILI_BOT_GH_PAT }}
         file: target/debian/meilisearch.deb
@@ -53,6 +53,5 @@ jobs:
         uses: mislav/bump-homebrew-formula-action@v2
         with:
           formula-name: meilisearch
-          formula-path: Formula/m/meilisearch.rb
         env:
           COMMITTER_TOKEN: ${{ secrets.HOMEBREW_COMMITTER_TOKEN }}

.github/workflows/publish-docker-images.yml

@@ -1,5 +1,4 @@
-name: Publish images to Docker Hub
-
+---
 on:
   push:
     # Will run for every tag pushed except `latest`
@@ -13,6 +12,8 @@ on:
     - cron: '0 23 * * *' # Every day at 11:00pm
   workflow_dispatch:
 
+name: Publish tagged images to Docker Hub
+
 jobs:
   docker:
     runs-on: docker
@@ -57,10 +58,10 @@ jobs:
           echo "date=$commit_date" >> $GITHUB_OUTPUT
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to Docker Hub
         uses: docker/login-action@v2
@@ -70,7 +71,7 @@ jobs:
 
       - name: Docker meta
         id: meta
-        uses: docker/metadata-action@v5
+        uses: docker/metadata-action@v4
         with:
           images: getmeili/meilisearch
           # Prevent `latest` to be updated for each new tag pushed.
@@ -83,7 +84,7 @@ jobs:
             type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' && steps.check-tag-format.outputs.latest == 'true' }}
 
       - name: Build and push
-        uses: docker/build-push-action@v5
+        uses: docker/build-push-action@v4
         with:
           push: true
           platforms: linux/amd64,linux/arm64

.github/workflows/push_benchmarks_indexing.yml

@@ -1,4 +1,4 @@
-name: Benchmarks of indexing (push)
+name: Benchmarks indexing (push)
 
 on:
   push:

.github/workflows/push_benchmarks_search_geo.yml

@@ -1,4 +1,4 @@
-name: Benchmarks of search for geo (push)
+name: Benchmarks search geo (push)
 
 on:
   push:

.github/workflows/push_benchmarks_search_songs.yml

@@ -1,4 +1,4 @@
-name: Benchmarks of search for songs (push)
+name: Benchmarks search songs (push)
 
 on:
   push:

.github/workflows/push_benchmarks_search_wiki.yml

@@ -1,4 +1,4 @@
-name: Benchmarks of search for Wikipedia articles (push)
+name: Benchmarks search wikipedia articles (push)
 
 on:
   push:

.github/workflows/rust.yml

@@ -0,0 +1,145 @@
+name: Rust
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Everyday at 5:00am
+    - cron: '0 5 * * *'
+  pull_request:
+  push:
+    # trying and staging branches are for Bors config
+    branches:
+      - trying
+      - staging
+
+env:
+  CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  RUSTFLAGS: "-D warnings"
+
+jobs:
+  test-linux:
+    name: Tests on ubuntu-18.04
+    runs-on: ubuntu-latest
+    container:
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
+    steps:
+    - uses: actions/checkout@v3
+    - name: Install needed dependencies
+      run: |
+        apt-get update && apt-get install -y curl
+        apt-get install build-essential -y
+    - name: Run test with Rust stable
+      if: github.event_name != 'schedule'
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: stable
+        override: true
+    - name: Run test with Rust nightly
+      if: github.event_name == 'schedule'
+      uses: actions-rs/toolchain@v1
+      with:
+        toolchain: nightly
+        override: true
+    # Disable cache due to disk space issues with Windows workers in CI
+    # - name: Cache dependencies
+    #   uses: Swatinem/rust-cache@v2.2.0
+    - name: Run cargo check without any default features
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --locked --release --no-default-features --all
+    - name: Run cargo test
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --locked --release --all
+
+  test-others:
+    name: Tests on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-12, windows-2022]
+    steps:
+    - uses: actions/checkout@v3
+#     - name: Cache dependencies
+#       uses: Swatinem/rust-cache@v2.2.0
+    - name: Run cargo check without any default features
+      uses: actions-rs/cargo@v1
+      with:
+        command: build
+        args: --locked --release --no-default-features --all
+    - name: Run cargo test
+      uses: actions-rs/cargo@v1
+      with:
+        command: test
+        args: --locked --release --all
+
+  # We run tests in debug also, to make sure that the debug_assertions are hit
+  test-debug:
+    name: Run tests in debug
+    runs-on: ubuntu-latest
+    container:
+      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
+      image: ubuntu:18.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Install needed dependencies
+        run: |
+          apt-get update && apt-get install -y curl
+          apt-get install build-essential -y
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+      # - name: Cache dependencies
+      #   uses: Swatinem/rust-cache@v2.2.0
+      - name: Run tests in debug
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --locked --all
+
+  clippy:
+    name: Run Clippy
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: 1.67.0
+          override: true
+          components: clippy
+      # - name: Cache dependencies
+      #   uses: Swatinem/rust-cache@v2.2.0
+      - name: Run cargo clippy
+        uses: actions-rs/cargo@v1
+        with:
+          command: clippy
+          # allow unlined_format_args https://github.com/rust-lang/rust-clippy/issues/10087
+          args: --all-targets -- --deny warnings --allow clippy::uninlined_format_args
+
+  fmt:
+    name: Run Rustfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: nightly
+          override: true
+          components: rustfmt
+      # - name: Cache dependencies
+      #   uses: Swatinem/rust-cache@v2.2.0
+      - name: Run cargo fmt
+        # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
+        # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate
+        # we are going to create an empty file where rustfmt expects it.
+        run: |
+          echo -ne "\n" > benchmarks/benches/datasets_paths.rs
+          cargo fmt --all -- --check

.github/workflows/sdks-tests.yml

@@ -1,386 +0,0 @@
-# If any test fails, the engine team should ensure the "breaking" changes are expected and contact the integration team
-name: SDKs tests
-
-on:
-  workflow_dispatch:
-    inputs:
-      docker_image:
-        description: 'The Meilisearch Docker image used'
-        required: false
-        default: nightly
-  schedule:
-    - cron: "0 6 * * MON" # Every Monday at 6:00AM
-
-env:
-  MEILI_MASTER_KEY: 'masterKey'
-  MEILI_NO_ANALYTICS: 'true'
-  DISABLE_COVERAGE: 'true'
-
-jobs:
-  define-docker-image:
-    runs-on: ubuntu-latest
-    outputs:
-      docker-image: ${{ steps.define-image.outputs.docker-image }}
-    steps:
-      - uses: actions/checkout@v3
-      - name: Define the Docker image we need to use
-        id: define-image
-        run: |
-          event=${{ github.event_name }}
-          echo "docker-image=nightly" >> $GITHUB_OUTPUT
-          if [[ $event == 'workflow_dispatch' ]]; then
-            echo "docker-image=${{ github.event.inputs.docker_image }}" >> $GITHUB_OUTPUT
-          fi
-      - name: Docker image is ${{ steps.define-image.outputs.docker-image }}
-        run: echo "Docker image is ${{ steps.define-image.outputs.docker-image }}"
-
-##########
-## SDKs ##
-##########
-
-  meilisearch-dotnet-tests:
-    needs: define-docker-image
-    name: .NET SDK tests
-    runs-on: ubuntu-latest
-    env:
-      MEILISEARCH_VERSION: ${{ needs.define-docker-image.outputs.docker-image }}
-
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-dotnet
-      - name: Setup .NET Core
-        uses: actions/setup-dotnet@v3
-        with:
-          dotnet-version: "6.0.x"
-      - name: Install dependencies
-        run: dotnet restore
-      - name: Build
-        run: dotnet build --configuration Release --no-restore
-      - name: Meilisearch (latest version) setup with Docker
-        run: docker compose up -d
-      - name: Run tests
-        run: dotnet test --no-restore --verbosity normal
-
-  meilisearch-dart-tests:
-    needs: define-docker-image
-    name: Dart SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-dart
-      - uses: dart-lang/setup-dart@v1
-        with:
-          sdk: 3.1.1
-      - name: Install dependencies
-        run: dart pub get
-      - name: Run integration tests
-        run: dart test --concurrency=4
-
-  meilisearch-go-tests:
-    needs: define-docker-image
-    name: Go SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - name: Set up Go
-        uses: actions/setup-go@v4
-        with:
-          go-version: stable
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-go
-      - name: Get dependencies
-        run: |
-          go get -v -t -d ./...
-          if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-          fi
-      - name: Run integration tests
-        run: go test -v ./...
-
-  meilisearch-java-tests:
-    needs: define-docker-image
-    name: Java SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-java
-      - name: Set up Java
-        uses: actions/setup-java@v3
-        with:
-          java-version: 8
-          distribution: 'zulu'
-          cache: gradle
-      - name: Grant execute permission for gradlew
-        run: chmod +x gradlew
-      - name: Build and run unit and integration tests
-        run: ./gradlew build integrationTest
-
-  meilisearch-js-tests:
-    needs: define-docker-image
-    name: JS SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-js
-      - name: Setup node
-        uses: actions/setup-node@v3
-        with:
-          cache: 'yarn'
-      - name: Install dependencies
-        run: yarn --dev
-      - name: Run tests
-        run: yarn test
-      - name: Build project
-        run: yarn build
-      - name: Run ESM env
-        run: yarn test:env:esm
-      - name: Run Node.js env
-        run: yarn test:env:nodejs
-      - name: Run node typescript env
-        run: yarn test:env:node-ts
-      - name: Run Browser env
-        run: yarn test:env:browser
-
-  meilisearch-php-tests:
-    needs: define-docker-image
-    name: PHP SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-php
-      - name: Install PHP
-        uses: shivammathur/setup-php@v2
-      - name: Validate composer.json and composer.lock
-        run: composer validate
-      - name: Install dependencies
-        run: |
-          composer remove --dev friendsofphp/php-cs-fixer --no-update --no-interaction
-          composer update --prefer-dist --no-progress
-      - name: Run test suite - default HTTP client (Guzzle 7)
-        run: |
-          sh scripts/tests.sh
-          composer remove --dev guzzlehttp/guzzle http-interop/http-factory-guzzle
-
-  meilisearch-python-tests:
-    needs: define-docker-image
-    name: Python SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-python
-      - name: Set up Python
-        uses: actions/setup-python@v4
-      - name: Install pipenv
-        uses: dschep/install-pipenv-action@v1
-      - name: Install dependencies
-        run: pipenv install --dev --python=${{ matrix.python-version }}
-      - name: Test with pytest
-        run: pipenv run pytest
-
-  meilisearch-ruby-tests:
-    needs: define-docker-image
-    name: Ruby SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-ruby
-      - name: Set up Ruby 3
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3
-      - name: Install ruby dependencies
-        run: bundle install --with test
-      - name: Run test suite
-        run: bundle exec rspec
-
-  meilisearch-rust-tests:
-    needs: define-docker-image
-    name: Rust SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-rust
-      - name: Build
-        run: cargo build --verbose
-      - name: Run tests
-        run: cargo test --verbose
-
-  meilisearch-swift-tests:
-    needs: define-docker-image
-    name: Swift SDK tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-swift
-      - name: Run tests
-        run: swift test
-
-########################
-## FRONT-END PLUGINS ##
-########################
-
-  meilisearch-js-plugins-tests:
-    needs: define-docker-image
-    name: meilisearch-js-plugins tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-js-plugins
-      - name: Setup node
-        uses: actions/setup-node@v3
-        with:
-          cache: yarn
-      - name: Install dependencies
-        run: yarn install
-      - name: Run tests
-        run: yarn test
-      - name: Build all the playgrounds and the packages
-        run: yarn build
-
-########################
-## BACK-END PLUGINS ###
-########################
-
-  meilisearch-rails-tests:
-    needs: define-docker-image
-    name: meilisearch-rails tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-rails
-      - name: Set up Ruby 3
-        uses: ruby/setup-ruby@v1
-        with:
-          ruby-version: 3
-          bundler-cache: true
-      - name: Run tests
-        run: bundle exec rspec
-
-  meilisearch-symfony-tests:
-    needs: define-docker-image
-    name: meilisearch-symfony tests
-    runs-on: ubuntu-latest
-    services:
-      meilisearch:
-        image: getmeili/meilisearch:${{ needs.define-docker-image.outputs.docker-image }}
-        env:
-          MEILI_MASTER_KEY: ${{ env.MEILI_MASTER_KEY }}
-          MEILI_NO_ANALYTICS: ${{ env.MEILI_NO_ANALYTICS }}
-        ports:
-          - '7700:7700'
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          repository: meilisearch/meilisearch-symfony
-      - name: Install PHP
-        uses: shivammathur/setup-php@v2
-        with:
-          tools: composer:v2, flex
-      - name: Validate composer.json and composer.lock
-        run: composer validate
-      - name: Install dependencies
-        run: composer install --prefer-dist --no-progress --quiet
-      - name: Remove doctrine/annotations
-        run: composer remove --dev doctrine/annotations
-      - name: Run test suite
-        run: composer test:unit

.github/workflows/test-suite.yml

@@ -1,197 +0,0 @@
-name: Test suite
-
-on:
-  workflow_dispatch:
-  schedule:
-    # Everyday at 5:00am
-    - cron: '0 5 * * *'
-  pull_request:
-  push:
-    # trying and staging branches are for Bors config
-    branches:
-      - trying
-      - staging
-
-env:
-  CARGO_TERM_COLOR: always
-  RUST_BACKTRACE: 1
-  RUSTFLAGS: "-D warnings"
-
-jobs:
-  test-linux:
-    name: Tests on ubuntu-18.04
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update && apt-get install -y curl
-          apt-get install build-essential -y
-      - name: Setup test with Rust stable
-        if: github.event_name != 'schedule'
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-      - name: Setup test with Rust nightly
-        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: nightly
-          override: true
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
-      - name: Run cargo check without any default features
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --locked --release --no-default-features --all
-      - name: Run cargo test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked --release --all
-
-  test-others:
-    name: Tests on ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [macos-12, windows-2022]
-    steps:
-      - uses: actions/checkout@v3
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
-      - name: Run cargo check without any default features
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --locked --release --no-default-features --all
-      - name: Run cargo test
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked --release --all
-
-  test-all-features:
-    name: Tests all features
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update
-          apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-      - name: Run cargo build with all features
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --workspace --locked --release --all-features
-      - name: Run cargo test with all features
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --workspace --locked --release --all-features
-
-  test-disabled-tokenization:
-    name: Test disabled tokenization
-    runs-on: ubuntu-latest
-    container:
-      image: ubuntu:18.04
-    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update
-          apt-get install --assume-yes build-essential curl
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-      - name: Run cargo tree without default features and check lindera is not present
-        run: |
-          if cargo tree -f '{p} {f}' -e normal --no-default-features | grep -vqz lindera; then
-            echo "lindera has been found in the sources and it shouldn't"
-            exit 1
-          fi
-      - name: Run cargo tree with default features and check lindera is pressent
-        run: |
-          cargo tree -f '{p} {f}' -e normal | grep lindera -qz
-
-  # We run tests in debug also, to make sure that the debug_assertions are hit
-  test-debug:
-    name: Run tests in debug
-    runs-on: ubuntu-latest
-    container:
-      # Use ubuntu-18.04 to compile with glibc 2.27, which are the production expectations
-      image: ubuntu:18.04
-    steps:
-      - uses: actions/checkout@v3
-      - name: Install needed dependencies
-        run: |
-          apt-get update && apt-get install -y curl
-          apt-get install build-essential -y
-      - uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
-      - name: Run tests in debug
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --locked --all
-
-  clippy:
-    name: Run Clippy
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: 1.71.1
-          override: true
-          components: clippy
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
-      - name: Run cargo clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --all-targets -- --deny warnings
-
-  fmt:
-    name: Run Rustfmt
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - uses: actions-rs/toolchain@v1
-        with:
-          profile: minimal
-          toolchain: nightly
-          override: true
-          components: rustfmt
-      - name: Cache dependencies
-        uses: Swatinem/rust-cache@v2.6.2
-      - name: Run cargo fmt
-        # Since we never ran the `build.rs` script in the benchmark directory we are missing one auto-generated import file.
-        # Since we want to trigger (and fail) this action as fast as possible, instead of building the benchmark crate
-        # we are going to create an empty file where rustfmt expects it.
-        run: |
-          echo -ne "\n" > benchmarks/benches/datasets_paths.rs
-          cargo fmt --all -- --check

.github/workflows/uffizzi-build.yml

@@ -0,0 +1,100 @@
+name: Uffizzi - Build PR Image
+on:
+  pull_request:
+    types: [opened,synchronize,reopened,closed]
+
+jobs:
+  build-meilisearch:
+    name: Build and push `meilisearch`
+    runs-on: ubuntu-latest
+    outputs:
+      tags: ${{ steps.meta.outputs.tags }}
+    if: ${{ github.event.action != 'closed' }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Generate UUID image name
+        id: uuid
+        run: echo "UUID_TAG=$(uuidgen)" >> $GITHUB_ENV
+
+      - name: Docker metadata
+        id: meta
+        uses: docker/metadata-action@v3
+        with:
+          images: registry.uffizzi.com/${{ env.UUID_TAG }}
+          tags: | 
+            type=raw,value=60d
+
+      - name: Build Image
+        uses: docker/build-push-action@v3
+        with:
+          context: ./
+          file: .github/uffizzi/Dockerfile
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          push: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+  render-compose-file:
+    name: Render Docker Compose File
+    # Pass output of this workflow to another triggered by `workflow_run` event.
+    runs-on: ubuntu-latest
+    needs: 
+      - build-meilisearch
+    outputs:
+      compose-file-cache-key:  ${{ env.COMPOSE_FILE_HASH }}
+    steps:
+      - name: Checkout git repo
+        uses: actions/checkout@v3
+      - name: Render Compose File
+        run: |
+          MEILISEARCH_IMAGE=$(echo ${{ needs.build-meilisearch.outputs.tags }})
+          export MEILISEARCH_IMAGE
+          # Render simple template from environment variables.
+          envsubst < .github/uffizzi/docker-compose.uffizzi.yml > docker-compose.rendered.yml
+          cat docker-compose.rendered.yml
+      - name: Upload Rendered Compose File as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec 
+          path: docker-compose.rendered.yml
+          retention-days: 2
+      - name: Serialize PR Event to File
+        run:  |
+          cat << EOF > event.json
+          ${{ toJSON(github.event) }} 
+          
+          EOF
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2
+
+  delete-preview:
+    name: Call for Preview Deletion
+    runs-on: ubuntu-latest
+    if: ${{ github.event.action == 'closed' }}
+    steps:
+      # If this PR is closing, we will not render a compose file nor pass it to the next workflow.
+      - name: Serialize PR Event to File
+        run:  |
+          cat << EOF > event.json
+          ${{ toJSON(github.event) }} 
+          
+          EOF
+      - name: Upload PR Event as Artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: preview-spec
+          path: event.json
+          retention-days: 2

.github/workflows/uffizzi-preview-deploy.yml

@@ -0,0 +1,103 @@
+name: Uffizzi - Deploy Preview
+
+on:
+  workflow_run:
+    workflows:
+      - "Uffizzi - Build PR Image"
+    types:
+      - completed
+
+jobs:
+  cache-compose-file:
+    name: Cache Compose File
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    outputs:
+      compose-file-cache-key: ${{ env.COMPOSE_FILE_HASH }}
+      pr-number: ${{ env.PR_NUMBER }}
+      expected-url: ${{ env.EXPECTED_URL }}
+    steps:
+      - name: 'Download artifacts'
+        # Fetch output (zip archive) from the workflow run that triggered this workflow.
+        uses: actions/github-script@v6
+        with:
+          script: |
+            let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               run_id: context.payload.workflow_run.id,
+            });
+            let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => {
+              return artifact.name == "preview-spec"
+            })[0];
+            let download = await github.rest.actions.downloadArtifact({
+               owner: context.repo.owner,
+               repo: context.repo.repo,
+               artifact_id: matchArtifact.id,
+               archive_format: 'zip',
+            });
+            let fs = require('fs');
+            fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/preview-spec.zip`, Buffer.from(download.data));
+      
+      - name: 'Unzip artifact'
+        run: unzip preview-spec.zip
+      
+      - name: Read Event into ENV
+        run: |
+          echo 'EVENT_JSON<<EOF' >> $GITHUB_ENV
+          cat event.json >> $GITHUB_ENV
+          echo 'EOF' >> $GITHUB_ENV
+      
+      - name: Hash Rendered Compose File
+        id: hash
+        # If the previous workflow was triggered by a PR close event, we will not have a compose file artifact.
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        run: echo "COMPOSE_FILE_HASH=$(md5sum docker-compose.rendered.yml | awk '{ print $1 }')" >> $GITHUB_ENV
+      
+      - name: Cache Rendered Compose File
+        if: ${{ fromJSON(env.EVENT_JSON).action != 'closed' }}
+        uses: actions/cache@v3
+        with:
+          path: docker-compose.rendered.yml
+          key: ${{ env.COMPOSE_FILE_HASH }}
+
+      - name: Read PR Number From Event Object
+        id: pr
+        run: echo "PR_NUMBER=${{ fromJSON(env.EVENT_JSON).number }}" >> $GITHUB_ENV
+      
+      - name: DEBUG - Print Job Outputs
+        if: ${{ runner.debug }}
+        run: |
+          echo "PR number: ${{ env.PR_NUMBER }}"
+          echo "Compose file hash: ${{ env.COMPOSE_FILE_HASH }}"
+          cat event.json
+
+      - name: Add expected URL env var
+        if: ${{ runner.debug }}
+        run: |
+          REPO=$(echo ${{ github.repository }} | sed 's/\./+/g')
+          echo "EXPECTED_URL=${{ inputs.server }}/github.com/$REPO/pull/${{ env.PR_NUMBER }}" >> $GITHUB_ENV
+
+  deploy-uffizzi-preview:
+    name: Use Remote Workflow to Preview on Uffizzi
+    needs:
+      - cache-compose-file
+    uses: UffizziCloud/preview-action/.github/workflows/reusable.yaml@desc
+    with:
+      # If this workflow was triggered by a PR close event, cache-key will be an empty string
+      # and this reusable workflow will delete the preview deployment.
+      compose-file-cache-key: ${{ needs.cache-compose-file.outputs.compose-file-cache-key }}
+      compose-file-cache-path: docker-compose.rendered.yml
+      server: https://app.uffizzi.com
+      pr-number: ${{ needs.cache-compose-file.outputs.pr-number }}
+      description: |
+        The meilisearch preview environment contains a web terminal from where you can run the
+        `meilisearch` command. You should be able to access this instance of meilisearch running in
+        the preview from the link Meilisearch Endpoint link given below.
+         
+        Web Terminal Endpoint : ${{ needs.cache-compose-file.outputs.expected-url }}
+        Meilisearch Endpoint : ${{ needs.cache-compose-file.outputs.expected-url }}/meilisearch
+    permissions:
+      contents: read
+      pull-requests: write
+      id-token: write

.github/workflows/update-cargo-toml-version.yml

@@ -1,4 +1,4 @@
-name: Update Meilisearch version in Cargo.toml
+name: Update Meilisearch version in all Cargo.toml files
 
 on:
   workflow_dispatch:
@@ -14,7 +14,7 @@ env:
 
 jobs:
   update-version-cargo-toml:
-    name: Update version in Cargo.toml
+    name: Update version in Cargo.toml files
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -25,7 +25,7 @@ jobs:
           override: true
       - name: Install sd
         run: cargo install sd
-      - name: Update Cargo.toml file
+      - name: Update Cargo.toml files
         run: |
           raw_new_version=$(echo $NEW_VERSION | cut -d 'v' -f 2)
           new_string="version = \"$raw_new_version\""
@@ -35,13 +35,13 @@ jobs:
       - name: Commit and push the changes to the ${{ env.NEW_BRANCH }} branch
         uses: EndBug/add-and-commit@v9
         with:
-          message: "Update version for the next release (${{ env.NEW_VERSION }}) in Cargo.toml"
+          message: "Update version for the next release (${{ env.NEW_VERSION }}) in Cargo.toml files"
           new_branch: ${{ env.NEW_BRANCH }}
       - name: Create the PR pointing to ${{ github.ref_name }}
         run: |
           gh pr create \
-            --title "Update version for the next release ($NEW_VERSION) in Cargo.toml" \
-            --body '⚠️ This PR is automatically generated. Check the new version is the expected one and Cargo.lock has been updated before merging.' \
+            --title "Update version for the next release ($NEW_VERSION) in Cargo.toml files" \
+            --body '⚠️ This PR is automatically generated. Check the new version is the expected one before merging.' \
             --label 'skip changelog' \
             --milestone $NEW_VERSION \
             --base $GITHUB_REF_NAME

CONTRIBUTING.md

@@ -18,9 +18,9 @@ If Meilisearch does not offer optimized support for your language, please consid
 
 ## Assumptions
 
-1. **You're familiar with [GitHub](https://github.com) and the [Pull Requests (PR)](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests) workflow.**
-2. **You've read the Meilisearch [documentation](https://www.meilisearch.com/docs).**
-3. **You know about the [Meilisearch community on Discord](https://discord.meilisearch.com).
+1. **You're familiar with [GitHub](https://github.com) and the [Pull Requests](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)(PR) workflow.**
+2. **You've read the Meilisearch [documentation](https://docs.meilisearch.com).**
+3. **You know about the [Meilisearch community](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html).
    Please use this for help.**
 
 ## How to Contribute
@@ -120,9 +120,29 @@ The full Meilisearch release process is described in [this guide](https://github
 
 Depending on the developed feature, you might need to provide a prototyped version of Meilisearch to make it easier to test by the users.
 
-This happens in two steps:
-- [Release the prototype](https://github.com/meilisearch/engine-team/blob/main/resources/prototypes.md#how-to-publish-a-prototype)
-- [Communicate about it](https://github.com/meilisearch/engine-team/blob/main/resources/prototypes.md#communication)
+The prototype name must follow this convention: `prototype-X-Y` where
+- `X` is the feature name formatted in `kebab-case`. It should not end with a single number.
+- `Y` is the version of the prototype, starting from `0`.
+
+✅ Example: `prototype-auto-resize-0`. </br>
+❌ Bad example: `auto-resize-0`: lacks the `prototype` prefix. </br>
+❌ Bad example: `prototype-auto-resize`: lacks the version suffix. </br>
+❌ Bad example: `prototype-auto-resize-0-0`: feature name ends with a single number.
+
+Steps to create a prototype:
+
+1. In your terminal, go to the last commit of your branch (the one you want to provide as a prototype).
+2. Create a tag following the convention: `git tag prototype-X-Y`
+3. Run Meilisearch and check that its launch summary features a line: `Prototype: prototype-X-Y` (you may need to switch branches and back after tagging for this to work).
+3. Push the tag: `git push origin prototype-X-Y`
+4. Check the [Docker CI](https://github.com/meilisearch/meilisearch/actions/workflows/publish-docker-images.yml) is now running.
+
+🐳 Once the CI has finished to run (~1h30), a Docker image named `prototype-X-Y` will be available on [DockerHub](https://hub.docker.com/repository/docker/getmeili/meilisearch/general). People can use it with the following command: `docker run -p 7700:7700 -v $(pwd)/meili_data:/meili_data getmeili/meilisearch:prototype-X-Y`. <br>
+More information about [how to run Meilisearch with Docker](https://docs.meilisearch.com/learn/cookbooks/docker.html#download-meilisearch-with-docker).
+
+⚙️ However, no binaries will be created. If the users do not use Docker, they can go to the `prototype-X-Y` tag in the Meilisearch repository and compile from the source code.
+
+⚠️ When sharing a prototype with users, remind them to not use it in production. Prototypes are solely for test purposes.
 
 ### Release assets
 

Cargo.toml

@@ -13,12 +13,11 @@ members = [
     "filter-parser",
     "flatten-serde-json",
     "json-depth-checker",
-    "benchmarks",
-    "fuzzers",
+    "benchmarks"
 ]
 
 [workspace.package]
-version = "1.4.1"
+version = "1.0.0"
 authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
 description = "Meilisearch HTTP server"
 homepage = "https://meilisearch.com"

Dockerfile

@@ -1,7 +1,5 @@
 # Compile
-FROM    rust:alpine3.16 AS compiler
-
-RUN     apk add -q --update-cache --no-cache build-base openssl-dev
+FROM    rust:bullseye AS compiler
 
 WORKDIR /meilisearch
 
@@ -13,20 +11,22 @@ ENV     RUSTFLAGS="-C target-feature=-crt-static"
 
 COPY    . .
 RUN     set -eux; \
-        apkArch="$(apk --print-arch)"; \
-        if [ "$apkArch" = "aarch64" ]; then \
+        arch="$(dpkg --print-architecture)"; \
+        if [ "$arch" = "arm64" ]; then \
             export JEMALLOC_SYS_WITH_LG_PAGE=16; \
         fi && \
         cargo build --release
 
 # Run
-FROM    alpine:3.16
+FROM    debian:11.6
 
 ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
 ENV     MEILI_SERVER_PROVIDER docker
 
-RUN     apk update --quiet \
-        && apk add -q --no-cache libgcc tini curl
+RUN     set -ex; \
+        apt-get update -q; \
+        apt-get install -q -y --no-install-recommends tini; \
+        rm -rf /var/lib/apt/lists/*
 
 # add meilisearch to the `/bin` so you can run it from anywhere and it's easy
 # to find.

PROFILING.md

@@ -1,19 +0,0 @@
-# Profiling Meilisearch
-
-Search engine technologies are complex pieces of software that require thorough profiling tools. We chose to use [Puffin](https://github.com/EmbarkStudios/puffin), which the Rust gaming industry uses extensively. You can export and import the profiling reports using the top bar's _File_ menu options [in Puffin Viewer](https://github.com/embarkstudios/puffin#ui).
-
-![An example profiling with Puffin viewer](assets/profiling-example.png)
-
-## Profiling the Indexing Process
-
-When you enable [the `exportPuffinReports` experimental feature](https://www.meilisearch.com/docs/learn/experimental/overview) of Meilisearch, Puffin reports with the `.puffin` extension will be automatically exported to disk. When this option is enabled, the engine will automatically create a "frame" whenever it executes the `IndexScheduler::tick` method.
-
-[Puffin Viewer](https://github.com/EmbarkStudios/puffin/tree/main/puffin_viewer) is used to analyze the reports. Those reports show areas where Meilisearch spent time during indexing.
-
-Another piece of advice on the Puffin viewer UI interface is to consider the _Merge children with same ID_ option. It can hide the exact actual timings at which events were sent. Please turn it off when you see strange gaps on the Flamegraph. It can help.
-
-## Profiling the Search Process
-
-We still need to take the time to profile the search side of the engine with Puffin. It would require time to profile the filtering phase, query parsing, creation, and execution. We could even profile the Actix HTTP server.
-
-The only issue we see is the framing system. Puffin requires a global frame-based profiling phase, which collides with Meilisearch's ability to accept and answer multiple requests on different threads simultaneously.

README.md

@@ -1,105 +1,94 @@
 <p align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-light-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
-  </a>
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=logo#gh-dark-mode-only" target="_blank">
-    <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
-  </a>
+  <img src="assets/meilisearch-logo-light.svg?sanitize=true#gh-light-mode-only">
+  <img src="assets/meilisearch-logo-dark.svg?sanitize=true#gh-dark-mode-only">
 </p>
 
 <h4 align="center">
-  <a href="https://www.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Website</a> |
+  <a href="https://www.meilisearch.com">Website</a> |
   <a href="https://roadmap.meilisearch.com/tabs/1-under-consideration">Roadmap</a> |
-  <a href="https://www.meilisearch.com/pricing?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Meilisearch Cloud</a> |
-  <a href="https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Blog</a> |
-  <a href="https://www.meilisearch.com/docs?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Documentation</a> |
-  <a href="https://www.meilisearch.com/docs/faq?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">FAQ</a> |
-  <a href="https://discord.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=nav">Discord</a>
+  <a href="https://blog.meilisearch.com">Blog</a> |
+  <a href="https://docs.meilisearch.com">Documentation</a> |
+  <a href="https://docs.meilisearch.com/faq/">FAQ</a> |
+  <a href="https://discord.meilisearch.com">Discord</a>
 </h4>
 
 <p align="center">
   <a href="https://deps.rs/repo/github/meilisearch/meilisearch"><img src="https://deps.rs/repo/github/meilisearch/meilisearch/status.svg" alt="Dependency status"></a>
   <a href="https://github.com/meilisearch/meilisearch/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
-  <a href="https://ms-bors.herokuapp.com/repositories/52"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
+  <a href="https://app.bors.tech/repositories/26457"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
 </p>
 
 <p align="center">⚡ A lightning-fast search engine that fits effortlessly into your apps, websites, and workflow 🔍</p>
 
----
-
-### 🔥 On November 2nd, we are hosting our first-ever live demo and product updates for [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). Make sure to [register here](https://us06web.zoom.us/meeting/register/tZMlc-mqrjIsH912-HTRe-AaT-pp41bDe81a#/registration) and bring your questions for live Q&A!
-
----
-
 Meilisearch helps you shape a delightful search experience in a snap, offering features that work out-of-the-box to speed up your workflow.
 
 <p align="center" name="demo">
-  <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-light-mode-only" target="_blank">
+  <a href="https://where2watch.meilisearch.com/#gh-light-mode-only" target="_blank">
     <img src="assets/demo-light.gif#gh-light-mode-only" alt="A bright colored application for finding movies screening near the user">
   </a>
-  <a href="https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-gif#gh-dark-mode-only" target="_blank">
+  <a href="https://where2watch.meilisearch.com/#gh-dark-mode-only" target="_blank">
     <img src="assets/demo-dark.gif#gh-dark-mode-only" alt="A dark colored application for finding movies screening near the user">
   </a>
 </p>
 
-🔥 [**Try it!**](https://where2watch.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=demo-link) 🔥
+🔥 [**Try it!**](https://where2watch.meilisearch.com/) 🔥
 
 ## ✨ Features
 
 - **Search-as-you-type:** find search results in less than 50 milliseconds
-- **[Typo tolerance](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#typo-tolerance):** get relevant matches even when queries contain typos and misspellings
-- **[Filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features) and [faceted search](https://www.meilisearch.com/docs/learn/fine_tuning_results/faceted_search?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code
-- **[Sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** sort results based on price, date, or pretty much anything else your users need
-- **[Synonym support](https://www.meilisearch.com/docs/learn/getting_started/customizing_relevancy?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features#synonyms):** configure synonyms to include more relevant content in your search results
-- **[Geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** filter and sort documents based on geographic data
-- **[Extensive language support](https://www.meilisearch.com/docs/learn/what_is_meilisearch/language?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
-- **[Security management](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** control which users can access what data with API keys that allow fine-grained permissions handling
-- **[Multi-Tenancy](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** personalize search results for any number of application tenants
+- **[Typo tolerance](https://docs.meilisearch.com/learn/getting_started/customizing_relevancy.html#typo-tolerance):** get relevant matches even when queries contain typos and misspellings
+- **[Filtering and faceted search](https://docs.meilisearch.com/learn/advanced/filtering_and_faceted_search.html):** enhance your user's search experience with custom filters and build a faceted search interface in a few lines of code
+- **[Sorting](https://docs.meilisearch.com/learn/advanced/sorting.html):** sort results based on price, date, or pretty much anything else your users need
+- **[Synonym support](https://docs.meilisearch.com/learn/getting_started/customizing_relevancy.html#synonyms):** configure synonyms to include more relevant content in your search results
+- **[Geosearch](https://docs.meilisearch.com/learn/advanced/geosearch.html):** filter and sort documents based on geographic data
+- **[Extensive language support](https://docs.meilisearch.com/learn/what_is_meilisearch/language.html):** search datasets in any language, with optimized support for Chinese, Japanese, Hebrew, and languages using the Latin alphabet
+- **[Security management](https://docs.meilisearch.com/learn/security/master_api_keys.html):** control which users can access what data with API keys that allow fine-grained permissions handling
+- **[Multi-Tenancy](https://docs.meilisearch.com/learn/security/tenant_tokens.html):** personalize search results for any number of application tenants
 - **Highly Customizable:** customize Meilisearch to your specific needs or use our out-of-the-box and hassle-free presets
-- **[RESTful API](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=features):** integrate Meilisearch in your technical stack with our plugins and SDKs
+- **[RESTful API](https://docs.meilisearch.com/reference/api/overview.html):** integrate Meilisearch in your technical stack with our plugins and SDKs
 - **Easy to install, deploy, and maintain**
 
 ## 📖 Documentation
 
-You can consult Meilisearch's documentation at [https://www.meilisearch.com/docs](https://www.meilisearch.com/docs/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=docs).
+You can consult Meilisearch's documentation at [https://docs.meilisearch.com](https://docs.meilisearch.com/).
 
 ## 🚀 Getting started
 
-For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://www.meilisearch.com/docs/learn/getting_started/quick_start?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) guide.
+For basic instructions on how to set up Meilisearch, add documents to an index, and search for documents, take a look at our [Quick Start](https://docs.meilisearch.com/learn/getting_started/quick_start.html) guide.
 
-You may also want to check out [Meilisearch 101](https://www.meilisearch.com/docs/learn/getting_started/filtering_and_sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=get-started) for an introduction to some of Meilisearch's most popular features.
+You may also want to check out [Meilisearch 101](https://docs.meilisearch.com/learn/getting_started/filtering_and_sorting.html) for an introduction to some of Meilisearch's most popular features.
 
-## ⚡ Supercharge your Meilisearch experience
+## ☁️ Meilisearch cloud
 
-Say goodbye to server deployment and manual updates with [Meilisearch Cloud](https://www.meilisearch.com/cloud?utm_campaign=oss&utm_source=github&utm_medium=meilisearch). No credit card required.
+Let us manage your infrastructure so you can focus on integrating a great search experience. Try [Meilisearch Cloud](https://meilisearch.com/pricing) today.
 
 ## 🧰 SDKs & integration tools
 
 Install one of our SDKs in your project for seamless integration between Meilisearch and your favorite language or framework!
 
-Take a look at the complete [Meilisearch integration list](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-link).
+Take a look at the complete [Meilisearch integration list](https://docs.meilisearch.com/learn/what_is_meilisearch/sdks.html).
 
-[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://www.meilisearch.com/docs/learn/what_is_meilisearch/sdks?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=sdks-logos)
+[![Logos belonging to different languages and frameworks supported by Meilisearch, including React, Ruby on Rails, Go, Rust, and PHP](assets/integrations.png)](https://docs.meilisearch.com/learn/what_is_meilisearch/sdks.html)
 
 ## ⚙️ Advanced usage
 
-Experienced users will want to keep our [API Reference](https://www.meilisearch.com/docs/reference/api/overview?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) close at hand.
+Experienced users will want to keep our [API Reference](https://docs.meilisearch.com/reference/api) close at hand.
 
-We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://www.meilisearch.com/docs/learn/fine_tuning_results/filtering?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [sorting](https://www.meilisearch.com/docs/learn/fine_tuning_results/sorting?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [geosearch](https://www.meilisearch.com/docs/learn/fine_tuning_results/geosearch?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), [API keys](https://www.meilisearch.com/docs/learn/security/master_api_keys?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced), and [tenant tokens](https://www.meilisearch.com/docs/learn/security/tenant_tokens?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
+We also offer a wide range of dedicated guides to all Meilisearch features, such as [filtering](https://docs.meilisearch.com/learn/advanced/filtering_and_faceted_search.html), [sorting](https://docs.meilisearch.com/learn/advanced/sorting.html), [geosearch](https://docs.meilisearch.com/learn/advanced/geosearch.html), [API keys](https://docs.meilisearch.com/learn/security/master_api_keys.html), and [tenant tokens](https://docs.meilisearch.com/learn/security/tenant_tokens.html).
 
-Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://www.meilisearch.com/docs/learn/core_concepts/documents?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced) and [indexes](https://www.meilisearch.com/docs/learn/core_concepts/indexes?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=advanced).
+Finally, for more in-depth information, refer to our articles explaining fundamental Meilisearch concepts such as [documents](https://docs.meilisearch.com/learn/core_concepts/documents.html) and [indexes](https://docs.meilisearch.com/learn/core_concepts/indexes.html).
 
 ## 📊 Telemetry
 
-Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) whenever you want.
+Meilisearch collects **anonymized** data from users to help us improve our product. You can [deactivate this](https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html#how-to-disable-data-collection) whenever you want.
 
-To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
+To request deletion of collected data, please write to us at [privacy@meilisearch.com](mailto:privacy@meilisearch.com). Don't forget to include your `Instance UID` in the message, as this helps us quickly find and delete your data.
 
-If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://www.meilisearch.com/docs/learn/what_is_meilisearch/telemetry?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=telemetry#how-to-disable-data-collection) of our documentation.
+If you want to know more about the kind of data we collect and what we use it for, check the [telemetry section](https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html) of our documentation.
 
 ## 📫 Get in touch!
 
-Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/?utm_campaign=oss&utm_source=github&utm_medium=meilisearch&utm_content=contact)
+Meilisearch is a search engine created by [Meili](https://www.welcometothejungle.com/en/companies/meilisearch), a software development company based in France and with team members all over the world. Want to know more about us? [Check out our blog!](https://blog.meilisearch.com/)
 
 🗞 [Subscribe to our newsletter](https://meilisearch.us2.list-manage.com/subscribe?u=27870f7b71c908a8b359599fb&id=79582d828e) if you don't want to miss any updates! We promise we won't clutter your mailbox: we only send one edition every two months.
 
@@ -108,6 +97,7 @@ Meilisearch is a search engine created by [Meili](https://www.welcometothejungle
 - For feature requests, please visit our [product repository](https://github.com/meilisearch/product/discussions)
 - Found a bug? Open an [issue](https://github.com/meilisearch/meilisearch/issues)!
 - Want to be part of our Discord community? [Join us!](https://discord.gg/meilisearch)
+- For everything else, please check [this page listing some of the other places where you can find us](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html)
 
 Thank you for your support!
 

assets/prometheus-basic-scraper.yml

@@ -1,19 +0,0 @@
-global:
-  scrape_interval:     15s # By default, scrape targets every 15 seconds.
-
-  # Attach these labels to any time series or alerts when communicating with
-  # external systems (federation, remote storage, Alertmanager).
-  external_labels:
-    monitor: 'codelab-monitor'
-
-# A scrape configuration containing exactly one endpoint to scrape:
-# Here it's Prometheus itself.
-scrape_configs:
-  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
-  - job_name: 'meilisearch'
-
-    # Override the global default and scrape targets from this job every 5 seconds.
-    scrape_interval: 5s
-
-    static_configs:
-      - targets: ['localhost:7700']

benchmarks/Cargo.toml

@@ -11,27 +11,27 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.70"
-csv = "1.2.1"
-milli = { path = "../milli" }
-mimalloc = { version = "0.1.37", default-features = false }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
+anyhow = "1.0.65"
+csv = "1.1.6"
+milli = { path = "../milli", default-features = false }
+mimalloc = { version = "0.1.29", default-features = false }
+serde_json = { version = "1.0.85", features = ["preserve_order"] }
 
 [dev-dependencies]
-criterion = { version = "0.5.1", features = ["html_reports"] }
+criterion = { version = "0.4.0", features = ["html_reports"] }
 rand = "0.8.5"
 rand_chacha = "0.3.1"
 roaring = "0.10.1"
 
 [build-dependencies]
-anyhow = "1.0.70"
-bytes = "1.4.0"
+anyhow = "1.0.65"
+bytes = "1.2.1"
 convert_case = "0.6.0"
-flate2 = "1.0.25"
-reqwest = { version = "0.11.16", features = ["blocking", "rustls-tls"], default-features = false }
+flate2 = "1.0.24"
+reqwest = { version = "0.11.12", features = ["blocking", "rustls-tls"], default-features = false }
 
 [features]
-default = ["milli/all-tokenizations"]
+default = ["milli/default"]
 
 [[bench]]
 name = "search_songs"
@@ -48,3 +48,7 @@ harness = false
 [[bench]]
 name = "indexing"
 harness = false
+
+[[bench]]
+name = "formatting"
+harness = false

benchmarks/README.md

@@ -119,9 +119,9 @@ _[Download the `smol-wiki` dataset](https://milli-benchmarks.fra1.digitaloceansp
 
 ### Movies
 
-`movies` is a really small dataset we uses as our example in the [getting started](https://www.meilisearch.com/docs/learn/getting_started/quick_start)
+`movies` is a really small dataset we uses as our example in the [getting started](https://docs.meilisearch.com/learn/getting_started/)
 
-_[Download the `movies` dataset](https://www.meilisearch.com/movies.json)._
+_[Download the `movies` dataset](https://docs.meilisearch.com/movies.json)._
 
 
 ### All Countries

benchmarks/benches/formatting.rs

@@ -0,0 +1,67 @@
+use std::rc::Rc;
+
+use criterion::{criterion_group, criterion_main};
+use milli::tokenizer::TokenizerBuilder;
+use milli::{FormatOptions, MatcherBuilder, MatchingWord, MatchingWords};
+
+#[global_allocator]
+static ALLOC: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
+struct Conf<'a> {
+    name: &'a str,
+    text: &'a str,
+    matching_words: MatcherBuilder<'a, Vec<u8>>,
+}
+
+fn bench_formatting(c: &mut criterion::Criterion) {
+    #[rustfmt::skip]
+    let confs = &[
+    	Conf {
+    		name: "'the door d'",
+			text: r#"He used to do the door sounds in "Star Trek" with his mouth, phssst, phssst. The MD-11 passenger and cargo doors also tend to behave like electromagnetic apertures, because the doors do not have continuous electrical contact with the door frames around the door perimeter. But Theodor said that the doors don't work."#,
+			matching_words: MatcherBuilder::new(MatchingWords::new(vec![
+	            (vec![Rc::new(MatchingWord::new("t".to_string(), 0, false).unwrap()), Rc::new(MatchingWord::new("he".to_string(), 0, false).unwrap())], vec![0]),
+	            (vec![Rc::new(MatchingWord::new("the".to_string(), 0, false).unwrap())], vec![0]),
+	            (vec![Rc::new(MatchingWord::new("door".to_string(), 1, false).unwrap())], vec![1]),
+	            (vec![Rc::new(MatchingWord::new("do".to_string(), 0, false).unwrap()), Rc::new(MatchingWord::new("or".to_string(), 0, false).unwrap())], vec![0]),
+	            (vec![Rc::new(MatchingWord::new("thedoor".to_string(), 1, false).unwrap())], vec![0, 1]),
+	            (vec![Rc::new(MatchingWord::new("d".to_string(), 0, true).unwrap())], vec![2]),
+	            (vec![Rc::new(MatchingWord::new("thedoord".to_string(), 1, true).unwrap())], vec![0, 1, 2]),
+	            (vec![Rc::new(MatchingWord::new("doord".to_string(), 1, true).unwrap())], vec![1, 2]),
+        	]
+            ), TokenizerBuilder::default().build()),
+		},
+    ];
+
+    let format_options = &[
+        FormatOptions { highlight: false, crop: None },
+        FormatOptions { highlight: true, crop: None },
+        FormatOptions { highlight: false, crop: Some(10) },
+        FormatOptions { highlight: true, crop: Some(10) },
+        FormatOptions { highlight: false, crop: Some(20) },
+        FormatOptions { highlight: true, crop: Some(20) },
+    ];
+
+    for option in format_options {
+        let highlight = if option.highlight { "highlight" } else { "no-highlight" };
+
+        let name = match option.crop {
+            Some(size) => format!("{}-crop({})", highlight, size),
+            None => format!("{}-no-crop", highlight),
+        };
+
+        let mut group = c.benchmark_group(&name);
+        for conf in confs {
+            group.bench_function(conf.name, |b| {
+                b.iter(|| {
+                    let mut matcher = conf.matching_words.build(conf.text);
+                    matcher.format(*option);
+                })
+            });
+        }
+        group.finish();
+    }
+}
+
+criterion_group!(benches, bench_formatting);
+criterion_main!(benches);

benchmarks/benches/indexing.rs

@@ -6,7 +6,9 @@ use std::path::Path;
 
 use criterion::{criterion_group, criterion_main, Criterion};
 use milli::heed::{EnvOpenOptions, RwTxn};
-use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings};
+use milli::update::{
+    DeleteDocuments, IndexDocuments, IndexDocumentsConfig, IndexerConfig, Settings,
+};
 use milli::Index;
 use rand::seq::SliceRandom;
 use rand_chacha::rand_core::SeedableRng;
@@ -264,7 +266,17 @@ fn deleting_songs_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                delete_documents_from_ids(index, document_ids_to_delete)
+                let mut wtxn = index.write_txn().unwrap();
+
+                for ids in document_ids_to_delete {
+                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
+                    builder.delete_documents(&ids);
+                    builder.execute().unwrap();
+                }
+
+                wtxn.commit().unwrap();
+
+                index.prepare_for_closing().wait();
             },
         )
     });
@@ -601,7 +613,17 @@ fn deleting_wiki_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                delete_documents_from_ids(index, document_ids_to_delete)
+                let mut wtxn = index.write_txn().unwrap();
+
+                for ids in document_ids_to_delete {
+                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
+                    builder.delete_documents(&ids);
+                    builder.execute().unwrap();
+                }
+
+                wtxn.commit().unwrap();
+
+                index.prepare_for_closing().wait();
             },
         )
     });
@@ -853,31 +875,22 @@ fn deleting_movies_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                delete_documents_from_ids(index, document_ids_to_delete)
+                let mut wtxn = index.write_txn().unwrap();
+
+                for ids in document_ids_to_delete {
+                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
+                    builder.delete_documents(&ids);
+                    builder.execute().unwrap();
+                }
+
+                wtxn.commit().unwrap();
+
+                index.prepare_for_closing().wait();
             },
         )
     });
 }
 
-fn delete_documents_from_ids(index: Index, document_ids_to_delete: Vec<RoaringBitmap>) {
-    let mut wtxn = index.write_txn().unwrap();
-
-    let indexer_config = IndexerConfig::default();
-    for ids in document_ids_to_delete {
-        let config = IndexDocumentsConfig::default();
-
-        let mut builder =
-            IndexDocuments::new(&mut wtxn, &index, &indexer_config, config, |_| (), || false)
-                .unwrap();
-        (builder, _) = builder.remove_documents_from_db_no_batch(&ids).unwrap();
-        builder.execute().unwrap();
-    }
-
-    wtxn.commit().unwrap();
-
-    index.prepare_for_closing().wait();
-}
-
 fn indexing_movies_in_three_batches(c: &mut Criterion) {
     let mut group = c.benchmark_group("indexing");
     group.sample_size(BENCHMARK_ITERATION);
@@ -1099,7 +1112,17 @@ fn deleting_nested_movies_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                delete_documents_from_ids(index, document_ids_to_delete)
+                let mut wtxn = index.write_txn().unwrap();
+
+                for ids in document_ids_to_delete {
+                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
+                    builder.delete_documents(&ids);
+                    builder.execute().unwrap();
+                }
+
+                wtxn.commit().unwrap();
+
+                index.prepare_for_closing().wait();
             },
         )
     });
@@ -1315,7 +1338,17 @@ fn deleting_geo_in_batches_default(c: &mut Criterion) {
                 (index, document_ids_to_delete)
             },
             move |(index, document_ids_to_delete)| {
-                delete_documents_from_ids(index, document_ids_to_delete)
+                let mut wtxn = index.write_txn().unwrap();
+
+                for ids in document_ids_to_delete {
+                    let mut builder = DeleteDocuments::new(&mut wtxn, &index).unwrap();
+                    builder.delete_documents(&ids);
+                    builder.execute().unwrap();
+                }
+
+                wtxn.commit().unwrap();
+
+                index.prepare_for_closing().wait();
             },
         )
     });

config.toml

@@ -1,131 +1,120 @@
 # This file shows the default configuration of Meilisearch.
-# All variables are defined here: https://www.meilisearch.com/docs/learn/configuration/instance_options#environment-variables
+# All variables are defined here: https://docs.meilisearch.com/learn/configuration/instance_options.html#environment-variables
 
-# Designates the location where database files will be created and retrieved.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#database-path
 db_path = "./data.ms"
+# Designates the location where database files will be created and retrieved.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#database-path
 
-# Configures the instance's environment. Value must be either `production` or `development`.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#environment
 env = "development"
+# Configures the instance's environment. Value must be either `production` or `development`.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#environment
 
-# The address on which the HTTP server will listen.
 http_addr = "localhost:7700"
+# The address on which the HTTP server will listen.
 
-# Sets the instance's master key, automatically protecting all routes except GET /health.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#master-key
 # master_key = "YOUR_MASTER_KEY_VALUE"
+# Sets the instance's master key, automatically protecting all routes except GET /health.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#master-key
 
+# no_analytics = true
 # Deactivates Meilisearch's built-in telemetry when provided.
 # Meilisearch automatically collects data from all instances that do not opt out using this flag.
 # All gathered data is used solely for the purpose of improving Meilisearch, and can be deleted at any time.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#disable-analytics
-# no_analytics = true
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#disable-analytics
 
-# Sets the maximum size of accepted payloads.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#payload-limit-size
 http_payload_size_limit = "100 MB"
+# Sets the maximum size of accepted payloads.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#payload-limit-size
 
+log_level = "INFO"
 # Defines how much detail should be present in Meilisearch's logs.
 # Meilisearch currently supports six log levels, listed in order of increasing verbosity:  `OFF`, `ERROR`, `WARN`, `INFO`, `DEBUG`, `TRACE`
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#log-level
-log_level = "INFO"
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#log-level
 
-# Sets the maximum amount of RAM Meilisearch can use when indexing.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#max-indexing-memory
 # max_indexing_memory = "2 GiB"
+# Sets the maximum amount of RAM Meilisearch can use when indexing.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#max-indexing-memory
 
-# Sets the maximum number of threads Meilisearch can use during indexing.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#max-indexing-threads
 # max_indexing_threads = 4
+# Sets the maximum number of threads Meilisearch can use during indexing.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#max-indexing-threads
 
 #############
 ### DUMPS ###
 #############
 
-# Sets the directory where Meilisearch will create dump files.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#dump-directory
 dump_dir = "dumps/"
+# Sets the directory where Meilisearch will create dump files.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#dump-directory
 
-# Imports the dump file located at the specified path. Path must point to a .dump file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#import-dump
 # import_dump = "./path/to/my/file.dump"
+# Imports the dump file located at the specified path. Path must point to a .dump file.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#import-dump
 
-# Prevents Meilisearch from throwing an error when `import_dump` does not point to a valid dump file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-missing-dump
 ignore_missing_dump = false
+# Prevents Meilisearch from throwing an error when `import_dump` does not point to a valid dump file.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ignore-missing-dump
 
-# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_dump`.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-dump-if-db-exists
 ignore_dump_if_db_exists = false
+# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_dump`.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ignore-dump-if-db-exists
 
 
 #################
 ### SNAPSHOTS ###
 #################
 
+schedule_snapshot = false
 # Enables scheduled snapshots when true, disable when false (the default).
 # If the value is given as an integer, then enables the scheduled snapshot with the passed value as the interval
 # between each snapshot, in seconds.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#schedule-snapshot-creation
-schedule_snapshot = false
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#schedule-snapshot-creation
 
-# Sets the directory where Meilisearch will store snapshots.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#snapshot-destination
 snapshot_dir = "snapshots/"
+# Sets the directory where Meilisearch will store snapshots.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#snapshot-destination
 
-# Launches Meilisearch after importing a previously-generated snapshot at the given filepath.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#import-snapshot
 # import_snapshot = "./path/to/my/snapshot"
+# Launches Meilisearch after importing a previously-generated snapshot at the given filepath.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#import-snapshot
 
-# Prevents a Meilisearch instance from throwing an error when `import_snapshot` does not point to a valid snapshot file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-missing-snapshot
 ignore_missing_snapshot = false
+# Prevents a Meilisearch instance from throwing an error when `import_snapshot` does not point to a valid snapshot file.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ignore-missing-snapshot
 
-# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_snapshot`.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ignore-snapshot-if-db-exists
 ignore_snapshot_if_db_exists = false
+# Prevents a Meilisearch instance with an existing database from throwing an error when using `import_snapshot`.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ignore-snapshot-if-db-exists
 
 
 ###########
 ### SSL ###
 ###########
 
-# Enables client authentication in the specified path.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-authentication-path
 # ssl_auth_path = "./path/to/root"
+# Enables client authentication in the specified path.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-authentication-path
 
-# Sets the server's SSL certificates.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-certificates-path
 # ssl_cert_path = "./path/to/certfile"
+# Sets the server's SSL certificates.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-certificates-path
 
-# Sets the server's SSL key files.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-key-path
 # ssl_key_path = "./path/to/private-key"
+# Sets the server's SSL key files.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-key-path
 
-# Sets the server's OCSP file.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-ocsp-path
 # ssl_ocsp_path = "./path/to/ocsp-file"
+# Sets the server's OCSP file.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-ocsp-path
 
-# Makes SSL authentication mandatory.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-require-auth
 ssl_require_auth = false
+# Makes SSL authentication mandatory.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-require-auth
 
-# Activates SSL session resumption.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-resumption
 ssl_resumption = false
+# Activates SSL session resumption.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-resumption
 
-# Activates SSL tickets.
-# https://www.meilisearch.com/docs/learn/configuration/instance_options#ssl-tickets
 ssl_tickets = false
-
-#############################
-### Experimental features ###
-#############################
-
-# Experimental metrics feature. For more information, see: <https://github.com/meilisearch/meilisearch/discussions/3518>
-# Enables the Prometheus metrics on the `GET /metrics` endpoint.
-experimental_enable_metrics = false
-
-# Experimental RAM reduction during indexing, do not use in production, see: <https://github.com/meilisearch/product/discussions/652>
-experimental_reduce_indexing_memory_usage = false
+# Activates SSL tickets.
+# https://docs.meilisearch.com/learn/configuration/instance_options.html#ssl-tickets

download-latest.sh

@@ -103,7 +103,7 @@ not_available_failure_usage() {
     printf "$RED%s\n$DEFAULT" 'ERROR: Meilisearch binary is not available for your OS distribution or your architecture yet.'
     echo ''
     echo 'However, you can easily compile the binary from the source files.'
-    echo 'Follow the steps at the page ("Source" tab): https://www.meilisearch.com/docs/learn/getting_started/installation'
+    echo 'Follow the steps at the page ("Source" tab): https://docs.meilisearch.com/learn/getting_started/installation.html'
 }
 
 fetch_release_failure_usage() {

dump/Cargo.toml

@@ -11,22 +11,22 @@ readme.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.70"
-flate2 = "1.0.25"
-http = "0.2.9"
+anyhow = "1.0.65"
+flate2 = "1.0.22"
+http = "0.2.8"
 log = "0.4.17"
 meilisearch-auth = { path = "../meilisearch-auth" }
 meilisearch-types = { path = "../meilisearch-types" }
-once_cell = "1.17.1"
-regex = "1.7.3"
-roaring = { version = "0.10.1", features = ["serde"] }
-serde = { version = "1.0.160", features = ["derive"] }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
+once_cell = "1.15.0"
+regex = "1.6.0"
+roaring = { version = "0.10.0", features = ["serde"] }
+serde = { version = "1.0.136", features = ["derive"] }
+serde_json = { version = "1.0.85", features = ["preserve_order"] }
 tar = "0.4.38"
-tempfile = "3.5.0"
-thiserror = "1.0.40"
-time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-uuid = { version = "1.3.1", features = ["serde", "v4"] }
+tempfile = "3.3.0"
+thiserror = "1.0.30"
+time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+uuid = { version = "1.1.2", features = ["serde", "v4"] }
 
 [dev-dependencies]
 big_s = "1.0.2"

dump/src/lib.rs

@@ -101,9 +101,6 @@ pub enum KindDump {
         documents_ids: Vec<String>,
     },
     DocumentClear,
-    DocumentDeletionByFilter {
-        filter: serde_json::Value,
-    },
     Settings {
         settings: Box<meilisearch_types::settings::Settings<Unchecked>>,
         is_deletion: bool,
@@ -169,9 +166,6 @@ impl From<KindWithContent> for KindDump {
             KindWithContent::DocumentDeletion { documents_ids, .. } => {
                 KindDump::DocumentDeletion { documents_ids }
             }
-            KindWithContent::DocumentDeletionByFilter { filter_expr, .. } => {
-                KindDump::DocumentDeletionByFilter { filter: filter_expr }
-            }
             KindWithContent::DocumentClear { .. } => KindDump::DocumentClear,
             KindWithContent::SettingsUpdate {
                 new_settings,
@@ -208,14 +202,12 @@ pub(crate) mod test {
     use std::str::FromStr;
 
     use big_s::S;
-    use maplit::{btreemap, btreeset};
-    use meilisearch_types::facet_values_sort::FacetValuesSort;
-    use meilisearch_types::features::RuntimeTogglableFeatures;
+    use maplit::btreeset;
     use meilisearch_types::index_uid_pattern::IndexUidPattern;
     use meilisearch_types::keys::{Action, Key};
-    use meilisearch_types::milli;
     use meilisearch_types::milli::update::Setting;
-    use meilisearch_types::settings::{Checked, FacetingSettings, Settings};
+    use meilisearch_types::milli::{self};
+    use meilisearch_types::settings::{Checked, Settings};
     use meilisearch_types::tasks::{Details, Status};
     use serde_json::{json, Map, Value};
     use time::macros::datetime;
@@ -262,18 +254,10 @@ pub(crate) mod test {
             sortable_attributes: Setting::Set(btreeset! { S("age") }),
             ranking_rules: Setting::NotSet,
             stop_words: Setting::NotSet,
-            non_separator_tokens: Setting::NotSet,
-            separator_tokens: Setting::NotSet,
-            dictionary: Setting::NotSet,
             synonyms: Setting::NotSet,
             distinct_attribute: Setting::NotSet,
             typo_tolerance: Setting::NotSet,
-            faceting: Setting::Set(FacetingSettings {
-                max_values_per_facet: Setting::Set(111),
-                sort_facet_values_by: Setting::Set(
-                    btreemap! { S("age") => FacetValuesSort::Count },
-                ),
-            }),
+            faceting: Setting::NotSet,
             pagination: Setting::NotSet,
             _kind: std::marker::PhantomData,
         };
@@ -422,11 +406,6 @@ pub(crate) mod test {
         }
         keys.flush().unwrap();
 
-        // ========== experimental features
-        let features = create_test_features();
-
-        dump.create_experimental_features(features).unwrap();
-
         // create the dump
         let mut file = tempfile::tempfile().unwrap();
         dump.persist_to(&mut file).unwrap();
@@ -435,10 +414,6 @@ pub(crate) mod test {
         file
     }
 
-    fn create_test_features() -> RuntimeTogglableFeatures {
-        RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
-    }
-
     #[test]
     fn test_creating_and_read_dump() {
         let mut file = create_test_dump();
@@ -483,9 +458,5 @@ pub(crate) mod test {
         for (key, expected) in dump.keys().unwrap().zip(create_test_api_keys()) {
             assert_eq!(key.unwrap(), expected);
         }
-
-        // ==== checking the features
-        let expected = create_test_features();
-        assert_eq!(dump.features().unwrap().unwrap(), expected);
     }
 }

dump/src/reader/compat/v2_to_v3.rs

@@ -25,6 +25,7 @@ impl CompatV2ToV3 {
             CompatV2ToV3::Compat(compat) => compat.index_uuid(),
         };
         v2_uuids
+            .into_iter()
             .into_iter()
             .map(|index| v3::meta::IndexUuid { uid: index.uid, uuid: index.uuid })
             .collect()

dump/src/reader/compat/v5_to_v6.rs

@@ -191,10 +191,6 @@ impl CompatV5ToV6 {
             })
         })))
     }
-
-    pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
-        Ok(None)
-    }
 }
 
 pub enum CompatIndexV5ToV6 {
@@ -340,9 +336,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
                 }
             },
             stop_words: settings.stop_words.into(),
-            non_separator_tokens: v6::Setting::NotSet,
-            separator_tokens: v6::Setting::NotSet,
-            dictionary: v6::Setting::NotSet,
             synonyms: settings.synonyms.into(),
             distinct_attribute: settings.distinct_attribute.into(),
             typo_tolerance: match settings.typo_tolerance {
@@ -365,7 +358,6 @@ impl<T> From<v5::Settings<T>> for v6::Settings<v6::Unchecked> {
             faceting: match settings.faceting {
                 v5::Setting::Set(faceting) => v6::Setting::Set(v6::FacetingSettings {
                     max_values_per_facet: faceting.max_values_per_facet.into(),
-                    sort_facet_values_by: v6::Setting::NotSet,
                 }),
                 v5::Setting::Reset => v6::Setting::Reset,
                 v5::Setting::NotSet => v6::Setting::NotSet,

dump/src/reader/mod.rs

@@ -107,13 +107,6 @@ impl DumpReader {
             DumpReader::Compat(compat) => compat.keys(),
         }
     }
-
-    pub fn features(&self) -> Result<Option<v6::RuntimeTogglableFeatures>> {
-        match self {
-            DumpReader::Current(current) => Ok(current.features()),
-            DumpReader::Compat(compat) => compat.features(),
-        }
-    }
 }
 
 impl From<V6Reader> for DumpReader {
@@ -195,53 +188,6 @@ pub(crate) mod test {
     use meili_snap::insta;
 
     use super::*;
-    use crate::reader::v6::RuntimeTogglableFeatures;
-
-    #[test]
-    fn import_dump_v6_experimental() {
-        let dump = File::open("tests/assets/v6-with-experimental.dump").unwrap();
-        let mut dump = DumpReader::open(dump).unwrap();
-
-        // top level infos
-        insta::assert_display_snapshot!(dump.date().unwrap(), @"2023-07-06 7:10:27.21958 +00:00:00");
-        insta::assert_debug_snapshot!(dump.instance_uid().unwrap(), @"None");
-
-        // tasks
-        let tasks = dump.tasks().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        let (tasks, update_files): (Vec<_>, Vec<_>) = tasks.into_iter().unzip();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(tasks), @"d45cd8571703e58ae53c7bd7ce3f5c22");
-        assert_eq!(update_files.len(), 2);
-        assert!(update_files[0].is_none()); // the dump creation
-        assert!(update_files[1].is_none()); // the processed document addition
-
-        // keys
-        let keys = dump.keys().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        meili_snap::snapshot_hash!(meili_snap::json_string!(keys), @"13c2da155e9729c2344688cab29af71d");
-
-        // indexes
-        let mut indexes = dump.indexes().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        // the index are not ordered in any way by default
-        indexes.sort_by_key(|index| index.metadata().uid.to_string());
-
-        let mut test = indexes.pop().unwrap();
-        assert!(indexes.is_empty());
-
-        insta::assert_json_snapshot!(test.metadata(), @r###"
-        {
-          "uid": "test",
-          "primaryKey": "id",
-          "createdAt": "2023-07-06T07:07:41.364694Z",
-          "updatedAt": "2023-07-06T07:07:41.396114Z"
-        }
-        "###);
-
-        assert_eq!(test.documents().unwrap().count(), 1);
-
-        assert_eq!(
-            dump.features().unwrap().unwrap(),
-            RuntimeTogglableFeatures { vector_store: true, ..Default::default() }
-        );
-    }
 
     #[test]
     fn import_dump_v5() {
@@ -319,8 +265,6 @@ pub(crate) mod test {
         let documents = spells.documents().unwrap().collect::<Result<Vec<_>>>().unwrap();
         assert_eq!(documents.len(), 10);
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"235016433dd04262c7f2da01d1e808ce");
-
-        assert_eq!(dump.features().unwrap(), None);
     }
 
     #[test]
@@ -526,12 +470,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2022-10-09T20:27:22.688964637Z",
-          "updatedAt": "2022-10-09T20:27:23.951017769Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -541,12 +485,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2022-10-09T20:27:22.197788495Z",
-          "updatedAt": "2022-10-09T20:28:01.93111053Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -571,12 +515,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2022-10-09T20:27:24.242683494Z",
-          "updatedAt": "2022-10-09T20:27:24.312809641Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -617,12 +561,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2023-01-30T16:25:56.595257Z",
-          "updatedAt": "2023-01-30T16:25:58.70348Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -632,12 +576,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2023-01-30T16:25:56.192178Z",
-          "updatedAt": "2023-01-30T16:25:56.455714Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -647,12 +591,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"0227598af846e574139ee0b80e03a720");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2023-01-30T16:25:58.876405Z",
-          "updatedAt": "2023-01-30T16:25:59.079906Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 

dump/src/reader/v2/mod.rs

@@ -46,7 +46,6 @@ pub type Checked = settings::Checked;
 pub type Unchecked = settings::Unchecked;
 
 pub type Task = updates::UpdateEntry;
-pub type Kind = updates::UpdateMeta;
 
 // everything related to the errors
 pub type ResponseError = errors::ResponseError;
@@ -108,11 +107,8 @@ impl V2Reader {
     pub fn indexes(&self) -> Result<impl Iterator<Item = Result<V2IndexReader>> + '_> {
         Ok(self.index_uuid.iter().map(|index| -> Result<_> {
             V2IndexReader::new(
+                index.uid.clone(),
                 &self.dump.path().join("indexes").join(format!("index-{}", index.uuid)),
-                index,
-                BufReader::new(
-                    File::open(self.dump.path().join("updates").join("data.jsonl")).unwrap(),
-                ),
             )
         }))
     }
@@ -147,41 +143,16 @@ pub struct V2IndexReader {
 }
 
 impl V2IndexReader {
-    pub fn new(path: &Path, index_uuid: &IndexUuid, tasks: BufReader<File>) -> Result<Self> {
+    pub fn new(name: String, path: &Path) -> Result<Self> {
         let meta = File::open(path.join("meta.json"))?;
         let meta: DumpMeta = serde_json::from_reader(meta)?;
 
-        let mut created_at = None;
-        let mut updated_at = None;
-
-        for line in tasks.lines() {
-            let task: Task = serde_json::from_str(&line?)?;
-            if !(task.uuid == index_uuid.uuid && task.is_finished()) {
-                continue;
-            }
-
-            let new_created_at = match task.update.meta() {
-                Kind::DocumentsAddition { .. } | Kind::Settings(_) => task.update.finished_at(),
-                _ => None,
-            };
-            let new_updated_at = task.update.finished_at();
-
-            if created_at.is_none() || created_at > new_created_at {
-                created_at = new_created_at;
-            }
-
-            if updated_at.is_none() || updated_at < new_updated_at {
-                updated_at = new_updated_at;
-            }
-        }
-
-        let current_time = OffsetDateTime::now_utc();
-
         let metadata = IndexMetadata {
-            uid: index_uuid.uid.clone(),
+            uid: name,
             primary_key: meta.primary_key,
-            created_at: created_at.unwrap_or(current_time),
-            updated_at: updated_at.unwrap_or(current_time),
+            // FIXME: Iterate over the whole task queue to find the creation and last update date.
+            created_at: OffsetDateTime::now_utc(),
+            updated_at: OffsetDateTime::now_utc(),
         };
 
         let ret = V2IndexReader {
@@ -277,12 +248,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2022-10-09T20:27:22.688964637Z",
-          "updatedAt": "2022-10-09T20:27:23.951017769Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -292,12 +263,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2022-10-09T20:27:22.197788495Z",
-          "updatedAt": "2022-10-09T20:28:01.93111053Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -322,12 +293,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"d751713988987e9331980363e24189ce");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2022-10-09T20:27:24.242683494Z",
-          "updatedAt": "2022-10-09T20:27:24.312809641Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -369,12 +340,12 @@ pub(crate) mod test {
         assert!(indexes.is_empty());
 
         // products
-        insta::assert_json_snapshot!(products.metadata(), @r###"
+        insta::assert_json_snapshot!(products.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "products",
           "primaryKey": "sku",
-          "createdAt": "2023-01-30T16:25:56.595257Z",
-          "updatedAt": "2023-01-30T16:25:58.70348Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -384,12 +355,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"548284a84de510f71e88e6cdea495cf5");
 
         // movies
-        insta::assert_json_snapshot!(movies.metadata(), @r###"
+        insta::assert_json_snapshot!(movies.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "movies",
           "primaryKey": "id",
-          "createdAt": "2023-01-30T16:25:56.192178Z",
-          "updatedAt": "2023-01-30T16:25:56.455714Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 
@@ -399,12 +370,12 @@ pub(crate) mod test {
         meili_snap::snapshot_hash!(format!("{:#?}", documents), @"0227598af846e574139ee0b80e03a720");
 
         // spells
-        insta::assert_json_snapshot!(spells.metadata(), @r###"
+        insta::assert_json_snapshot!(spells.metadata(), { ".createdAt" => "[now]", ".updatedAt" => "[now]" }, @r###"
         {
           "uid": "dnd_spells",
           "primaryKey": "index",
-          "createdAt": "2023-01-30T16:25:58.876405Z",
-          "updatedAt": "2023-01-30T16:25:59.079906Z"
+          "createdAt": "[now]",
+          "updatedAt": "[now]"
         }
         "###);
 

dump/src/reader/v2/updates.rs

@@ -227,14 +227,4 @@ impl UpdateStatus {
             _ => None,
         }
     }
-
-    pub fn finished_at(&self) -> Option<OffsetDateTime> {
-        match self {
-            UpdateStatus::Processing(_) => None,
-            UpdateStatus::Enqueued(_) => None,
-            UpdateStatus::Processed(u) => Some(u.processed_at),
-            UpdateStatus::Aborted(_) => None,
-            UpdateStatus::Failed(u) => Some(u.failed_at),
-        }
-    }
 }

dump/src/reader/v6/mod.rs

@@ -2,7 +2,6 @@ use std::fs::{self, File};
 use std::io::{BufRead, BufReader, ErrorKind};
 use std::path::Path;
 
-use log::debug;
 pub use meilisearch_types::milli;
 use tempfile::TempDir;
 use time::OffsetDateTime;
@@ -19,7 +18,6 @@ pub type Unchecked = meilisearch_types::settings::Unchecked;
 
 pub type Task = crate::TaskDump;
 pub type Key = meilisearch_types::keys::Key;
-pub type RuntimeTogglableFeatures = meilisearch_types::features::RuntimeTogglableFeatures;
 
 // ===== Other types to clarify the code of the compat module
 // everything related to the tasks
@@ -49,7 +47,6 @@ pub struct V6Reader {
     metadata: Metadata,
     tasks: BufReader<File>,
     keys: BufReader<File>,
-    features: Option<RuntimeTogglableFeatures>,
 }
 
 impl V6Reader {
@@ -61,29 +58,11 @@ impl V6Reader {
             Err(e) => return Err(e.into()),
         };
 
-        let feature_file = match fs::read(dump.path().join("experimental-features.json")) {
-            Ok(feature_file) => Some(feature_file),
-            Err(error) => match error.kind() {
-                // Allows the file to be missing, this will only result in all experimental features disabled.
-                ErrorKind::NotFound => {
-                    debug!("`experimental-features.json` not found in dump");
-                    None
-                }
-                _ => return Err(error.into()),
-            },
-        };
-        let features = if let Some(feature_file) = feature_file {
-            Some(serde_json::from_reader(&*feature_file)?)
-        } else {
-            None
-        };
-
         Ok(V6Reader {
             metadata: serde_json::from_reader(&*meta_file)?,
             instance_uid,
             tasks: BufReader::new(File::open(dump.path().join("tasks").join("queue.jsonl"))?),
             keys: BufReader::new(File::open(dump.path().join("keys.jsonl"))?),
-            features,
             dump,
         })
     }
@@ -150,10 +129,6 @@ impl V6Reader {
             (&mut self.keys).lines().map(|line| -> Result<_> { Ok(serde_json::from_str(&line?)?) }),
         )
     }
-
-    pub fn features(&self) -> Option<RuntimeTogglableFeatures> {
-        self.features
-    }
 }
 
 pub struct UpdateFile {

dump/src/writer.rs

@@ -4,7 +4,6 @@ use std::path::PathBuf;
 
 use flate2::write::GzEncoder;
 use flate2::Compression;
-use meilisearch_types::features::RuntimeTogglableFeatures;
 use meilisearch_types::keys::Key;
 use meilisearch_types::settings::{Checked, Settings};
 use serde_json::{Map, Value};
@@ -54,13 +53,6 @@ impl DumpWriter {
         TaskWriter::new(self.dir.path().join("tasks"))
     }
 
-    pub fn create_experimental_features(&self, features: RuntimeTogglableFeatures) -> Result<()> {
-        Ok(std::fs::write(
-            self.dir.path().join("experimental-features.json"),
-            serde_json::to_string(&features)?,
-        )?)
-    }
-
     pub fn persist_to(self, mut writer: impl Write) -> Result<()> {
         let gz_encoder = GzEncoder::new(&mut writer, Compression::default());
         let mut tar_encoder = tar::Builder::new(gz_encoder);
@@ -292,7 +284,6 @@ pub(crate) mod test {
         │    ├---- update_files/
         │    │    └---- 1.jsonl
         │    └---- queue.jsonl
-        ├---- experimental-features.json
         ├---- instance_uid.uuid
         ├---- keys.jsonl
         └---- metadata.json

file-store/Cargo.toml

@@ -11,9 +11,9 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-tempfile = "3.5.0"
-thiserror = "1.0.40"
-uuid = { version = "1.3.1", features = ["serde", "v4"] }
+tempfile = "3.3.0"
+thiserror = "1.0.30"
+uuid = { version = "1.1.2", features = ["serde", "v4"] }
 
 [dev-dependencies]
-faux = "0.1.9"
+faux = "0.1.8"

file-store/src/lib.rs

@@ -116,20 +116,10 @@ impl FileStore {
 
     /// List the Uuids of the files in the FileStore
     pub fn all_uuids(&self) -> Result<impl Iterator<Item = Result<Uuid>>> {
-        Ok(self.path.read_dir()?.filter_map(|entry| {
-            let file_name = match entry {
-                Ok(entry) => entry.file_name(),
-                Err(e) => return Some(Err(e.into())),
-            };
-            let file_name = match file_name.to_str() {
-                Some(file_name) => file_name,
-                None => return Some(Err(Error::CouldNotParseFileNameAsUtf8)),
-            };
-            if file_name.starts_with('.') {
-                None
-            } else {
-                Some(Uuid::from_str(file_name).map_err(|e| e.into()))
-            }
+        Ok(self.path.read_dir()?.map(|entry| {
+            Ok(Uuid::from_str(
+                entry?.file_name().to_str().ok_or(Error::CouldNotParseFileNameAsUtf8)?,
+            )?)
         }))
     }
 }
@@ -145,34 +135,3 @@ impl File {
         Ok(())
     }
 }
-
-#[cfg(test)]
-mod test {
-    use std::io::Write;
-
-    use tempfile::TempDir;
-
-    use super::*;
-
-    #[test]
-    fn all_uuids() {
-        let dir = TempDir::new().unwrap();
-        let fs = FileStore::new(dir.path()).unwrap();
-        let (uuid, mut file) = fs.new_update().unwrap();
-        file.write_all(b"Hello world").unwrap();
-        file.persist().unwrap();
-        let all_uuids = fs.all_uuids().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(all_uuids, vec![uuid]);
-
-        let (uuid2, file) = fs.new_update().unwrap();
-        let all_uuids = fs.all_uuids().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        assert_eq!(all_uuids, vec![uuid]);
-
-        file.persist().unwrap();
-        let mut all_uuids = fs.all_uuids().unwrap().collect::<Result<Vec<_>>>().unwrap();
-        all_uuids.sort();
-        let mut expected = vec![uuid, uuid2];
-        expected.sort();
-        assert_eq!(all_uuids, expected);
-    }
-}

filter-parser/Cargo.toml

@@ -12,9 +12,8 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-nom = "7.1.3"
-nom_locate = "4.1.0"
-unescaper = "0.1.2"
+nom = "7.1.1"
+nom_locate = "4.0.0"
 
 [dev-dependencies]
-insta = "1.29.0"
+insta = "1.21.0"

filter-parser/src/condition.rs

@@ -20,8 +20,6 @@ pub enum Condition<'a> {
     GreaterThanOrEqual(Token<'a>),
     Equal(Token<'a>),
     NotEqual(Token<'a>),
-    Null,
-    Empty,
     Exists,
     LowerThan(Token<'a>),
     LowerThanOrEqual(Token<'a>),
@@ -46,38 +44,6 @@ pub fn parse_condition(input: Span) -> IResult<FilterCondition> {
     Ok((input, condition))
 }
 
-/// null          = value "IS" WS+ "NULL"
-pub fn parse_is_null(input: Span) -> IResult<FilterCondition> {
-    let (input, key) = parse_value(input)?;
-
-    let (input, _) = tuple((tag("IS"), multispace1, tag("NULL")))(input)?;
-    Ok((input, FilterCondition::Condition { fid: key, op: Null }))
-}
-
-/// null          = value "IS" WS+ "NOT" WS+ "NULL"
-pub fn parse_is_not_null(input: Span) -> IResult<FilterCondition> {
-    let (input, key) = parse_value(input)?;
-
-    let (input, _) = tuple((tag("IS"), multispace1, tag("NOT"), multispace1, tag("NULL")))(input)?;
-    Ok((input, FilterCondition::Not(Box::new(FilterCondition::Condition { fid: key, op: Null }))))
-}
-
-/// empty          = value "IS" WS+ "EMPTY"
-pub fn parse_is_empty(input: Span) -> IResult<FilterCondition> {
-    let (input, key) = parse_value(input)?;
-
-    let (input, _) = tuple((tag("IS"), multispace1, tag("EMPTY")))(input)?;
-    Ok((input, FilterCondition::Condition { fid: key, op: Empty }))
-}
-
-/// empty          = value "IS" WS+ "NOT" WS+ "EMPTY"
-pub fn parse_is_not_empty(input: Span) -> IResult<FilterCondition> {
-    let (input, key) = parse_value(input)?;
-
-    let (input, _) = tuple((tag("IS"), multispace1, tag("NOT"), multispace1, tag("EMPTY")))(input)?;
-    Ok((input, FilterCondition::Not(Box::new(FilterCondition::Condition { fid: key, op: Empty }))))
-}
-
 /// exist          = value "EXISTS"
 pub fn parse_exists(input: Span) -> IResult<FilterCondition> {
     let (input, key) = terminated(parse_value, tag("EXISTS"))(input)?;

filter-parser/src/error.rs

@@ -62,7 +62,6 @@ pub enum ErrorKind<'a> {
     MisusedGeoRadius,
     MisusedGeoBoundingBox,
     InvalidPrimary,
-    InvalidEscapedNumber,
     ExpectedEof,
     ExpectedValue(ExpectedValueKind),
     MalformedValue,
@@ -144,12 +143,11 @@ impl<'a> Display for Error<'a> {
             ErrorKind::MissingClosingDelimiter(c) => {
                 writeln!(f, "Expression `{}` is missing the following closing delimiter: `{}`.", escaped_input, c)?
             }
-            ErrorKind::InvalidPrimary => {
-                let text = if input.trim().is_empty() { "but instead got nothing.".to_string() } else { format!("at `{}`.", escaped_input) };
-                writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` {}", text)?
+            ErrorKind::InvalidPrimary if input.trim().is_empty() => {
+                writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.")?
             }
-            ErrorKind::InvalidEscapedNumber => {
-                writeln!(f, "Found an invalid escaped sequence number: `{}`.", escaped_input)?
+            ErrorKind::InvalidPrimary => {
+                writeln!(f, "Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` at `{}`.", escaped_input)?
             }
             ErrorKind::ExpectedEof => {
                 writeln!(f, "Found unexpected characters at the end of the filter: `{}`. You probably forgot an `OR` or an `AND` rule.", escaped_input)?
@@ -161,7 +159,7 @@ impl<'a> Display for Error<'a> {
                 writeln!(f, "The `_geoBoundingBox` filter expects two pairs of arguments: `_geoBoundingBox([latitude, longitude], [latitude, longitude])`.")?
             }
             ErrorKind::ReservedGeo(name) => {
-                writeln!(f, "`{}` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.", name.escape_debug())?
+                writeln!(f, "`{}` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance), or _geoBoundingBox([latitude, longitude], [latitude, longitude]) built-in rules to filter on `_geo` coordinates.", name.escape_debug())?
             }
             ErrorKind::MisusedGeoRadius => {
                 writeln!(f, "The `_geoRadius` filter is an operation and can't be used as a value.")?

filter-parser/src/lib.rs

@@ -47,10 +47,7 @@ mod value;
 use std::fmt::Debug;
 
 pub use condition::{parse_condition, parse_to, Condition};
-use condition::{
-    parse_exists, parse_is_empty, parse_is_not_empty, parse_is_not_null, parse_is_null,
-    parse_not_exists,
-};
+use condition::{parse_exists, parse_not_exists};
 use error::{cut_with_err, ExpectedValueKind, NomErrorExt};
 pub use error::{Error, ErrorKind};
 use nom::branch::alt;
@@ -144,7 +141,7 @@ pub enum FilterCondition<'a> {
     Or(Vec<Self>),
     And(Vec<Self>),
     GeoLowerThan { point: [Token<'a>; 2], radius: Token<'a> },
-    GeoBoundingBox { top_right_point: [Token<'a>; 2], bottom_left_point: [Token<'a>; 2] },
+    GeoBoundingBox { top_left_point: [Token<'a>; 2], bottom_right_point: [Token<'a>; 2] },
 }
 
 impl<'a> FilterCondition<'a> {
@@ -365,8 +362,8 @@ fn parse_geo_bounding_box(input: Span) -> IResult<FilterCondition> {
     }
 
     let res = FilterCondition::GeoBoundingBox {
-        top_right_point: [args[0][0].into(), args[0][1].into()],
-        bottom_left_point: [args[1][0].into(), args[1][1].into()],
+        top_left_point: [args[0][0].into(), args[0][1].into()],
+        bottom_right_point: [args[1][0].into(), args[1][1].into()],
     };
     Ok((input, res))
 }
@@ -385,34 +382,6 @@ fn parse_geo_point(input: Span) -> IResult<FilterCondition> {
     Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoPoint"))))
 }
 
-/// geoPoint      = WS* "_geoDistance(float WS* "," WS* float WS* "," WS* float)
-fn parse_geo_distance(input: Span) -> IResult<FilterCondition> {
-    // we want to forbid space BEFORE the _geoDistance but not after
-    tuple((
-        multispace0,
-        tag("_geoDistance"),
-        // if we were able to parse `_geoDistance` we are going to return a Failure whatever happens next.
-        cut(delimited(char('('), separated_list1(tag(","), ws(recognize_float)), char(')'))),
-    ))(input)
-    .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))))?;
-    // if we succeeded we still return a `Failure` because `geoDistance` filters are not allowed
-    Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geoDistance"))))
-}
-
-/// geo      = WS* "_geo(float WS* "," WS* float WS* "," WS* float)
-fn parse_geo(input: Span) -> IResult<FilterCondition> {
-    // we want to forbid space BEFORE the _geo but not after
-    tuple((
-        multispace0,
-        word_exact("_geo"),
-        // if we were able to parse `_geo` we are going to return a Failure whatever happens next.
-        cut(delimited(char('('), separated_list1(tag(","), ws(recognize_float)), char(')'))),
-    ))(input)
-    .map_err(|e| e.map(|_| Error::new_from_kind(input, ErrorKind::ReservedGeo("_geo"))))?;
-    // if we succeeded we still return a `Failure` because `_geo` filter is not allowed
-    Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::ReservedGeo("_geo"))))
-}
-
 fn parse_error_reserved_keyword(input: Span) -> IResult<FilterCondition> {
     match parse_condition(input) {
         Ok(result) => Ok(result),
@@ -445,16 +414,10 @@ fn parse_primary(input: Span, depth: usize) -> IResult<FilterCondition> {
         parse_in,
         parse_not_in,
         parse_condition,
-        parse_is_null,
-        parse_is_not_null,
-        parse_is_empty,
-        parse_is_not_empty,
         parse_exists,
         parse_not_exists,
         parse_to,
         // the next lines are only for error handling and are written at the end to have the less possible performance impact
-        parse_geo,
-        parse_geo_distance,
         parse_geo_point,
         parse_error_reserved_keyword,
     ))(input)
@@ -472,81 +435,8 @@ pub fn parse_filter(input: Span) -> IResult<FilterCondition> {
     terminated(|input| parse_expression(input, 0), eof)(input)
 }
 
-impl<'a> std::fmt::Display for FilterCondition<'a> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            FilterCondition::Not(filter) => {
-                write!(f, "NOT ({filter})")
-            }
-            FilterCondition::Condition { fid, op } => {
-                write!(f, "{fid} {op}")
-            }
-            FilterCondition::In { fid, els } => {
-                write!(f, "{fid} IN[")?;
-                for el in els {
-                    write!(f, "{el}, ")?;
-                }
-                write!(f, "]")
-            }
-            FilterCondition::Or(els) => {
-                write!(f, "OR[")?;
-                for el in els {
-                    write!(f, "{el}, ")?;
-                }
-                write!(f, "]")
-            }
-            FilterCondition::And(els) => {
-                write!(f, "AND[")?;
-                for el in els {
-                    write!(f, "{el}, ")?;
-                }
-                write!(f, "]")
-            }
-            FilterCondition::GeoLowerThan { point, radius } => {
-                write!(f, "_geoRadius({}, {}, {})", point[0], point[1], radius)
-            }
-            FilterCondition::GeoBoundingBox {
-                top_right_point: top_left_point,
-                bottom_left_point: bottom_right_point,
-            } => {
-                write!(
-                    f,
-                    "_geoBoundingBox([{}, {}], [{}, {}])",
-                    top_left_point[0],
-                    top_left_point[1],
-                    bottom_right_point[0],
-                    bottom_right_point[1]
-                )
-            }
-        }
-    }
-}
-impl<'a> std::fmt::Display for Condition<'a> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match self {
-            Condition::GreaterThan(token) => write!(f, "> {token}"),
-            Condition::GreaterThanOrEqual(token) => write!(f, ">= {token}"),
-            Condition::Equal(token) => write!(f, "= {token}"),
-            Condition::NotEqual(token) => write!(f, "!= {token}"),
-            Condition::Null => write!(f, "IS NULL"),
-            Condition::Empty => write!(f, "IS EMPTY"),
-            Condition::Exists => write!(f, "EXISTS"),
-            Condition::LowerThan(token) => write!(f, "< {token}"),
-            Condition::LowerThanOrEqual(token) => write!(f, "<= {token}"),
-            Condition::Between { from, to } => write!(f, "{from} TO {to}"),
-        }
-    }
-}
-impl<'a> std::fmt::Display for Token<'a> {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{{{}}}", self.value())
-    }
-}
-
 #[cfg(test)]
 pub mod tests {
-    use FilterCondition as Fc;
-
     use super::*;
 
     /// Create a raw [Token]. You must specify the string that appear BEFORE your element followed by your element
@@ -558,22 +448,14 @@ pub mod tests {
         unsafe { Span::new_from_raw_offset(offset, lines as u32, value, "") }.into()
     }
 
-    fn p(s: &str) -> impl std::fmt::Display + '_ {
-        Fc::parse(s).unwrap().unwrap()
-    }
-
-    #[test]
-    fn parse_escaped() {
-        insta::assert_display_snapshot!(p(r#"title = 'foo\\'"#), @r#"{title} = {foo\}"#);
-        insta::assert_display_snapshot!(p(r#"title = 'foo\\\\'"#), @r#"{title} = {foo\\}"#);
-        insta::assert_display_snapshot!(p(r#"title = 'foo\\\\\\'"#), @r#"{title} = {foo\\\}"#);
-        insta::assert_display_snapshot!(p(r#"title = 'foo\\\\\\\\'"#), @r#"{title} = {foo\\\\}"#);
-        // but it also works with other sequencies
-        insta::assert_display_snapshot!(p(r#"title = 'foo\x20\n\t\"\'"'"#), @"{title} = {foo \n\t\"\'\"}");
-    }
-
     #[test]
     fn parse() {
+        use FilterCondition as Fc;
+
+        fn p(s: &str) -> impl std::fmt::Display + '_ {
+            Fc::parse(s).unwrap().unwrap()
+        }
+
         // Test equal
         insta::assert_display_snapshot!(p("channel = Ponce"), @"{channel} = {Ponce}");
         insta::assert_display_snapshot!(p("subscribers = 12"), @"{subscribers} = {12}");
@@ -614,30 +496,14 @@ pub mod tests {
         insta::assert_display_snapshot!(p("subscribers <= 1000"), @"{subscribers} <= {1000}");
         insta::assert_display_snapshot!(p("subscribers 100 TO 1000"), @"{subscribers} {100} TO {1000}");
 
-        // Test NOT
-        insta::assert_display_snapshot!(p("NOT subscribers < 1000"), @"NOT ({subscribers} < {1000})");
-        insta::assert_display_snapshot!(p("NOT subscribers 100 TO 1000"), @"NOT ({subscribers} {100} TO {1000})");
-
-        // Test NULL + NOT NULL
-        insta::assert_display_snapshot!(p("subscribers IS NULL"), @"{subscribers} IS NULL");
-        insta::assert_display_snapshot!(p("NOT subscribers IS NULL"), @"NOT ({subscribers} IS NULL)");
-        insta::assert_display_snapshot!(p("subscribers IS NOT NULL"), @"NOT ({subscribers} IS NULL)");
-        insta::assert_display_snapshot!(p("NOT subscribers IS NOT NULL"), @"{subscribers} IS NULL");
-        insta::assert_display_snapshot!(p("subscribers  IS   NOT   NULL"), @"NOT ({subscribers} IS NULL)");
-
-        // Test EMPTY + NOT EMPTY
-        insta::assert_display_snapshot!(p("subscribers IS EMPTY"), @"{subscribers} IS EMPTY");
-        insta::assert_display_snapshot!(p("NOT subscribers IS EMPTY"), @"NOT ({subscribers} IS EMPTY)");
-        insta::assert_display_snapshot!(p("subscribers IS NOT EMPTY"), @"NOT ({subscribers} IS EMPTY)");
-        insta::assert_display_snapshot!(p("NOT subscribers IS NOT EMPTY"), @"{subscribers} IS EMPTY");
-        insta::assert_display_snapshot!(p("subscribers  IS   NOT   EMPTY"), @"NOT ({subscribers} IS EMPTY)");
-
-        // Test EXISTS + NOT EXITS
+        // Test NOT + EXISTS
         insta::assert_display_snapshot!(p("subscribers EXISTS"), @"{subscribers} EXISTS");
+        insta::assert_display_snapshot!(p("NOT subscribers < 1000"), @"NOT ({subscribers} < {1000})");
         insta::assert_display_snapshot!(p("NOT subscribers EXISTS"), @"NOT ({subscribers} EXISTS)");
         insta::assert_display_snapshot!(p("subscribers NOT EXISTS"), @"NOT ({subscribers} EXISTS)");
         insta::assert_display_snapshot!(p("NOT subscribers NOT EXISTS"), @"{subscribers} EXISTS");
         insta::assert_display_snapshot!(p("subscribers NOT   EXISTS"), @"NOT ({subscribers} EXISTS)");
+        insta::assert_display_snapshot!(p("NOT subscribers 100 TO 1000"), @"NOT ({subscribers} {100} TO {1000})");
 
         // Test nested NOT
         insta::assert_display_snapshot!(p("NOT NOT NOT NOT x = 5"), @"{x} = {5}");
@@ -710,7 +576,7 @@ pub mod tests {
         "###);
 
         insta::assert_display_snapshot!(p("'OR'"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `\'OR\'`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` at `\'OR\'`.
         1:5 'OR'
         "###);
 
@@ -720,12 +586,12 @@ pub mod tests {
         "###);
 
         insta::assert_display_snapshot!(p("channel Ponce"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `channel Ponce`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` at `channel Ponce`.
         1:14 channel Ponce
         "###);
 
         insta::assert_display_snapshot!(p("channel = Ponce OR"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` but instead got nothing.
         19:19 channel = Ponce OR
         "###);
 
@@ -755,35 +621,15 @@ pub mod tests {
         "###);
 
         insta::assert_display_snapshot!(p("_geoPoint(12, 13, 14)"), @r###"
-        `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
+        `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance), or _geoBoundingBox([latitude, longitude], [latitude, longitude]) built-in rules to filter on `_geo` coordinates.
         1:22 _geoPoint(12, 13, 14)
         "###);
 
         insta::assert_display_snapshot!(p("position <= _geoPoint(12, 13, 14)"), @r###"
-        `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
+        `_geoPoint` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance), or _geoBoundingBox([latitude, longitude], [latitude, longitude]) built-in rules to filter on `_geo` coordinates.
         13:34 position <= _geoPoint(12, 13, 14)
         "###);
 
-        insta::assert_display_snapshot!(p("_geoDistance(12, 13, 14)"), @r###"
-        `_geoDistance` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
-        1:25 _geoDistance(12, 13, 14)
-        "###);
-
-        insta::assert_display_snapshot!(p("position <= _geoDistance(12, 13, 14)"), @r###"
-        `_geoDistance` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
-        13:37 position <= _geoDistance(12, 13, 14)
-        "###);
-
-        insta::assert_display_snapshot!(p("_geo(12, 13, 14)"), @r###"
-        `_geo` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
-        1:17 _geo(12, 13, 14)
-        "###);
-
-        insta::assert_display_snapshot!(p("position <= _geo(12, 13, 14)"), @r###"
-        `_geo` is a reserved keyword and thus can't be used as a filter expression. Use the `_geoRadius(latitude, longitude, distance)` or `_geoBoundingBox([latitude, longitude], [latitude, longitude])` built-in rules to filter on `_geo` coordinates.
-        13:29 position <= _geo(12, 13, 14)
-        "###);
-
         insta::assert_display_snapshot!(p("position <= _geoRadius(12, 13, 14)"), @r###"
         The `_geoRadius` filter is an operation and can't be used as a value.
         13:35 position <= _geoRadius(12, 13, 14)
@@ -810,12 +656,12 @@ pub mod tests {
         "###);
 
         insta::assert_display_snapshot!(p("colour NOT EXIST"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `colour NOT EXIST`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` at `colour NOT EXIST`.
         1:17 colour NOT EXIST
         "###);
 
         insta::assert_display_snapshot!(p("subscribers 100 TO1000"), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `subscribers 100 TO1000`.
+        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `_geoRadius`, or `_geoBoundingBox` at `subscribers 100 TO1000`.
         1:23 subscribers 100 TO1000
         "###);
 
@@ -876,39 +722,6 @@ pub mod tests {
         Was expecting a value but instead got `OR`, which is a reserved keyword. To use `OR` as a field name or a value, surround it by quotes.
         5:7 NOT OR EXISTS AND EXISTS NOT EXISTS
         "###);
-
-        insta::assert_display_snapshot!(p(r#"value NULL"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value NULL`.
-        1:11 value NULL
-        "###);
-        insta::assert_display_snapshot!(p(r#"value NOT NULL"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value NOT NULL`.
-        1:15 value NOT NULL
-        "###);
-        insta::assert_display_snapshot!(p(r#"value EMPTY"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value EMPTY`.
-        1:12 value EMPTY
-        "###);
-        insta::assert_display_snapshot!(p(r#"value NOT EMPTY"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value NOT EMPTY`.
-        1:16 value NOT EMPTY
-        "###);
-        insta::assert_display_snapshot!(p(r#"value IS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS`.
-        1:9 value IS
-        "###);
-        insta::assert_display_snapshot!(p(r#"value IS NOT"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT`.
-        1:13 value IS NOT
-        "###);
-        insta::assert_display_snapshot!(p(r#"value IS EXISTS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS EXISTS`.
-        1:16 value IS EXISTS
-        "###);
-        insta::assert_display_snapshot!(p(r#"value IS NOT EXISTS"#), @r###"
-        Was expecting an operation `=`, `!=`, `>=`, `>`, `<=`, `<`, `IN`, `NOT IN`, `TO`, `EXISTS`, `NOT EXISTS`, `IS NULL`, `IS NOT NULL`, `IS EMPTY`, `IS NOT EMPTY`, `_geoRadius`, or `_geoBoundingBox` at `value IS NOT EXISTS`.
-        1:20 value IS NOT EXISTS
-        "###);
     }
 
     #[test]
@@ -933,3 +746,69 @@ pub mod tests {
         assert_eq!(token.value(), s);
     }
 }
+
+impl<'a> std::fmt::Display for FilterCondition<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            FilterCondition::Not(filter) => {
+                write!(f, "NOT ({filter})")
+            }
+            FilterCondition::Condition { fid, op } => {
+                write!(f, "{fid} {op}")
+            }
+            FilterCondition::In { fid, els } => {
+                write!(f, "{fid} IN[")?;
+                for el in els {
+                    write!(f, "{el}, ")?;
+                }
+                write!(f, "]")
+            }
+            FilterCondition::Or(els) => {
+                write!(f, "OR[")?;
+                for el in els {
+                    write!(f, "{el}, ")?;
+                }
+                write!(f, "]")
+            }
+            FilterCondition::And(els) => {
+                write!(f, "AND[")?;
+                for el in els {
+                    write!(f, "{el}, ")?;
+                }
+                write!(f, "]")
+            }
+            FilterCondition::GeoLowerThan { point, radius } => {
+                write!(f, "_geoRadius({}, {}, {})", point[0], point[1], radius)
+            }
+            FilterCondition::GeoBoundingBox { top_left_point, bottom_right_point } => {
+                write!(
+                    f,
+                    "_geoBoundingBox([{}, {}], [{}, {}])",
+                    top_left_point[0],
+                    top_left_point[1],
+                    bottom_right_point[0],
+                    bottom_right_point[1]
+                )
+            }
+        }
+    }
+}
+impl<'a> std::fmt::Display for Condition<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Condition::GreaterThan(token) => write!(f, "> {token}"),
+            Condition::GreaterThanOrEqual(token) => write!(f, ">= {token}"),
+            Condition::Equal(token) => write!(f, "= {token}"),
+            Condition::NotEqual(token) => write!(f, "!= {token}"),
+            Condition::Exists => write!(f, "EXISTS"),
+            Condition::LowerThan(token) => write!(f, "< {token}"),
+            Condition::LowerThanOrEqual(token) => write!(f, "<= {token}"),
+            Condition::Between { from, to } => write!(f, "{from} TO {to}"),
+        }
+    }
+}
+impl<'a> std::fmt::Display for Token<'a> {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "{{{}}}", self.value())
+    }
+}

filter-parser/src/value.rs

@@ -7,8 +7,8 @@ use nom::{InputIter, InputLength, InputTake, Slice};
 
 use crate::error::{ExpectedValueKind, NomErrorExt};
 use crate::{
-    parse_geo, parse_geo_bounding_box, parse_geo_distance, parse_geo_point, parse_geo_radius,
-    Error, ErrorKind, IResult, Span, Token,
+    parse_geo_bounding_box, parse_geo_point, parse_geo_radius, Error, ErrorKind, IResult, Span,
+    Token,
 };
 
 /// This function goes through all characters in the [Span] if it finds any escaped character (`\`).
@@ -88,16 +88,11 @@ pub fn parse_value(input: Span) -> IResult<Token> {
     // then, we want to check if the user is misusing a geo expression
     // This expression can’t finish without error.
     // We want to return an error in case of failure.
-    let geo_reserved_parse_functions = [parse_geo_point, parse_geo_distance, parse_geo];
-
-    for parser in geo_reserved_parse_functions {
-        if let Err(err) = parser(input) {
-            if err.is_failure() {
-                return Err(err);
-            }
+    if let Err(err) = parse_geo_point(input) {
+        if err.is_failure() {
+            return Err(err);
         }
     }
-
     match parse_geo_radius(input) {
         Ok(_) => {
             return Err(nom::Err::Failure(Error::new_from_kind(input, ErrorKind::MisusedGeoRadius)))
@@ -171,24 +166,7 @@ pub fn parse_value(input: Span) -> IResult<Token> {
         })
     })?;
 
-    match unescaper::unescape(value.value()) {
-        Ok(content) => {
-            if content.len() != value.value().len() {
-                Ok((input, Token::new(value.original_span(), Some(content))))
-            } else {
-                Ok((input, value))
-            }
-        }
-        Err(unescaper::Error::IncompleteStr(_)) => Err(nom::Err::Incomplete(nom::Needed::Unknown)),
-        Err(unescaper::Error::ParseIntError { .. }) => Err(nom::Err::Error(Error::new_from_kind(
-            value.original_span(),
-            ErrorKind::InvalidEscapedNumber,
-        ))),
-        Err(unescaper::Error::InvalidChar { .. }) => Err(nom::Err::Error(Error::new_from_kind(
-            value.original_span(),
-            ErrorKind::MalformedValue,
-        ))),
-    }
+    Ok((input, value))
 }
 
 fn is_value_component(c: char) -> bool {
@@ -200,20 +178,7 @@ fn is_syntax_component(c: char) -> bool {
 }
 
 fn is_keyword(s: &str) -> bool {
-    matches!(
-        s,
-        "AND"
-            | "OR"
-            | "IN"
-            | "NOT"
-            | "TO"
-            | "EXISTS"
-            | "IS"
-            | "NULL"
-            | "EMPTY"
-            | "_geoRadius"
-            | "_geoBoundingBox"
-    )
+    matches!(s, "AND" | "OR" | "IN" | "NOT" | "TO" | "EXISTS" | "_geoRadius" | "_geoBoundingBox")
 }
 
 #[cfg(test)]
@@ -335,17 +300,17 @@ pub mod test {
             ("\"cha'nnel\"", "cha'nnel", false),
             ("I'm tamo", "I", false),
             // escaped thing but not quote
-            (r#""\\""#, r#"\"#, true),
-            (r#""\\\\\\""#, r#"\\\"#, true),
-            (r#""aa\\aa""#, r#"aa\aa"#, true),
+            (r#""\\""#, r#"\\"#, false),
+            (r#""\\\\\\""#, r#"\\\\\\"#, false),
+            (r#""aa\\aa""#, r#"aa\\aa"#, false),
             // with double quote
             (r#""Hello \"world\"""#, r#"Hello "world""#, true),
-            (r#""Hello \\\"world\\\"""#, r#"Hello \"world\""#, true),
+            (r#""Hello \\\"world\\\"""#, r#"Hello \\"world\\""#, true),
             (r#""I'm \"super\" tamo""#, r#"I'm "super" tamo"#, true),
             (r#""\"\"""#, r#""""#, true),
             // with simple quote
             (r#"'Hello \'world\''"#, r#"Hello 'world'"#, true),
-            (r#"'Hello \\\'world\\\''"#, r#"Hello \'world\'"#, true),
+            (r#"'Hello \\\'world\\\''"#, r#"Hello \\'world\\'"#, true),
             (r#"'I\'m "super" tamo'"#, r#"I'm "super" tamo"#, true),
             (r#"'\'\''"#, r#"''"#, true),
         ];
@@ -367,14 +332,7 @@ pub mod test {
                 "Filter `{}` was not supposed to be escaped",
                 input
             );
-            assert_eq!(
-                token.value(),
-                expected,
-                "Filter `{}` failed by giving `{}` instead of `{}`.",
-                input,
-                token.value(),
-                expected
-            );
+            assert_eq!(token.value(), expected, "Filter `{}` failed.", input);
         }
     }
 

flatten-serde-json/Cargo.toml

@@ -16,7 +16,7 @@ license.workspace = true
 serde_json = "1.0"
 
 [dev-dependencies]
-criterion = { version = "0.5.1", features = ["html_reports"] }
+criterion = { version = "0.4.0", features = ["html_reports"] }
 
 [[bench]]
 name = "benchmarks"

flatten-serde-json/src/lib.rs

@@ -4,56 +4,51 @@ use serde_json::{Map, Value};
 
 pub fn flatten(json: &Map<String, Value>) -> Map<String, Value> {
     let mut obj = Map::new();
-    let mut all_entries = vec![];
-    insert_object(&mut obj, None, json, &mut all_entries);
-    for (key, old_val) in all_entries {
-        obj.entry(key).or_insert(old_val.clone());
+    let mut all_keys = vec![];
+    insert_object(&mut obj, None, json, &mut all_keys);
+    for key in all_keys {
+        obj.entry(key).or_insert(Value::Array(vec![]));
     }
     obj
 }
 
-fn insert_object<'a>(
+fn insert_object(
     base_json: &mut Map<String, Value>,
     base_key: Option<&str>,
-    object: &'a Map<String, Value>,
-    all_entries: &mut Vec<(String, &'a Value)>,
+    object: &Map<String, Value>,
+    all_keys: &mut Vec<String>,
 ) {
     for (key, value) in object {
         let new_key = base_key.map_or_else(|| key.clone(), |base_key| format!("{base_key}.{key}"));
-        all_entries.push((new_key.clone(), value));
+        all_keys.push(new_key.clone());
         if let Some(array) = value.as_array() {
-            insert_array(base_json, &new_key, array, all_entries);
+            insert_array(base_json, &new_key, array, all_keys);
         } else if let Some(object) = value.as_object() {
-            insert_object(base_json, Some(&new_key), object, all_entries);
+            insert_object(base_json, Some(&new_key), object, all_keys);
         } else {
-            insert_value(base_json, &new_key, value.clone(), false);
+            insert_value(base_json, &new_key, value.clone());
         }
     }
 }
 
-fn insert_array<'a>(
+fn insert_array(
     base_json: &mut Map<String, Value>,
     base_key: &str,
-    array: &'a Vec<Value>,
-    all_entries: &mut Vec<(String, &'a Value)>,
+    array: &Vec<Value>,
+    all_keys: &mut Vec<String>,
 ) {
     for value in array {
         if let Some(object) = value.as_object() {
-            insert_object(base_json, Some(base_key), object, all_entries);
+            insert_object(base_json, Some(base_key), object, all_keys);
         } else if let Some(sub_array) = value.as_array() {
-            insert_array(base_json, base_key, sub_array, all_entries);
+            insert_array(base_json, base_key, sub_array, all_keys);
         } else {
-            insert_value(base_json, base_key, value.clone(), true);
+            insert_value(base_json, base_key, value.clone());
         }
     }
 }
 
-fn insert_value(
-    base_json: &mut Map<String, Value>,
-    key: &str,
-    to_insert: Value,
-    came_from_array: bool,
-) {
+fn insert_value(base_json: &mut Map<String, Value>, key: &str, to_insert: Value) {
     debug_assert!(!to_insert.is_object());
     debug_assert!(!to_insert.is_array());
 
@@ -68,8 +63,6 @@ fn insert_value(
             base_json[key] = Value::Array(vec![value, to_insert]);
         }
         // if it does not exist we can push the value untouched
-    } else if came_from_array {
-        base_json.insert(key.to_string(), Value::Array(vec![to_insert]));
     } else {
         base_json.insert(key.to_string(), to_insert);
     }
@@ -120,11 +113,7 @@ mod tests {
         assert_eq!(
             &flat,
             json!({
-                "a": {
-                  "b": "c",
-                  "d": "e",
-                  "f": "g"
-                },
+                "a": [],
                 "a.b": "c",
                 "a.d": "e",
                 "a.f": "g"
@@ -175,7 +164,7 @@ mod tests {
         assert_eq!(
             &flat,
             json!({
-                "a": [42],
+                "a": 42,
                 "a.b": ["c", "d", "e"],
             })
             .as_object()
@@ -197,7 +186,7 @@ mod tests {
         assert_eq!(
             &flat,
             json!({
-                "a": [null],
+                "a": null,
                 "a.b": ["c", "d", "e"],
             })
             .as_object()
@@ -219,9 +208,7 @@ mod tests {
         assert_eq!(
             &flat,
             json!({
-                "a": {
-                    "b": "c"
-                },
+                "a": [],
                 "a.b": ["c", "d"],
             })
             .as_object()
@@ -247,7 +234,7 @@ mod tests {
             json!({
                 "a.b": ["c", "d", "f"],
                 "a.c": "e",
-                "a": [35],
+                "a": 35,
             })
             .as_object()
             .unwrap()
@@ -315,53 +302,4 @@ mod tests {
             .unwrap()
         );
     }
-
-    #[test]
-    fn flatten_nested_values_keep_original_values() {
-        let mut base: Value = json!({
-            "tags": {
-                "t1": "v1"
-            },
-            "prices": {
-                "p1": [null],
-                "p1000": {"tamo": {"le": {}}}
-            },
-            "kiki": [[]]
-        });
-        let json = std::mem::take(base.as_object_mut().unwrap());
-        let flat = flatten(&json);
-
-        println!("{}", serde_json::to_string_pretty(&flat).unwrap());
-
-        assert_eq!(
-            &flat,
-            json!({
-              "prices": {
-                "p1": [null],
-                "p1000": {
-                  "tamo": {
-                    "le": {}
-                  }
-                }
-              },
-              "prices.p1": [null],
-              "prices.p1000": {
-                "tamo": {
-                  "le": {}
-                }
-              },
-              "prices.p1000.tamo": {
-                "le": {}
-              },
-              "prices.p1000.tamo.le": {},
-              "tags": {
-                "t1": "v1"
-              },
-              "tags.t1": "v1",
-              "kiki": [[]]
-            })
-            .as_object()
-            .unwrap()
-        );
-    }
 }

fuzzers/Cargo.toml

@@ -1,20 +0,0 @@
-[package]
-name = "fuzzers"
-publish = false
-
-version.workspace = true
-authors.workspace = true
-description.workspace = true
-homepage.workspace = true
-readme.workspace = true
-edition.workspace = true
-license.workspace = true
-
-[dependencies]
-arbitrary = { version = "1.3.0", features = ["derive"] }
-clap = { version = "4.3.0", features = ["derive"] }
-fastrand = "2.0.0"
-milli = { path = "../milli" }
-serde = { version = "1.0.160", features = ["derive"] }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
-tempfile = "3.5.0"

fuzzers/README.md

@@ -1,3 +0,0 @@
-# Fuzzers
-
-The purpose of this crate is to contains all the handmade "fuzzer" we may need.

fuzzers/src/bin/fuzz-indexing.rs

@@ -1,152 +0,0 @@
-use std::num::NonZeroUsize;
-use std::path::PathBuf;
-use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
-use std::time::Duration;
-
-use arbitrary::{Arbitrary, Unstructured};
-use clap::Parser;
-use fuzzers::Operation;
-use milli::heed::EnvOpenOptions;
-use milli::update::{IndexDocuments, IndexDocumentsConfig, IndexerConfig};
-use milli::Index;
-use tempfile::TempDir;
-
-#[derive(Debug, Arbitrary)]
-struct Batch([Operation; 5]);
-
-#[derive(Debug, Clone, Parser)]
-struct Opt {
-    /// The number of fuzzer to run in parallel.
-    #[clap(long)]
-    par: Option<NonZeroUsize>,
-    // We need to put a lot of newlines in the following documentation or else everything gets collapsed on one line
-    /// The path in which the databases will be created.
-    /// Using a ramdisk is recommended.
-    ///
-    /// Linux:
-    ///
-    /// sudo mount -t tmpfs -o size=2g tmpfs ramdisk # to create it
-    ///
-    /// sudo umount ramdisk # to remove it
-    ///
-    /// MacOS:
-    ///
-    /// diskutil erasevolume HFS+ 'RAM Disk' `hdiutil attach -nobrowse -nomount ram://4194304 # create it
-    ///
-    /// hdiutil detach /dev/:the_disk
-    #[clap(long)]
-    path: Option<PathBuf>,
-}
-
-fn main() {
-    let opt = Opt::parse();
-    let progression: &'static AtomicUsize = Box::leak(Box::new(AtomicUsize::new(0)));
-    let stop: &'static AtomicBool = Box::leak(Box::new(AtomicBool::new(false)));
-
-    let par = opt.par.unwrap_or_else(|| std::thread::available_parallelism().unwrap()).get();
-    let mut handles = Vec::with_capacity(par);
-
-    for _ in 0..par {
-        let opt = opt.clone();
-
-        let handle = std::thread::spawn(move || {
-            let mut options = EnvOpenOptions::new();
-            options.map_size(1024 * 1024 * 1024 * 1024);
-            let tempdir = match opt.path {
-                Some(path) => TempDir::new_in(path).unwrap(),
-                None => TempDir::new().unwrap(),
-            };
-            let index = Index::new(options, tempdir.path()).unwrap();
-            let indexer_config = IndexerConfig::default();
-            let index_documents_config = IndexDocumentsConfig::default();
-
-            std::thread::scope(|s| {
-                loop {
-                    if stop.load(Ordering::Relaxed) {
-                        return;
-                    }
-                    let v: Vec<u8> =
-                        std::iter::repeat_with(|| fastrand::u8(..)).take(1000).collect();
-
-                    let mut data = Unstructured::new(&v);
-                    let batches = <[Batch; 5]>::arbitrary(&mut data).unwrap();
-                    // will be used to display the error once a thread crashes
-                    let dbg_input = format!("{:#?}", batches);
-
-                    let handle = s.spawn(|| {
-                        let mut wtxn = index.write_txn().unwrap();
-
-                        for batch in batches {
-                            let mut builder = IndexDocuments::new(
-                                &mut wtxn,
-                                &index,
-                                &indexer_config,
-                                index_documents_config.clone(),
-                                |_| (),
-                                || false,
-                            )
-                            .unwrap();
-
-                            for op in batch.0 {
-                                match op {
-                                    Operation::AddDoc(doc) => {
-                                        let documents =
-                                            milli::documents::objects_from_json_value(doc.to_d());
-                                        let documents =
-                                            milli::documents::documents_batch_reader_from_objects(
-                                                documents,
-                                            );
-                                        let (b, _added) = builder.add_documents(documents).unwrap();
-                                        builder = b;
-                                    }
-                                    Operation::DeleteDoc(id) => {
-                                        let (b, _removed) =
-                                            builder.remove_documents(vec![id.to_s()]).unwrap();
-                                        builder = b;
-                                    }
-                                }
-                            }
-                            builder.execute().unwrap();
-
-                            // after executing a batch we check if the database is corrupted
-                            let res = index.search(&wtxn).execute().unwrap();
-                            index.documents(&wtxn, res.documents_ids).unwrap();
-                            progression.fetch_add(1, Ordering::Relaxed);
-                        }
-                        wtxn.abort().unwrap();
-                    });
-                    if let err @ Err(_) = handle.join() {
-                        stop.store(true, Ordering::Relaxed);
-                        err.expect(&dbg_input);
-                    }
-                }
-            });
-        });
-        handles.push(handle);
-    }
-
-    std::thread::spawn(|| {
-        let mut last_value = 0;
-        let start = std::time::Instant::now();
-        loop {
-            let total = progression.load(Ordering::Relaxed);
-            let elapsed = start.elapsed().as_secs();
-            if elapsed > 3600 {
-                // after 1 hour, stop the fuzzer, success
-                std::process::exit(0);
-            }
-            println!(
-                "Has been running for {:?} seconds. Tested {} new values for a total of {}.",
-                elapsed,
-                total - last_value,
-                total
-            );
-            last_value = total;
-            std::thread::sleep(Duration::from_secs(1));
-        }
-    });
-
-    for handle in handles {
-        handle.join().unwrap();
-    }
-}

fuzzers/src/lib.rs

@@ -1,46 +0,0 @@
-use arbitrary::Arbitrary;
-use serde_json::{json, Value};
-
-#[derive(Debug, Arbitrary)]
-pub enum Document {
-    One,
-    Two,
-    Three,
-    Four,
-    Five,
-    Six,
-}
-
-impl Document {
-    pub fn to_d(&self) -> Value {
-        match self {
-            Document::One => json!({ "id": 0, "doggo": "bernese" }),
-            Document::Two => json!({ "id": 0, "doggo": "golden" }),
-            Document::Three => json!({ "id": 0, "catto": "jorts" }),
-            Document::Four => json!({ "id": 1, "doggo": "bernese" }),
-            Document::Five => json!({ "id": 1, "doggo": "golden" }),
-            Document::Six => json!({ "id": 1, "catto": "jorts" }),
-        }
-    }
-}
-
-#[derive(Debug, Arbitrary)]
-pub enum DocId {
-    Zero,
-    One,
-}
-
-impl DocId {
-    pub fn to_s(&self) -> String {
-        match self {
-            DocId::Zero => "0".to_string(),
-            DocId::One => "1".to_string(),
-        }
-    }
-}
-
-#[derive(Debug, Arbitrary)]
-pub enum Operation {
-    AddDoc(Document),
-    DeleteDoc(DocId),
-}

index-scheduler/Cargo.toml

@@ -11,30 +11,28 @@ edition.workspace = true
 license.workspace = true
 
 [dependencies]
-anyhow = "1.0.70"
+anyhow = "1.0.64"
 bincode = "1.3.3"
-csv = "1.2.1"
-derive_builder = "0.12.0"
+csv = "1.1.6"
+derive_builder = "0.11.2"
 dump = { path = "../dump" }
-enum-iterator = "1.4.0"
+enum-iterator = "1.1.3"
 file-store = { path = "../file-store" }
-log = "0.4.17"
-meilisearch-auth = { path = "../meilisearch-auth" }
+log = "0.4.14"
 meilisearch-types = { path = "../meilisearch-types" }
 page_size = "0.5.0"
-puffin = "0.16.0"
-roaring = { version = "0.10.1", features = ["serde"] }
-serde = { version = "1.0.160", features = ["derive"] }
-serde_json = { version = "1.0.95", features = ["preserve_order"] }
+roaring = { version = "0.10.0", features = ["serde"] }
+serde = { version = "1.0.136", features = ["derive"] }
+serde_json = { version = "1.0.85", features = ["preserve_order"] }
 synchronoise = "1.0.1"
-tempfile = "3.5.0"
-thiserror = "1.0.40"
-time = { version = "0.3.20", features = ["serde-well-known", "formatting", "parsing", "macros"] }
-uuid = { version = "1.3.1", features = ["serde", "v4"] }
+tempfile = "3.3.0"
+thiserror = "1.0.30"
+time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+uuid = { version = "1.1.2", features = ["serde", "v4"] }
 
 [dev-dependencies]
 big_s = "1.0.2"
 crossbeam = "0.8.2"
-insta = { version = "1.29.0", features = ["json", "redactions"] }
+insta = { version = "1.19.1", features = ["json", "redactions"] }
 meili-snap = { path = "../meili-snap" }
 nelson = { git = "https://github.com/meilisearch/nelson.git", rev = "675f13885548fb415ead8fbb447e9e6d9314000a"}

index-scheduler/src/autobatcher.rs

@@ -25,7 +25,6 @@ enum AutobatchKind {
         primary_key: Option<String>,
     },
     DocumentDeletion,
-    DocumentDeletionByFilter,
     DocumentClear,
     Settings {
         allow_index_creation: bool,
@@ -65,9 +64,6 @@ impl From<KindWithContent> for AutobatchKind {
             } => AutobatchKind::DocumentImport { method, allow_index_creation, primary_key },
             KindWithContent::DocumentDeletion { .. } => AutobatchKind::DocumentDeletion,
             KindWithContent::DocumentClear { .. } => AutobatchKind::DocumentClear,
-            KindWithContent::DocumentDeletionByFilter { .. } => {
-                AutobatchKind::DocumentDeletionByFilter
-            }
             KindWithContent::SettingsUpdate { allow_index_creation, is_deletion, .. } => {
                 AutobatchKind::Settings {
                     allow_index_creation: allow_index_creation && !is_deletion,
@@ -101,9 +97,6 @@ pub enum BatchKind {
     DocumentDeletion {
         deletion_ids: Vec<TaskId>,
     },
-    DocumentDeletionByFilter {
-        id: TaskId,
-    },
     ClearAndSettings {
         other: Vec<TaskId>,
         allow_index_creation: bool,
@@ -160,7 +153,7 @@ impl BatchKind {
 impl BatchKind {
     /// Returns a `ControlFlow::Break` if you must stop right now.
     /// The boolean tell you if an index has been created by the batched task.
-    /// To ease the writing of the code. `true` can be returned when you don't need to create an index
+    /// To ease the writting of the code. `true` can be returned when you don't need to create an index
     /// but false can't be returned if you needs to create an index.
     // TODO use an AutoBatchKind as input
     pub fn new(
@@ -202,9 +195,6 @@ impl BatchKind {
             K::DocumentDeletion => {
                 (Continue(BatchKind::DocumentDeletion { deletion_ids: vec![task_id] }), false)
             }
-            K::DocumentDeletionByFilter => {
-                (Break(BatchKind::DocumentDeletionByFilter { id: task_id }), false)
-            }
             K::Settings { allow_index_creation } => (
                 Continue(BatchKind::Settings { allow_index_creation, settings_ids: vec![task_id] }),
                 allow_index_creation,
@@ -214,7 +204,7 @@ impl BatchKind {
 
     /// Returns a `ControlFlow::Break` if you must stop right now.
     /// The boolean tell you if an index has been created by the batched task.
-    /// To ease the writing of the code. `true` can be returned when you don't need to create an index
+    /// To ease the writting of the code. `true` can be returned when you don't need to create an index
     /// but false can't be returned if you needs to create an index.
     #[rustfmt::skip]
     fn accumulate(self, id: TaskId, kind: AutobatchKind, index_already_exists: bool, primary_key: Option<&str>) -> ControlFlow<BatchKind, BatchKind> {
@@ -222,7 +212,7 @@ impl BatchKind {
 
         match (self, kind) {
             // We don't batch any of these operations
-            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap | K::DocumentDeletionByFilter) => Break(this),
+            (this, K::IndexCreation | K::IndexUpdate | K::IndexSwap) => Break(this),
             // We must not batch tasks that don't have the same index creation rights if the index doesn't already exists.
             (this, kind) if !index_already_exists && this.allow_index_creation() == Some(false) && kind.allow_index_creation() == Some(true) => {
                 Break(this)
@@ -518,8 +508,7 @@ impl BatchKind {
                 BatchKind::IndexCreation { .. }
                 | BatchKind::IndexDeletion { .. }
                 | BatchKind::IndexUpdate { .. }
-                | BatchKind::IndexSwap { .. }
-                | BatchKind::DocumentDeletionByFilter { .. },
+                | BatchKind::IndexSwap { .. },
                 _,
             ) => {
                 unreachable!()

index-scheduler/src/batch.rs

@@ -19,20 +19,18 @@ one indexing operation.
 
 use std::collections::{BTreeSet, HashSet};
 use std::ffi::OsStr;
-use std::fmt;
 use std::fs::{self, File};
 use std::io::BufWriter;
 
 use dump::IndexMetadata;
-use log::{debug, error, info, trace};
-use meilisearch_types::error::Code;
+use log::{debug, error, info};
 use meilisearch_types::heed::{RoTxn, RwTxn};
 use meilisearch_types::milli::documents::{obkv_to_object, DocumentsBatchReader};
 use meilisearch_types::milli::heed::CompactionOption;
 use meilisearch_types::milli::update::{
-    IndexDocumentsConfig, IndexDocumentsMethod, IndexerConfig, Settings as MilliSettings,
+    DocumentDeletionResult, IndexDocumentsConfig, IndexDocumentsMethod, Settings as MilliSettings,
 };
-use meilisearch_types::milli::{self, Filter, BEU32};
+use meilisearch_types::milli::{self, BEU32};
 use meilisearch_types::settings::{apply_settings_to_builder, Settings, Unchecked};
 use meilisearch_types::tasks::{Details, IndexSwap, Kind, KindWithContent, Status, Task};
 use meilisearch_types::{compression, Index, VERSION_FILE_NAME};
@@ -43,7 +41,7 @@ use uuid::Uuid;
 
 use crate::autobatcher::{self, BatchKind};
 use crate::utils::{self, swap_index_uid_in_task};
-use crate::{Error, IndexScheduler, MustStopProcessing, ProcessingTasks, Result, TaskId};
+use crate::{Error, IndexScheduler, ProcessingTasks, Result, TaskId};
 
 /// Represents a combination of tasks that can all be processed at the same time.
 ///
@@ -104,9 +102,11 @@ pub(crate) enum IndexOperation {
         operations: Vec<DocumentOperation>,
         tasks: Vec<Task>,
     },
-    IndexDocumentDeletionByFilter {
+    DocumentDeletion {
         index_uid: String,
-        task: Task,
+        // The vec associated with each document deletion tasks.
+        documents: Vec<Vec<String>>,
+        tasks: Vec<Task>,
     },
     DocumentClear {
         index_uid: String,
@@ -155,11 +155,11 @@ impl Batch {
             }
             Batch::IndexOperation { op, .. } => match op {
                 IndexOperation::DocumentOperation { tasks, .. }
+                | IndexOperation::DocumentDeletion { tasks, .. }
                 | IndexOperation::Settings { tasks, .. }
                 | IndexOperation::DocumentClear { tasks, .. } => {
                     tasks.iter().map(|task| task.uid).collect()
                 }
-                IndexOperation::IndexDocumentDeletionByFilter { task, .. } => vec![task.uid],
                 IndexOperation::SettingsAndDocumentOperation {
                     document_import_tasks: tasks,
                     settings_tasks: other,
@@ -192,34 +192,11 @@ impl Batch {
     }
 }
 
-impl fmt::Display for Batch {
-    /// A text used when we debug the profiling reports.
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let index_uid = self.index_uid();
-        let tasks = self.ids();
-        match self {
-            Batch::TaskCancelation { .. } => f.write_str("TaskCancelation")?,
-            Batch::TaskDeletion(_) => f.write_str("TaskDeletion")?,
-            Batch::SnapshotCreation(_) => f.write_str("SnapshotCreation")?,
-            Batch::Dump(_) => f.write_str("Dump")?,
-            Batch::IndexOperation { op, .. } => write!(f, "{op}")?,
-            Batch::IndexCreation { .. } => f.write_str("IndexCreation")?,
-            Batch::IndexUpdate { .. } => f.write_str("IndexUpdate")?,
-            Batch::IndexDeletion { .. } => f.write_str("IndexDeletion")?,
-            Batch::IndexSwap { .. } => f.write_str("IndexSwap")?,
-        };
-        match index_uid {
-            Some(name) => f.write_fmt(format_args!(" on {name:?} from tasks: {tasks:?}")),
-            None => f.write_fmt(format_args!(" from tasks: {tasks:?}")),
-        }
-    }
-}
-
 impl IndexOperation {
     pub fn index_uid(&self) -> &str {
         match self {
             IndexOperation::DocumentOperation { index_uid, .. }
-            | IndexOperation::IndexDocumentDeletionByFilter { index_uid, .. }
+            | IndexOperation::DocumentDeletion { index_uid, .. }
             | IndexOperation::DocumentClear { index_uid, .. }
             | IndexOperation::Settings { index_uid, .. }
             | IndexOperation::DocumentClearAndSetting { index_uid, .. }
@@ -228,27 +205,6 @@ impl IndexOperation {
     }
 }
 
-impl fmt::Display for IndexOperation {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        match self {
-            IndexOperation::DocumentOperation { .. } => {
-                f.write_str("IndexOperation::DocumentOperation")
-            }
-            IndexOperation::IndexDocumentDeletionByFilter { .. } => {
-                f.write_str("IndexOperation::IndexDocumentDeletionByFilter")
-            }
-            IndexOperation::DocumentClear { .. } => f.write_str("IndexOperation::DocumentClear"),
-            IndexOperation::Settings { .. } => f.write_str("IndexOperation::Settings"),
-            IndexOperation::DocumentClearAndSetting { .. } => {
-                f.write_str("IndexOperation::DocumentClearAndSetting")
-            }
-            IndexOperation::SettingsAndDocumentOperation { .. } => {
-                f.write_str("IndexOperation::SettingsAndDocumentOperation")
-            }
-        }
-    }
-}
-
 impl IndexScheduler {
     /// Convert an [`BatchKind`](crate::autobatcher::BatchKind) into a [`Batch`].
     ///
@@ -271,21 +227,6 @@ impl IndexScheduler {
                 },
                 must_create_index,
             })),
-            BatchKind::DocumentDeletionByFilter { id } => {
-                let task = self.get_task(rtxn, id)?.ok_or(Error::CorruptedTaskQueue)?;
-                match &task.kind {
-                    KindWithContent::DocumentDeletionByFilter { index_uid, .. } => {
-                        Ok(Some(Batch::IndexOperation {
-                            op: IndexOperation::IndexDocumentDeletionByFilter {
-                                index_uid: index_uid.clone(),
-                                task,
-                            },
-                            must_create_index: false,
-                        }))
-                    }
-                    _ => unreachable!(),
-                }
-            }
             BatchKind::DocumentOperation { method, operation_ids, .. } => {
                 let tasks = self.get_existing_tasks(rtxn, operation_ids)?;
                 let primary_key = tasks
@@ -336,27 +277,18 @@ impl IndexScheduler {
             BatchKind::DocumentDeletion { deletion_ids } => {
                 let tasks = self.get_existing_tasks(rtxn, deletion_ids)?;
 
-                let mut operations = Vec::with_capacity(tasks.len());
-                let mut documents_counts = Vec::with_capacity(tasks.len());
+                let mut documents = Vec::new();
                 for task in &tasks {
                     match task.kind {
                         KindWithContent::DocumentDeletion { ref documents_ids, .. } => {
-                            operations.push(DocumentOperation::Delete(documents_ids.clone()));
-                            documents_counts.push(documents_ids.len() as u64);
+                            documents.push(documents_ids.clone())
                         }
                         _ => unreachable!(),
                     }
                 }
 
                 Ok(Some(Batch::IndexOperation {
-                    op: IndexOperation::DocumentOperation {
-                        index_uid,
-                        primary_key: None,
-                        method: IndexDocumentsMethod::ReplaceDocuments,
-                        documents_counts,
-                        operations,
-                        tasks,
-                    },
+                    op: IndexOperation::DocumentDeletion { index_uid, documents, tasks },
                     must_create_index,
                 }))
             }
@@ -519,8 +451,6 @@ impl IndexScheduler {
         #[cfg(test)]
         self.maybe_fail(crate::tests::FailureLocation::InsideCreateBatch)?;
 
-        puffin::profile_function!();
-
         let enqueued = &self.get_status(rtxn, Status::Enqueued)?;
         let to_cancel = self.get_kind(rtxn, Kind::TaskCancelation)? & enqueued;
 
@@ -625,9 +555,6 @@ impl IndexScheduler {
             self.maybe_fail(crate::tests::FailureLocation::PanicInsideProcessBatch)?;
             self.breakpoint(crate::Breakpoint::InsideProcessBatch);
         }
-
-        puffin::profile_function!(batch.to_string());
-
         match batch {
             Batch::TaskCancelation { mut task, previous_started_at, previous_processing_tasks } => {
                 // 1. Retrieve the tasks that matched the query at enqueue-time.
@@ -748,6 +675,9 @@ impl IndexScheduler {
                 }
 
                 // 3. Snapshot every indexes
+                // TODO we are opening all of the indexes it can be too much we should unload all
+                //      of the indexes we are trying to open. It would be even better to only unload
+                //      the ones that were opened by us. Or maybe use a LRU in the index mapper.
                 for result in self.index_mapper.index_mapping.iter(&rtxn)? {
                     let (name, uuid) = result?;
                     let index = self.index_mapper.index(&rtxn, name)?;
@@ -784,14 +714,6 @@ impl IndexScheduler {
                 // 5.3 Change the permission to make the snapshot readonly
                 let mut permissions = file.metadata()?.permissions();
                 permissions.set_readonly(true);
-                #[cfg(unix)]
-                {
-                    use std::os::unix::fs::PermissionsExt;
-                    #[allow(clippy::non_octal_unix_permissions)]
-                    //                     rwxrwxrwx
-                    permissions.set_mode(0b100100100);
-                }
-
                 file.set_permissions(permissions)?;
 
                 for task in &mut tasks {
@@ -866,15 +788,15 @@ impl IndexScheduler {
                 dump_tasks.flush()?;
 
                 // 3. Dump the indexes
-                self.index_mapper.try_for_each_index(&rtxn, |uid, index| -> Result<()> {
+                for (uid, index) in self.index_mapper.indexes(&rtxn)? {
                     let rtxn = index.read_txn()?;
                     let metadata = IndexMetadata {
-                        uid: uid.to_owned(),
+                        uid: uid.clone(),
                         primary_key: index.primary_key(&rtxn)?.map(String::from),
                         created_at: index.created_at(&rtxn)?,
                         updated_at: index.updated_at(&rtxn)?,
                     };
-                    let mut index_dumper = dump.create_index(uid, &metadata)?;
+                    let mut index_dumper = dump.create_index(&uid, &metadata)?;
 
                     let fields_ids_map = index.fields_ids_map(&rtxn)?;
                     let all_fields: Vec<_> = fields_ids_map.iter().map(|(id, _)| id).collect();
@@ -887,14 +809,9 @@ impl IndexScheduler {
                     }
 
                     // 3.2. Dump the settings
-                    let settings = meilisearch_types::settings::settings(index, &rtxn)?;
+                    let settings = meilisearch_types::settings::settings(&index, &rtxn)?;
                     index_dumper.settings(&settings)?;
-                    Ok(())
-                })?;
-
-                // 4. Dump experimental feature settings
-                let features = self.features().runtime_features();
-                dump.create_experimental_features(features)?;
+                }
 
                 let dump_uid = started_at.format(format_description!(
                     "[year repr:full][month repr:numerical][day padding:zero]-[hour padding:zero][minute padding:zero][second padding:zero][subsecond digits:3]"
@@ -910,38 +827,20 @@ impl IndexScheduler {
                 Ok(vec![task])
             }
             Batch::IndexOperation { op, must_create_index } => {
-                let index_uid = op.index_uid().to_string();
+                let index_uid = op.index_uid();
                 let index = if must_create_index {
                     // create the index if it doesn't already exist
                     let wtxn = self.env.write_txn()?;
-                    self.index_mapper.create_index(wtxn, &index_uid, None)?
+                    self.index_mapper.create_index(wtxn, index_uid, None)?
                 } else {
                     let rtxn = self.env.read_txn()?;
-                    self.index_mapper.index(&rtxn, &index_uid)?
+                    self.index_mapper.index(&rtxn, index_uid)?
                 };
 
                 let mut index_wtxn = index.write_txn()?;
                 let tasks = self.apply_index_operation(&mut index_wtxn, &index, op)?;
                 index_wtxn.commit()?;
 
-                // if the update processed successfully, we're going to store the new
-                // stats of the index. Since the tasks have already been processed and
-                // this is a non-critical operation. If it fails, we should not fail
-                // the entire batch.
-                let res = || -> Result<()> {
-                    let index_rtxn = index.read_txn()?;
-                    let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn)?;
-                    let mut wtxn = self.env.write_txn()?;
-                    self.index_mapper.store_stats_of(&mut wtxn, &index_uid, &stats)?;
-                    wtxn.commit()?;
-                    Ok(())
-                }();
-
-                match res {
-                    Ok(_) => (),
-                    Err(e) => error!("Could not write the stats of the index {}", e),
-                }
-
                 Ok(tasks)
             }
             Batch::IndexCreation { index_uid, primary_key, task } => {
@@ -972,31 +871,9 @@ impl IndexScheduler {
                     )?;
                     index_wtxn.commit()?;
                 }
-
-                // drop rtxn before starting a new wtxn on the same db
-                rtxn.commit()?;
-
                 task.status = Status::Succeeded;
                 task.details = Some(Details::IndexInfo { primary_key });
 
-                // if the update processed successfully, we're going to store the new
-                // stats of the index. Since the tasks have already been processed and
-                // this is a non-critical operation. If it fails, we should not fail
-                // the entire batch.
-                let res = || -> Result<()> {
-                    let mut wtxn = self.env.write_txn()?;
-                    let index_rtxn = index.read_txn()?;
-                    let stats = crate::index_mapper::IndexStats::new(&index, &index_rtxn)?;
-                    self.index_mapper.store_stats_of(&mut wtxn, &index_uid, &stats)?;
-                    wtxn.commit()?;
-                    Ok(())
-                }();
-
-                match res {
-                    Ok(_) => (),
-                    Err(e) => error!("Could not write the stats of the index {}", e),
-                }
-
                 Ok(vec![task])
             }
             Batch::IndexDeletion { index_uid, index_has_been_created, mut tasks } => {
@@ -1010,7 +887,7 @@ impl IndexScheduler {
                 }()
                 .unwrap_or_default();
 
-                // The write transaction is directly owned and committed inside.
+                // The write transaction is directly owned and commited inside.
                 match self.index_mapper.delete_index(wtxn, &index_uid) {
                     Ok(()) => (),
                     Err(Error::IndexNotFound(_)) if index_has_been_created => (),
@@ -1119,8 +996,6 @@ impl IndexScheduler {
         index: &'i Index,
         operation: IndexOperation,
     ) -> Result<Vec<Task>> {
-        puffin::profile_function!();
-
         match operation {
             IndexOperation::DocumentClear { mut tasks, .. } => {
                 let count = milli::update::ClearDocuments::new(index_wtxn, index).execute()?;
@@ -1188,7 +1063,7 @@ impl IndexScheduler {
                     index,
                     indexer_config,
                     config,
-                    |indexing_step| trace!("update: {:?}", indexing_step),
+                    |indexing_step| debug!("update: {:?}", indexing_step),
                     || must_stop_processing.get(),
                 )?;
 
@@ -1235,8 +1110,7 @@ impl IndexScheduler {
                             let (new_builder, user_result) =
                                 builder.remove_documents(document_ids)?;
                             builder = new_builder;
-                            // Uses Invariant: remove documents actually always returns Ok for the inner result
-                            let count = user_result.unwrap();
+
                             let provided_ids =
                                 if let Some(Details::DocumentDeletion { provided_ids, .. }) =
                                     task.details
@@ -1247,11 +1121,23 @@ impl IndexScheduler {
                                     unreachable!();
                                 };
 
-                            task.status = Status::Succeeded;
-                            task.details = Some(Details::DocumentDeletion {
-                                provided_ids,
-                                deleted_documents: Some(count),
-                            });
+                            match user_result {
+                                Ok(count) => {
+                                    task.status = Status::Succeeded;
+                                    task.details = Some(Details::DocumentDeletion {
+                                        provided_ids,
+                                        deleted_documents: Some(count),
+                                    });
+                                }
+                                Err(e) => {
+                                    task.status = Status::Failed;
+                                    task.details = Some(Details::DocumentDeletion {
+                                        provided_ids,
+                                        deleted_documents: Some(0),
+                                    });
+                                    task.error = Some(milli::Error::from(e).into());
+                                }
+                            }
                         }
                     }
                 }
@@ -1266,59 +1152,30 @@ impl IndexScheduler {
                         milli::update::Settings::new(index_wtxn, index, indexer_config);
                     builder.reset_primary_key();
                     builder.execute(
-                        |indexing_step| trace!("update: {:?}", indexing_step),
+                        |indexing_step| debug!("update: {:?}", indexing_step),
                         || must_stop_processing.clone().get(),
                     )?;
                 }
 
                 Ok(tasks)
             }
-            IndexOperation::IndexDocumentDeletionByFilter { mut task, index_uid: _ } => {
-                let filter =
-                    if let KindWithContent::DocumentDeletionByFilter { filter_expr, .. } =
-                        &task.kind
-                    {
-                        filter_expr
-                    } else {
-                        unreachable!()
-                    };
-                let deleted_documents = delete_document_by_filter(
-                    index_wtxn,
-                    filter,
-                    self.index_mapper.indexer_config(),
-                    self.must_stop_processing.clone(),
-                    index,
-                );
-                let original_filter = if let Some(Details::DocumentDeletionByFilter {
-                    original_filter,
-                    deleted_documents: _,
-                }) = task.details
-                {
-                    original_filter
-                } else {
-                    // In the case of a `documentDeleteByFilter` the details MUST be set
-                    unreachable!();
-                };
+            IndexOperation::DocumentDeletion { index_uid: _, documents, mut tasks } => {
+                let mut builder = milli::update::DeleteDocuments::new(index_wtxn, index)?;
+                documents.iter().flatten().for_each(|id| {
+                    builder.delete_external_id(id);
+                });
 
-                match deleted_documents {
-                    Ok(deleted_documents) => {
-                        task.status = Status::Succeeded;
-                        task.details = Some(Details::DocumentDeletionByFilter {
-                            original_filter,
-                            deleted_documents: Some(deleted_documents),
-                        });
-                    }
-                    Err(e) => {
-                        task.status = Status::Failed;
-                        task.details = Some(Details::DocumentDeletionByFilter {
-                            original_filter,
-                            deleted_documents: Some(0),
-                        });
-                        task.error = Some(e.into());
-                    }
+                let DocumentDeletionResult { deleted_documents, .. } = builder.execute()?;
+
+                for (task, documents) in tasks.iter_mut().zip(documents) {
+                    task.status = Status::Succeeded;
+                    task.details = Some(Details::DocumentDeletion {
+                        provided_ids: documents.len(),
+                        deleted_documents: Some(deleted_documents.min(documents.len() as u64)),
+                    });
                 }
 
-                Ok(vec![task])
+                Ok(tasks)
             }
             IndexOperation::Settings { index_uid: _, settings, mut tasks } => {
                 let indexer_config = self.index_mapper.indexer_config();
@@ -1518,43 +1375,3 @@ impl IndexScheduler {
         Ok(content_files_to_delete)
     }
 }
-
-fn delete_document_by_filter<'a>(
-    wtxn: &mut RwTxn<'a, '_>,
-    filter: &serde_json::Value,
-    indexer_config: &IndexerConfig,
-    must_stop_processing: MustStopProcessing,
-    index: &'a Index,
-) -> Result<u64> {
-    let filter = Filter::from_json(filter)?;
-    Ok(if let Some(filter) = filter {
-        let candidates = filter.evaluate(wtxn, index).map_err(|err| match err {
-            milli::Error::UserError(milli::UserError::InvalidFilter(_)) => {
-                Error::from(err).with_custom_error_code(Code::InvalidDocumentFilter)
-            }
-            e => e.into(),
-        })?;
-
-        let config = IndexDocumentsConfig {
-            update_method: IndexDocumentsMethod::ReplaceDocuments,
-            ..Default::default()
-        };
-
-        let mut builder = milli::update::IndexDocuments::new(
-            wtxn,
-            index,
-            indexer_config,
-            config,
-            |indexing_step| debug!("update: {:?}", indexing_step),
-            || must_stop_processing.get(),
-        )?;
-
-        let (new_builder, count) = builder.remove_documents_from_db_no_batch(&candidates)?;
-        builder = new_builder;
-
-        let _ = builder.execute()?;
-        count
-    } else {
-        0
-    })
-}

index-scheduler/src/error.rs

@@ -46,8 +46,6 @@ impl From<DateField> for Code {
 #[allow(clippy::large_enum_variant)]
 #[derive(Error, Debug)]
 pub enum Error {
-    #[error("{1}")]
-    WithCustomErrorCode(Code, Box<Self>),
     #[error("Index `{0}` not found.")]
     IndexNotFound(String),
     #[error("Index `{0}` already exists.")]
@@ -63,8 +61,6 @@ pub enum Error {
     SwapDuplicateIndexesFound(Vec<String>),
     #[error("Index `{0}` not found.")]
     SwapIndexNotFound(String),
-    #[error("Meilisearch cannot receive write operations because the limit of the task database has been reached. Please delete tasks to continue performing write operations.")]
-    NoSpaceLeftInTaskQueue,
     #[error(
         "Indexes {} not found.",
         .0.iter().map(|s| format!("`{}`", s)).collect::<Vec<_>>().join(", ")
@@ -123,8 +119,6 @@ pub enum Error {
     IoError(#[from] std::io::Error),
     #[error(transparent)]
     Persist(#[from] tempfile::PersistError),
-    #[error(transparent)]
-    FeatureNotEnabled(#[from] FeatureNotEnabledError),
 
     #[error(transparent)]
     Anyhow(#[from] anyhow::Error),
@@ -138,70 +132,11 @@ pub enum Error {
     TaskDatabaseUpdate(Box<Self>),
     #[error(transparent)]
     HeedTransaction(heed::Error),
-
-    #[cfg(test)]
-    #[error("Planned failure for tests.")]
-    PlannedFailure,
-}
-
-#[derive(Debug, thiserror::Error)]
-#[error(
-    "{disabled_action} requires enabling the `{feature}` experimental feature. See {issue_link}"
-)]
-pub struct FeatureNotEnabledError {
-    pub disabled_action: &'static str,
-    pub feature: &'static str,
-    pub issue_link: &'static str,
-}
-
-impl Error {
-    pub fn is_recoverable(&self) -> bool {
-        match self {
-            Error::IndexNotFound(_)
-            | Error::WithCustomErrorCode(_, _)
-            | Error::IndexAlreadyExists(_)
-            | Error::SwapDuplicateIndexFound(_)
-            | Error::SwapDuplicateIndexesFound(_)
-            | Error::SwapIndexNotFound(_)
-            | Error::NoSpaceLeftInTaskQueue
-            | Error::SwapIndexesNotFound(_)
-            | Error::CorruptedDump
-            | Error::InvalidTaskDate { .. }
-            | Error::InvalidTaskUids { .. }
-            | Error::InvalidTaskStatuses { .. }
-            | Error::InvalidTaskTypes { .. }
-            | Error::InvalidTaskCanceledBy { .. }
-            | Error::InvalidIndexUid { .. }
-            | Error::TaskNotFound(_)
-            | Error::TaskDeletionWithEmptyQuery
-            | Error::TaskCancelationWithEmptyQuery
-            | Error::Dump(_)
-            | Error::Heed(_)
-            | Error::Milli(_)
-            | Error::ProcessBatchPanicked
-            | Error::FileStore(_)
-            | Error::IoError(_)
-            | Error::Persist(_)
-            | Error::FeatureNotEnabled(_)
-            | Error::Anyhow(_) => true,
-            Error::CreateBatch(_)
-            | Error::CorruptedTaskQueue
-            | Error::TaskDatabaseUpdate(_)
-            | Error::HeedTransaction(_) => false,
-            #[cfg(test)]
-            Error::PlannedFailure => false,
-        }
-    }
-
-    pub fn with_custom_error_code(self, code: Code) -> Self {
-        Self::WithCustomErrorCode(code, Box::new(self))
-    }
 }
 
 impl ErrorCode for Error {
     fn error_code(&self) -> Code {
         match self {
-            Error::WithCustomErrorCode(code, _) => *code,
             Error::IndexNotFound(_) => Code::IndexNotFound,
             Error::IndexAlreadyExists(_) => Code::IndexAlreadyExists,
             Error::SwapDuplicateIndexesFound(_) => Code::InvalidSwapDuplicateIndexFound,
@@ -217,8 +152,6 @@ impl ErrorCode for Error {
             Error::TaskNotFound(_) => Code::TaskNotFound,
             Error::TaskDeletionWithEmptyQuery => Code::MissingTaskFilters,
             Error::TaskCancelationWithEmptyQuery => Code::MissingTaskFilters,
-            // TODO: not sure of the Code to use
-            Error::NoSpaceLeftInTaskQueue => Code::NoSpaceLeftOnDevice,
             Error::Dump(e) => e.error_code(),
             Error::Milli(e) => e.error_code(),
             Error::ProcessBatchPanicked => Code::Internal,
@@ -227,7 +160,6 @@ impl ErrorCode for Error {
             Error::FileStore(e) => e.error_code(),
             Error::IoError(e) => e.error_code(),
             Error::Persist(e) => e.error_code(),
-            Error::FeatureNotEnabled(_) => Code::FeatureNotEnabled,
 
             // Irrecoverable errors
             Error::Anyhow(_) => Code::Internal,
@@ -235,9 +167,6 @@ impl ErrorCode for Error {
             Error::CorruptedDump => Code::Internal,
             Error::TaskDatabaseUpdate(_) => Code::Internal,
             Error::CreateBatch(_) => Code::Internal,
-
-            #[cfg(test)]
-            Error::PlannedFailure => Code::Internal,
         }
     }
 }

index-scheduler/src/features.rs

@@ -1,129 +0,0 @@
-use std::sync::{Arc, RwLock};
-
-use meilisearch_types::features::{InstanceTogglableFeatures, RuntimeTogglableFeatures};
-use meilisearch_types::heed::types::{SerdeJson, Str};
-use meilisearch_types::heed::{Database, Env, RwTxn};
-
-use crate::error::FeatureNotEnabledError;
-use crate::Result;
-
-const EXPERIMENTAL_FEATURES: &str = "experimental-features";
-
-#[derive(Clone)]
-pub(crate) struct FeatureData {
-    persisted: Database<Str, SerdeJson<RuntimeTogglableFeatures>>,
-    runtime: Arc<RwLock<RuntimeTogglableFeatures>>,
-}
-
-#[derive(Debug, Clone, Copy)]
-pub struct RoFeatures {
-    runtime: RuntimeTogglableFeatures,
-}
-
-impl RoFeatures {
-    fn new(data: &FeatureData) -> Self {
-        let runtime = data.runtime_features();
-        Self { runtime }
-    }
-
-    pub fn runtime_features(&self) -> RuntimeTogglableFeatures {
-        self.runtime
-    }
-
-    pub fn check_score_details(&self) -> Result<()> {
-        if self.runtime.score_details {
-            Ok(())
-        } else {
-            Err(FeatureNotEnabledError {
-                disabled_action: "Computing score details",
-                feature: "score details",
-                issue_link: "https://github.com/meilisearch/product/discussions/674",
-            }
-            .into())
-        }
-    }
-
-    pub fn check_metrics(&self) -> Result<()> {
-        if self.runtime.metrics {
-            Ok(())
-        } else {
-            Err(FeatureNotEnabledError {
-                disabled_action: "Getting metrics",
-                feature: "metrics",
-                issue_link: "https://github.com/meilisearch/product/discussions/625",
-            }
-            .into())
-        }
-    }
-
-    pub fn check_vector(&self) -> Result<()> {
-        if self.runtime.vector_store {
-            Ok(())
-        } else {
-            Err(FeatureNotEnabledError {
-                disabled_action: "Passing `vector` as a query parameter",
-                feature: "vector store",
-                issue_link: "https://github.com/meilisearch/product/discussions/677",
-            }
-            .into())
-        }
-    }
-
-    pub fn check_puffin(&self) -> Result<()> {
-        if self.runtime.export_puffin_reports {
-            Ok(())
-        } else {
-            Err(FeatureNotEnabledError {
-                disabled_action: "Outputting Puffin reports to disk",
-                feature: "export puffin reports",
-                issue_link: "https://github.com/meilisearch/product/discussions/693",
-            }
-            .into())
-        }
-    }
-}
-
-impl FeatureData {
-    pub fn new(env: &Env, instance_features: InstanceTogglableFeatures) -> Result<Self> {
-        let mut wtxn = env.write_txn()?;
-        let runtime_features_db = env.create_database(&mut wtxn, Some(EXPERIMENTAL_FEATURES))?;
-        wtxn.commit()?;
-
-        let txn = env.read_txn()?;
-        let persisted_features: RuntimeTogglableFeatures =
-            runtime_features_db.get(&txn, EXPERIMENTAL_FEATURES)?.unwrap_or_default();
-        let runtime = Arc::new(RwLock::new(RuntimeTogglableFeatures {
-            metrics: instance_features.metrics || persisted_features.metrics,
-            ..persisted_features
-        }));
-
-        Ok(Self { persisted: runtime_features_db, runtime })
-    }
-
-    pub fn put_runtime_features(
-        &self,
-        mut wtxn: RwTxn,
-        features: RuntimeTogglableFeatures,
-    ) -> Result<()> {
-        self.persisted.put(&mut wtxn, EXPERIMENTAL_FEATURES, &features)?;
-        wtxn.commit()?;
-
-        // safe to unwrap, the lock will only fail if:
-        // 1. requested by the same thread concurrently -> it is called and released in methods that don't call each other
-        // 2. there's a panic while the thread is held -> it is only used for an assignment here.
-        let mut toggled_features = self.runtime.write().unwrap();
-        *toggled_features = features;
-        Ok(())
-    }
-
-    fn runtime_features(&self) -> RuntimeTogglableFeatures {
-        // sound to unwrap, the lock will only fail if:
-        // 1. requested by the same thread concurrently -> it is called and released in methods that don't call each other
-        // 2. there's a panic while the thread is held -> it is only used for copying the data here
-        *self.runtime.read().unwrap()
-    }
-
-    pub fn features(&self) -> RoFeatures {
-        RoFeatures::new(self)
-    }
-}

index-scheduler/src/index_mapper.rs

@@ -0,0 +1,362 @@
+use std::collections::hash_map::Entry;
+use std::collections::HashMap;
+use std::path::{Path, PathBuf};
+use std::sync::{Arc, RwLock};
+use std::{fs, thread};
+
+use log::error;
+use meilisearch_types::heed::types::Str;
+use meilisearch_types::heed::{Database, Env, EnvOpenOptions, RoTxn, RwTxn};
+use meilisearch_types::milli::update::IndexerConfig;
+use meilisearch_types::milli::Index;
+use synchronoise::SignalEvent;
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+use self::IndexStatus::{Available, BeingDeleted, BeingResized};
+use crate::uuid_codec::UuidCodec;
+use crate::{clamp_to_page_size, Error, Result};
+
+const INDEX_MAPPING: &str = "index-mapping";
+
+/// Structure managing meilisearch's indexes.
+///
+/// It is responsible for:
+/// 1. Creating new indexes
+/// 2. Opening indexes and storing references to these opened indexes
+/// 3. Accessing indexes through their uuid
+/// 4. Mapping a user-defined name to each index uuid.
+#[derive(Clone)]
+pub struct IndexMapper {
+    /// Keep track of the opened indexes. Used mainly by the index resolver.
+    index_map: Arc<RwLock<HashMap<Uuid, IndexStatus>>>,
+
+    /// Map an index name with an index uuid currently available on disk.
+    pub(crate) index_mapping: Database<Str, UuidCodec>,
+
+    /// Path to the folder where the LMDB environments of each index are.
+    base_path: PathBuf,
+    index_size: usize,
+    pub indexer_config: Arc<IndexerConfig>,
+}
+
+/// Whether the index is available for use or is forbidden to be inserted back in the index map
+#[allow(clippy::large_enum_variant)]
+#[derive(Clone)]
+pub enum IndexStatus {
+    /// Do not insert it back in the index map as it is currently being deleted.
+    BeingDeleted,
+    /// Temporarily do not insert the index in the index map as it is currently being resized.
+    BeingResized(Arc<SignalEvent>),
+    /// You can use the index without worrying about anything.
+    Available(Index),
+}
+
+impl IndexMapper {
+    pub fn new(
+        env: &Env,
+        base_path: PathBuf,
+        index_size: usize,
+        indexer_config: IndexerConfig,
+    ) -> Result<Self> {
+        Ok(Self {
+            index_map: Arc::default(),
+            index_mapping: env.create_database(Some(INDEX_MAPPING))?,
+            base_path,
+            index_size,
+            indexer_config: Arc::new(indexer_config),
+        })
+    }
+
+    /// Create or open an index in the specified path.
+    /// The path *must* exists or an error will be thrown.
+    fn create_or_open_index(
+        &self,
+        path: &Path,
+        date: Option<(OffsetDateTime, OffsetDateTime)>,
+        map_size: usize,
+    ) -> Result<Index> {
+        let mut options = EnvOpenOptions::new();
+        options.map_size(clamp_to_page_size(map_size));
+        options.max_readers(1024);
+
+        if let Some((created, updated)) = date {
+            Ok(Index::new_with_creation_dates(options, path, created, updated)?)
+        } else {
+            Ok(Index::new(options, path)?)
+        }
+    }
+
+    /// Get or create the index.
+    pub fn create_index(
+        &self,
+        mut wtxn: RwTxn,
+        name: &str,
+        date: Option<(OffsetDateTime, OffsetDateTime)>,
+    ) -> Result<Index> {
+        match self.index(&wtxn, name) {
+            Ok(index) => {
+                wtxn.commit()?;
+                Ok(index)
+            }
+            Err(Error::IndexNotFound(_)) => {
+                let uuid = Uuid::new_v4();
+                self.index_mapping.put(&mut wtxn, name, &uuid)?;
+
+                let index_path = self.base_path.join(uuid.to_string());
+                fs::create_dir_all(&index_path)?;
+
+                let index = self.create_or_open_index(&index_path, date, self.index_size)?;
+
+                wtxn.commit()?;
+                // Error if the UUIDv4 somehow already exists in the map, since it should be fresh.
+                // This is very unlikely to happen in practice.
+                // TODO: it would be better to lazily create the index. But we need an Index::open function for milli.
+                if self.index_map.write().unwrap().insert(uuid, Available(index.clone())).is_some()
+                {
+                    panic!("Uuid v4 conflict: index with UUID {uuid} already exists.");
+                }
+
+                Ok(index)
+            }
+            error => error,
+        }
+    }
+
+    /// Removes the index from the mapping table and the in-memory index map
+    /// but keeps the associated tasks.
+    pub fn delete_index(&self, mut wtxn: RwTxn, name: &str) -> Result<()> {
+        let uuid = self
+            .index_mapping
+            .get(&wtxn, name)?
+            .ok_or_else(|| Error::IndexNotFound(name.to_string()))?;
+
+        // Once we retrieved the UUID of the index we remove it from the mapping table.
+        assert!(self.index_mapping.delete(&mut wtxn, name)?);
+
+        wtxn.commit()?;
+        // We remove the index from the in-memory index map.
+        let closing_event = loop {
+            let mut lock = self.index_map.write().unwrap();
+            let resize_operation = match lock.insert(uuid, BeingDeleted) {
+                Some(Available(index)) => break Some(index.prepare_for_closing()),
+                // The target index is in the middle of a resize operation.
+                // Wait for this operation to complete, then try again.
+                Some(BeingResized(resize_operation)) => resize_operation.clone(),
+                // The index is already being deleted or doesn't exist.
+                // It's OK to remove it from the map again.
+                _ => break None,
+            };
+
+            // Avoiding deadlocks: we need to drop the lock before waiting for the end of the resize, which
+            // will involve operations on the very map we're locking.
+            drop(lock);
+            resize_operation.wait();
+        };
+
+        let index_map = self.index_map.clone();
+        let index_path = self.base_path.join(uuid.to_string());
+        let index_name = name.to_string();
+        thread::Builder::new()
+            .name(String::from("index_deleter"))
+            .spawn(move || {
+                // We first wait to be sure that the previously opened index is effectively closed.
+                // This can take a lot of time, this is why we do that in a seperate thread.
+                if let Some(closing_event) = closing_event {
+                    closing_event.wait();
+                }
+
+                // Then we remove the content from disk.
+                if let Err(e) = fs::remove_dir_all(&index_path) {
+                    error!(
+                        "An error happened when deleting the index {} ({}): {}",
+                        index_name, uuid, e
+                    );
+                }
+
+                // Finally we remove the entry from the index map.
+                assert!(matches!(index_map.write().unwrap().remove(&uuid), Some(BeingDeleted)));
+            })
+            .unwrap();
+
+        Ok(())
+    }
+
+    pub fn exists(&self, rtxn: &RoTxn, name: &str) -> Result<bool> {
+        Ok(self.index_mapping.get(rtxn, name)?.is_some())
+    }
+
+    /// Resizes the maximum size of the specified index to the double of its current maximum size.
+    ///
+    /// This operation involves closing the underlying environment and so can take a long time to complete.
+    ///
+    /// # Panics
+    ///
+    /// - If the Index corresponding to the passed name is concurrently being deleted/resized or cannot be found in the
+    ///   in memory hash map.
+    pub fn resize_index(&self, rtxn: &RoTxn, name: &str) -> Result<()> {
+        // fixme: factor to a function?
+        let uuid = self
+            .index_mapping
+            .get(rtxn, name)?
+            .ok_or_else(|| Error::IndexNotFound(name.to_string()))?;
+
+        // We remove the index from the in-memory index map.
+        let mut lock = self.index_map.write().unwrap();
+        // signal that will be sent when the resize operation completes
+        let resize_operation = Arc::new(SignalEvent::manual(false));
+        let index = match lock.insert(uuid, BeingResized(resize_operation)) {
+            Some(Available(index)) => index,
+            Some(previous_status) => {
+                lock.insert(uuid, previous_status);
+                panic!(
+                    "Attempting to resize index {name} that is already being resized or deleted."
+                )
+            }
+            None => {
+                panic!("Could not find the status of index {name} in the in-memory index mapper.")
+            }
+        };
+
+        drop(lock);
+
+        let resize_succeeded = (move || {
+            let current_size = index.map_size()?;
+            let new_size = current_size * 2;
+            let closing_event = index.prepare_for_closing();
+
+            log::debug!("Waiting for index {name} to close");
+
+            if !closing_event.wait_timeout(std::time::Duration::from_secs(600)) {
+                // fail after 10 minutes waiting
+                panic!("Could not resize index {name} (unable to close it)");
+            }
+
+            log::info!("Resized index {name} from {current_size} to {new_size} bytes");
+            let index_path = self.base_path.join(uuid.to_string());
+            let index = self.create_or_open_index(&index_path, None, new_size)?;
+            Ok(index)
+        })();
+
+        // Put the map back to a consistent state.
+        // Even if there was an error we don't want to leave the map in an inconsistent state as it would cause
+        // deadlocks.
+        let mut lock = self.index_map.write().unwrap();
+        let (resize_operation, resize_succeeded) = match resize_succeeded {
+            Ok(index) => {
+                // insert the resized index
+                let Some(BeingResized(resize_operation)) = lock.insert(uuid, Available(index)) else {
+                    panic!("Index state for index {name} was modified while it was being resized")
+                };
+
+                (resize_operation, Ok(()))
+            }
+            Err(error) => {
+                // there was an error, not much we can do... delete the index from the in-memory map to prevent future errors
+                let Some(BeingResized(resize_operation)) = lock.remove(&uuid) else {
+                                    panic!("Index state for index {name} was modified while it was being resized")
+                                };
+                (resize_operation, Err(error))
+            }
+        };
+
+        // drop the lock before signaling completion so that other threads don't immediately await on the lock after waking up.
+        drop(lock);
+        resize_operation.signal();
+
+        resize_succeeded
+    }
+
+    /// Return an index, may open it if it wasn't already opened.
+    pub fn index(&self, rtxn: &RoTxn, name: &str) -> Result<Index> {
+        let uuid = self
+            .index_mapping
+            .get(rtxn, name)?
+            .ok_or_else(|| Error::IndexNotFound(name.to_string()))?;
+
+        // we clone here to drop the lock before entering the match
+        let index = loop {
+            let index = self.index_map.read().unwrap().get(&uuid).cloned();
+
+            match index {
+                Some(Available(index)) => break index,
+                Some(BeingResized(ref resize_operation)) => {
+                    // Avoiding deadlocks: no lock taken while doing this operation.
+                    resize_operation.wait();
+                    continue;
+                }
+                Some(BeingDeleted) => return Err(Error::IndexNotFound(name.to_string())),
+                // since we're lazy, it's possible that the index has not been opened yet.
+                None => {
+                    let mut index_map = self.index_map.write().unwrap();
+                    // between the read lock and the write lock it's not impossible
+                    // that someone already opened the index (eg if two search happens
+                    // at the same time), thus before opening it we check a second time
+                    // if it's not already there.
+                    // Since there is a good chance it's not already there we can use
+                    // the entry method.
+                    match index_map.entry(uuid) {
+                        Entry::Vacant(entry) => {
+                            let index_path = self.base_path.join(uuid.to_string());
+
+                            let index =
+                                self.create_or_open_index(&index_path, None, self.index_size)?;
+                            entry.insert(Available(index.clone()));
+                            break index;
+                        }
+                        Entry::Occupied(entry) => match entry.get() {
+                            Available(index) => break index.clone(),
+                            BeingResized(resize_operation) => {
+                                // Avoiding the deadlock: we drop the lock before waiting
+                                let resize_operation = resize_operation.clone();
+                                drop(index_map);
+                                resize_operation.wait();
+                                continue;
+                            }
+                            BeingDeleted => return Err(Error::IndexNotFound(name.to_string())),
+                        },
+                    }
+                }
+            }
+        };
+
+        Ok(index)
+    }
+
+    /// Return all indexes, may open them if they weren't already opened.
+    pub fn indexes(&self, rtxn: &RoTxn) -> Result<Vec<(String, Index)>> {
+        self.index_mapping
+            .iter(rtxn)?
+            .map(|ret| {
+                ret.map_err(Error::from).and_then(|(name, _)| {
+                    self.index(rtxn, name).map(|index| (name.to_string(), index))
+                })
+            })
+            .collect()
+    }
+
+    /// Swap two index names.
+    pub fn swap(&self, wtxn: &mut RwTxn, lhs: &str, rhs: &str) -> Result<()> {
+        let lhs_uuid = self
+            .index_mapping
+            .get(wtxn, lhs)?
+            .ok_or_else(|| Error::IndexNotFound(lhs.to_string()))?;
+        let rhs_uuid = self
+            .index_mapping
+            .get(wtxn, rhs)?
+            .ok_or_else(|| Error::IndexNotFound(rhs.to_string()))?;
+
+        self.index_mapping.put(wtxn, lhs, &rhs_uuid)?;
+        self.index_mapping.put(wtxn, rhs, &lhs_uuid)?;
+
+        Ok(())
+    }
+
+    pub fn index_exists(&self, rtxn: &RoTxn, name: &str) -> Result<bool> {
+        Ok(self.index_mapping.get(rtxn, name)?.is_some())
+    }
+
+    pub fn indexer_config(&self) -> &IndexerConfig {
+        &self.indexer_config
+    }
+}

index-scheduler/src/index_mapper/index_map.rs

@@ -1,394 +0,0 @@
-/// the map size to use when we don't succeed in reading it in indexes.
-const DEFAULT_MAP_SIZE: usize = 10 * 1024 * 1024 * 1024; // 10 GiB
-
-use std::collections::BTreeMap;
-use std::path::Path;
-use std::time::Duration;
-
-use meilisearch_types::heed::flags::Flags;
-use meilisearch_types::heed::{EnvClosingEvent, EnvOpenOptions};
-use meilisearch_types::milli::Index;
-use time::OffsetDateTime;
-use uuid::Uuid;
-
-use super::IndexStatus::{self, Available, BeingDeleted, Closing, Missing};
-use crate::lru::{InsertionOutcome, LruMap};
-use crate::{clamp_to_page_size, Result};
-
-/// Keep an internally consistent view of the open indexes in memory.
-///
-/// This view is made of an LRU cache that will evict the least frequently used indexes when new indexes are opened.
-/// Indexes that are being closed (for resizing or due to cache eviction) or deleted cannot be evicted from the cache and
-/// are stored separately.
-///
-/// This view provides operations to change the state of the index as it is known in memory:
-/// open an index (making it available for queries), close an index (specifying the new size it should be opened with),
-/// delete an index.
-///
-/// External consistency with the other bits of data of an index is provided by the `IndexMapper` parent structure.
-pub struct IndexMap {
-    /// A LRU map of indexes that are in the open state and available for queries.
-    available: LruMap<Uuid, Index>,
-    /// A map of indexes that are not available for queries, either because they are being deleted
-    /// or because they are being closed.
-    ///
-    /// If they are being deleted, the UUID points to `None`.
-    unavailable: BTreeMap<Uuid, Option<ClosingIndex>>,
-
-    /// A monotonically increasing generation number, used to differentiate between multiple successive index closing requests.
-    ///
-    /// Because multiple readers could be waiting on an index to close, the following could theoretically happen:
-    ///
-    /// 1. Multiple readers wait for the index closing to occur.
-    /// 2. One of them "wins the race", takes the lock and then removes the index that finished closing from the map.
-    /// 3. The index is reopened, but must be closed again (such as being resized again).
-    /// 4. One reader that "lost the race" in (2) wakes up and tries to take the lock and remove the index from the map.
-    ///
-    /// In that situation, the index may or may not have finished closing. The `generation` field allows to remember which
-    /// closing request was made, so the reader that "lost the race" has the old generation and will need to wait again for the index
-    /// to close.
-    generation: usize,
-}
-
-#[derive(Clone)]
-pub struct ClosingIndex {
-    uuid: Uuid,
-    closing_event: EnvClosingEvent,
-    enable_mdb_writemap: bool,
-    map_size: usize,
-    generation: usize,
-}
-
-impl ClosingIndex {
-    /// Waits for the index to be definitely closed.
-    ///
-    /// To avoid blocking, users should relinquish their locks to the IndexMap before calling this function.
-    ///
-    /// After the index is physically closed, the in memory map must still be updated to take this into account.
-    /// To do so, a `ReopenableIndex` is returned, that can be used to either definitely close or definitely open
-    /// the index without waiting anymore.
-    pub fn wait_timeout(self, timeout: Duration) -> Option<ReopenableIndex> {
-        self.closing_event.wait_timeout(timeout).then_some(ReopenableIndex {
-            uuid: self.uuid,
-            enable_mdb_writemap: self.enable_mdb_writemap,
-            map_size: self.map_size,
-            generation: self.generation,
-        })
-    }
-}
-
-pub struct ReopenableIndex {
-    uuid: Uuid,
-    enable_mdb_writemap: bool,
-    map_size: usize,
-    generation: usize,
-}
-
-impl ReopenableIndex {
-    /// Attempts to reopen the index, which can result in the index being reopened again or not
-    /// (e.g. if another thread already opened and closed the index again).
-    ///
-    /// Use get again on the IndexMap to get the updated status.
-    ///
-    /// Fails if the underlying index creation fails.
-    ///
-    /// # Status table
-    ///
-    /// | Previous Status | New Status                                   |
-    /// |-----------------|----------------------------------------------|
-    /// | Missing         | Missing                                      |
-    /// | BeingDeleted    | BeingDeleted                                 |
-    /// | Closing         | Available or Closing depending on generation |
-    /// | Available       | Available                                    |
-    ///
-    pub fn reopen(self, map: &mut IndexMap, path: &Path) -> Result<()> {
-        if let Closing(reopen) = map.get(&self.uuid) {
-            if reopen.generation != self.generation {
-                return Ok(());
-            }
-            map.unavailable.remove(&self.uuid);
-            map.create(&self.uuid, path, None, self.enable_mdb_writemap, self.map_size)?;
-        }
-        Ok(())
-    }
-
-    /// Attempts to close the index, which may or may not result in the index being closed
-    /// (e.g. if another thread already reopened the index again).
-    ///
-    /// Use get again on the IndexMap to get the updated status.
-    ///
-    /// # Status table
-    ///
-    /// | Previous Status | New Status                                 |
-    /// |-----------------|--------------------------------------------|
-    /// | Missing         | Missing                                    |
-    /// | BeingDeleted    | BeingDeleted                               |
-    /// | Closing         | Missing or Closing depending on generation |
-    /// | Available       | Available                                  |
-    pub fn close(self, map: &mut IndexMap) {
-        if let Closing(reopen) = map.get(&self.uuid) {
-            if reopen.generation != self.generation {
-                return;
-            }
-            map.unavailable.remove(&self.uuid);
-        }
-    }
-}
-
-impl IndexMap {
-    pub fn new(cap: usize) -> IndexMap {
-        Self { unavailable: Default::default(), available: LruMap::new(cap), generation: 0 }
-    }
-
-    /// Gets the current status of an index in the map.
-    ///
-    /// If the index is available it can be accessed from the returned status.
-    pub fn get(&self, uuid: &Uuid) -> IndexStatus {
-        self.available
-            .get(uuid)
-            .map(|index| Available(index.clone()))
-            .unwrap_or_else(|| self.get_unavailable(uuid))
-    }
-
-    fn get_unavailable(&self, uuid: &Uuid) -> IndexStatus {
-        match self.unavailable.get(uuid) {
-            Some(Some(reopen)) => Closing(reopen.clone()),
-            Some(None) => BeingDeleted,
-            None => Missing,
-        }
-    }
-
-    /// Attempts to create a new index that wasn't existing before.
-    ///
-    /// # Status table
-    ///
-    /// | Previous Status | New Status |
-    /// |-----------------|------------|
-    /// | Missing         | Available  |
-    /// | BeingDeleted    | panics     |
-    /// | Closing         | panics     |
-    /// | Available       | panics     |
-    ///
-    pub fn create(
-        &mut self,
-        uuid: &Uuid,
-        path: &Path,
-        date: Option<(OffsetDateTime, OffsetDateTime)>,
-        enable_mdb_writemap: bool,
-        map_size: usize,
-    ) -> Result<Index> {
-        if !matches!(self.get_unavailable(uuid), Missing) {
-            panic!("Attempt to open an index that was unavailable");
-        }
-        let index = create_or_open_index(path, date, enable_mdb_writemap, map_size)?;
-        match self.available.insert(*uuid, index.clone()) {
-            InsertionOutcome::InsertedNew => (),
-            InsertionOutcome::Evicted(evicted_uuid, evicted_index) => {
-                self.close(evicted_uuid, evicted_index, enable_mdb_writemap, 0);
-            }
-            InsertionOutcome::Replaced(_) => {
-                panic!("Attempt to open an index that was already opened")
-            }
-        }
-        Ok(index)
-    }
-
-    /// Increases the current generation. See documentation for this field.
-    ///
-    /// In the unlikely event that the 2^64 generations would have been exhausted, we simply wrap-around.
-    ///
-    /// For this to cause an issue, one should be able to stop a reader in time after it got a `ReopenableIndex` and before it takes the lock
-    /// to remove it from the unavailable map, and keep the reader in this frozen state for 2^64 closing of other indexes.
-    ///
-    /// This seems overwhelmingly impossible to achieve in practice.
-    fn next_generation(&mut self) -> usize {
-        self.generation = self.generation.wrapping_add(1);
-        self.generation
-    }
-
-    /// Attempts to close an index.
-    ///
-    /// # Status table
-    ///
-    /// | Previous Status | New Status    |
-    /// |-----------------|---------------|
-    /// | Missing         | Missing       |
-    /// | BeingDeleted    | BeingDeleted  |
-    /// | Closing         | Closing       |
-    /// | Available       | Closing       |
-    ///
-    pub fn close_for_resize(
-        &mut self,
-        uuid: &Uuid,
-        enable_mdb_writemap: bool,
-        map_size_growth: usize,
-    ) {
-        let Some(index) = self.available.remove(uuid) else {
-            return;
-        };
-        self.close(*uuid, index, enable_mdb_writemap, map_size_growth);
-    }
-
-    fn close(
-        &mut self,
-        uuid: Uuid,
-        index: Index,
-        enable_mdb_writemap: bool,
-        map_size_growth: usize,
-    ) {
-        let map_size = index.map_size().unwrap_or(DEFAULT_MAP_SIZE) + map_size_growth;
-        let closing_event = index.prepare_for_closing();
-        let generation = self.next_generation();
-        self.unavailable.insert(
-            uuid,
-            Some(ClosingIndex { uuid, closing_event, enable_mdb_writemap, map_size, generation }),
-        );
-    }
-
-    /// Attempts to delete and index.
-    ///
-    ///  `end_deletion` must be called just after.
-    ///
-    /// # Status table
-    ///
-    /// | Previous Status | New Status   | Return value                |
-    /// |-----------------|--------------|-----------------------------|
-    /// | Missing         | BeingDeleted | Ok(None)                    |
-    /// | BeingDeleted    | BeingDeleted | Err(None)                   |
-    /// | Closing         | Closing      | Err(Some(reopen))           |
-    /// | Available       | BeingDeleted | Ok(Some(env_closing_event)) |
-    pub fn start_deletion(
-        &mut self,
-        uuid: &Uuid,
-    ) -> std::result::Result<Option<EnvClosingEvent>, Option<ClosingIndex>> {
-        if let Some(index) = self.available.remove(uuid) {
-            self.unavailable.insert(*uuid, None);
-            return Ok(Some(index.prepare_for_closing()));
-        }
-        match self.unavailable.remove(uuid) {
-            Some(Some(reopen)) => Err(Some(reopen)),
-            Some(None) => Err(None),
-            None => Ok(None),
-        }
-    }
-
-    /// Marks that an index deletion finished.
-    ///
-    /// Must be used after calling `start_deletion`.
-    ///
-    /// # Status table
-    ///
-    /// | Previous Status | New Status |
-    /// |-----------------|------------|
-    /// | Missing         | Missing    |
-    /// | BeingDeleted    | Missing    |
-    /// | Closing         | panics     |
-    /// | Available       | panics     |
-    pub fn end_deletion(&mut self, uuid: &Uuid) {
-        assert!(
-            self.available.get(uuid).is_none(),
-            "Attempt to finish deletion of an index that was not being deleted"
-        );
-        // Do not panic if the index was Missing or BeingDeleted
-        assert!(
-            !matches!(self.unavailable.remove(uuid), Some(Some(_))),
-            "Attempt to finish deletion of an index that was being closed"
-        );
-    }
-}
-
-/// Create or open an index in the specified path.
-/// The path *must* exist or an error will be thrown.
-fn create_or_open_index(
-    path: &Path,
-    date: Option<(OffsetDateTime, OffsetDateTime)>,
-    enable_mdb_writemap: bool,
-    map_size: usize,
-) -> Result<Index> {
-    let mut options = EnvOpenOptions::new();
-    options.map_size(clamp_to_page_size(map_size));
-    options.max_readers(1024);
-    if enable_mdb_writemap {
-        unsafe { options.flag(Flags::MdbWriteMap) };
-    }
-
-    if let Some((created, updated)) = date {
-        Ok(Index::new_with_creation_dates(options, path, created, updated)?)
-    } else {
-        Ok(Index::new(options, path)?)
-    }
-}
-
-/// Putting the tests of the LRU down there so we have access to the cache's private members
-#[cfg(test)]
-mod tests {
-
-    use meilisearch_types::heed::Env;
-    use meilisearch_types::Index;
-    use uuid::Uuid;
-
-    use super::super::IndexMapper;
-    use crate::tests::IndexSchedulerHandle;
-    use crate::utils::clamp_to_page_size;
-    use crate::IndexScheduler;
-
-    impl IndexMapper {
-        fn test() -> (Self, Env, IndexSchedulerHandle) {
-            let (index_scheduler, handle) = IndexScheduler::test(true, vec![]);
-            (index_scheduler.index_mapper, index_scheduler.env, handle)
-        }
-    }
-
-    fn check_first_unavailable(mapper: &IndexMapper, expected_uuid: Uuid, is_closing: bool) {
-        let index_map = mapper.index_map.read().unwrap();
-        let (uuid, state) = index_map.unavailable.first_key_value().unwrap();
-        assert_eq!(uuid, &expected_uuid);
-        assert_eq!(state.is_some(), is_closing);
-    }
-
-    #[test]
-    fn evict_indexes() {
-        let (mapper, env, _handle) = IndexMapper::test();
-        let mut uuids = vec![];
-        // LRU cap + 1
-        for i in 0..(5 + 1) {
-            let index_name = format!("index-{i}");
-            let wtxn = env.write_txn().unwrap();
-            mapper.create_index(wtxn, &index_name, None).unwrap();
-            let txn = env.read_txn().unwrap();
-            uuids.push(mapper.index_mapping.get(&txn, &index_name).unwrap().unwrap());
-        }
-        // index-0 was evicted
-        check_first_unavailable(&mapper, uuids[0], true);
-
-        // get back the evicted index
-        let wtxn = env.write_txn().unwrap();
-        mapper.create_index(wtxn, "index-0", None).unwrap();
-
-        // Least recently used is now index-1
-        check_first_unavailable(&mapper, uuids[1], true);
-    }
-
-    #[test]
-    fn resize_index() {
-        let (mapper, env, _handle) = IndexMapper::test();
-        let index = mapper.create_index(env.write_txn().unwrap(), "index", None).unwrap();
-        assert_index_size(index, mapper.index_base_map_size);
-
-        mapper.resize_index(&env.read_txn().unwrap(), "index").unwrap();
-
-        let index = mapper.create_index(env.write_txn().unwrap(), "index", None).unwrap();
-        assert_index_size(index, mapper.index_base_map_size + mapper.index_growth_amount);
-
-        mapper.resize_index(&env.read_txn().unwrap(), "index").unwrap();
-
-        let index = mapper.create_index(env.write_txn().unwrap(), "index", None).unwrap();
-        assert_index_size(index, mapper.index_base_map_size + mapper.index_growth_amount * 2);
-    }
-
-    fn assert_index_size(index: Index, expected: usize) {
-        let expected = clamp_to_page_size(expected);
-        let index_map_size = index.map_size().unwrap();
-        assert_eq!(index_map_size, expected);
-    }
-}

index-scheduler/src/index_mapper/mod.rs

@@ -1,477 +0,0 @@
-use std::path::PathBuf;
-use std::sync::{Arc, RwLock};
-use std::time::Duration;
-use std::{fs, thread};
-
-use log::error;
-use meilisearch_types::heed::types::{SerdeJson, Str};
-use meilisearch_types::heed::{Database, Env, RoTxn, RwTxn};
-use meilisearch_types::milli::update::IndexerConfig;
-use meilisearch_types::milli::{FieldDistribution, Index};
-use serde::{Deserialize, Serialize};
-use time::OffsetDateTime;
-use uuid::Uuid;
-
-use self::index_map::IndexMap;
-use self::IndexStatus::{Available, BeingDeleted, Closing, Missing};
-use crate::uuid_codec::UuidCodec;
-use crate::{Error, Result};
-
-mod index_map;
-
-const INDEX_MAPPING: &str = "index-mapping";
-const INDEX_STATS: &str = "index-stats";
-
-/// Structure managing meilisearch's indexes.
-///
-/// It is responsible for:
-/// 1. Creating new indexes
-/// 2. Opening indexes and storing references to these opened indexes
-/// 3. Accessing indexes through their uuid
-/// 4. Mapping a user-defined name to each index uuid.
-///
-/// # Implementation notes
-///
-/// An index exists as 3 bits of data:
-/// 1. The index data on disk, that can exist in 3 states: Missing, Present, or BeingDeleted.
-/// 2. The persistent database containing the association between the index' name and its UUID,
-///    that can exist in 2 states: Missing or Present.
-/// 3. The state of the index in the in-memory `IndexMap`, that can exist in multiple states:
-///   - Missing
-///   - Available
-///   - Closing (because an index needs resizing or was evicted from the cache)
-///   - BeingDeleted
-///
-/// All of this data should be kept consistent between index operations, which is achieved by the `IndexMapper`
-/// with the use of the following primitives:
-/// - A RwLock on the `IndexMap`.
-/// - Transactions on the association database.
-/// - ClosingEvent signals emitted when closing an environment.
-#[derive(Clone)]
-pub struct IndexMapper {
-    /// Keep track of the opened indexes. Used mainly by the index resolver.
-    index_map: Arc<RwLock<IndexMap>>,
-
-    /// Map an index name with an index uuid currently available on disk.
-    pub(crate) index_mapping: Database<Str, UuidCodec>,
-    /// Map an index UUID with the cached stats associated to the index.
-    ///
-    /// Using an UUID forces to use the index_mapping table to recover the index behind a name, ensuring
-    /// consistency wrt index swapping.
-    pub(crate) index_stats: Database<UuidCodec, SerdeJson<IndexStats>>,
-
-    /// Path to the folder where the LMDB environments of each index are.
-    base_path: PathBuf,
-    /// The map size an index is opened with on the first time.
-    index_base_map_size: usize,
-    /// The quantity by which the map size of an index is incremented upon reopening, in bytes.
-    index_growth_amount: usize,
-    /// Whether we open a meilisearch index with the MDB_WRITEMAP option or not.
-    enable_mdb_writemap: bool,
-    pub indexer_config: Arc<IndexerConfig>,
-}
-
-/// Whether the index is available for use or is forbidden to be inserted back in the index map
-#[allow(clippy::large_enum_variant)]
-#[derive(Clone)]
-pub enum IndexStatus {
-    /// Not currently in the index map.
-    Missing,
-    /// Do not insert it back in the index map as it is currently being deleted.
-    BeingDeleted,
-    /// Temporarily do not insert the index in the index map as it is currently being resized/evicted from the map.
-    Closing(index_map::ClosingIndex),
-    /// You can use the index without worrying about anything.
-    Available(Index),
-}
-
-/// The statistics that can be computed from an `Index` object.
-#[derive(Serialize, Deserialize, Debug)]
-pub struct IndexStats {
-    /// Number of documents in the index.
-    pub number_of_documents: u64,
-    /// Size taken up by the index' DB, in bytes.
-    ///
-    /// This includes the size taken by both the used and free pages of the DB, and as the free pages
-    /// are not returned to the disk after a deletion, this number is typically larger than
-    /// `used_database_size` that only includes the size of the used pages.
-    pub database_size: u64,
-    /// Size taken by the used pages of the index' DB, in bytes.
-    ///
-    /// As the DB backend does not return to the disk the pages that are not currently used by the DB,
-    /// this value is typically smaller than `database_size`.
-    pub used_database_size: u64,
-    /// Association of every field name with the number of times it occurs in the documents.
-    pub field_distribution: FieldDistribution,
-    /// Creation date of the index.
-    pub created_at: OffsetDateTime,
-    /// Date of the last update of the index.
-    pub updated_at: OffsetDateTime,
-}
-
-impl IndexStats {
-    /// Compute the stats of an index
-    ///
-    /// # Parameters
-    ///
-    /// - rtxn: a RO transaction for the index, obtained from `Index::read_txn()`.
-    pub fn new(index: &Index, rtxn: &RoTxn) -> Result<Self> {
-        Ok(IndexStats {
-            number_of_documents: index.number_of_documents(rtxn)?,
-            database_size: index.on_disk_size()?,
-            used_database_size: index.used_size()?,
-            field_distribution: index.field_distribution(rtxn)?,
-            created_at: index.created_at(rtxn)?,
-            updated_at: index.updated_at(rtxn)?,
-        })
-    }
-}
-
-impl IndexMapper {
-    pub fn new(
-        env: &Env,
-        base_path: PathBuf,
-        index_base_map_size: usize,
-        index_growth_amount: usize,
-        index_count: usize,
-        enable_mdb_writemap: bool,
-        indexer_config: IndexerConfig,
-    ) -> Result<Self> {
-        let mut wtxn = env.write_txn()?;
-        let index_mapping = env.create_database(&mut wtxn, Some(INDEX_MAPPING))?;
-        let index_stats = env.create_database(&mut wtxn, Some(INDEX_STATS))?;
-        wtxn.commit()?;
-
-        Ok(Self {
-            index_map: Arc::new(RwLock::new(IndexMap::new(index_count))),
-            index_mapping,
-            index_stats,
-            base_path,
-            index_base_map_size,
-            index_growth_amount,
-            enable_mdb_writemap,
-            indexer_config: Arc::new(indexer_config),
-        })
-    }
-
-    /// Get or create the index.
-    pub fn create_index(
-        &self,
-        mut wtxn: RwTxn,
-        name: &str,
-        date: Option<(OffsetDateTime, OffsetDateTime)>,
-    ) -> Result<Index> {
-        match self.index(&wtxn, name) {
-            Ok(index) => {
-                wtxn.commit()?;
-                Ok(index)
-            }
-            Err(Error::IndexNotFound(_)) => {
-                let uuid = Uuid::new_v4();
-                self.index_mapping.put(&mut wtxn, name, &uuid)?;
-
-                let index_path = self.base_path.join(uuid.to_string());
-                fs::create_dir_all(&index_path)?;
-
-                // Error if the UUIDv4 somehow already exists in the map, since it should be fresh.
-                // This is very unlikely to happen in practice.
-                // TODO: it would be better to lazily create the index. But we need an Index::open function for milli.
-                let index = self.index_map.write().unwrap().create(
-                    &uuid,
-                    &index_path,
-                    date,
-                    self.enable_mdb_writemap,
-                    self.index_base_map_size,
-                )?;
-
-                wtxn.commit()?;
-
-                Ok(index)
-            }
-            error => error,
-        }
-    }
-
-    /// Removes the index from the mapping table and the in-memory index map
-    /// but keeps the associated tasks.
-    pub fn delete_index(&self, mut wtxn: RwTxn, name: &str) -> Result<()> {
-        let uuid = self
-            .index_mapping
-            .get(&wtxn, name)?
-            .ok_or_else(|| Error::IndexNotFound(name.to_string()))?;
-
-        // Not an error if the index had no stats in cache.
-        self.index_stats.delete(&mut wtxn, &uuid)?;
-
-        // Once we retrieved the UUID of the index we remove it from the mapping table.
-        assert!(self.index_mapping.delete(&mut wtxn, name)?);
-
-        wtxn.commit()?;
-
-        let mut tries = 0;
-        // Attempts to remove the index from the in-memory index map in a loop.
-        //
-        // If the index is currently being closed, we will wait for it to be closed and retry getting it in a subsequent
-        // loop iteration.
-        //
-        // We make 100 attempts before giving up.
-        // This could happen in the following situations:
-        //
-        // 1. There is a bug preventing the index from being correctly closed, or us from detecting this.
-        // 2. A user of the index is keeping it open for more than 600 seconds. This could happen e.g. during a pathological search.
-        //    This can not be caused by indexation because deleting an index happens in the scheduler itself, so cannot be concurrent with indexation.
-        //
-        // In these situations, reporting the error through a panic is in order.
-        let closing_event = loop {
-            let mut lock = self.index_map.write().unwrap();
-            match lock.start_deletion(&uuid) {
-                Ok(env_closing) => break env_closing,
-                Err(Some(reopen)) => {
-                    // drop the lock here so that we don't synchronously wait for the index to close.
-                    drop(lock);
-                    tries += 1;
-                    if tries >= 100 {
-                        panic!("Too many attempts to close index {name} prior to deletion.")
-                    }
-                    let reopen = if let Some(reopen) = reopen.wait_timeout(Duration::from_secs(6)) {
-                        reopen
-                    } else {
-                        continue;
-                    };
-                    reopen.close(&mut self.index_map.write().unwrap());
-                    continue;
-                }
-                Err(None) => return Ok(()),
-            }
-        };
-
-        let index_map = self.index_map.clone();
-        let index_path = self.base_path.join(uuid.to_string());
-        let index_name = name.to_string();
-        thread::Builder::new()
-            .name(String::from("index_deleter"))
-            .spawn(move || {
-                // We first wait to be sure that the previously opened index is effectively closed.
-                // This can take a lot of time, this is why we do that in a separate thread.
-                if let Some(closing_event) = closing_event {
-                    closing_event.wait();
-                }
-
-                // Then we remove the content from disk.
-                if let Err(e) = fs::remove_dir_all(&index_path) {
-                    error!(
-                        "An error happened when deleting the index {} ({}): {}",
-                        index_name, uuid, e
-                    );
-                }
-
-                // Finally we remove the entry from the index map.
-                index_map.write().unwrap().end_deletion(&uuid);
-            })
-            .unwrap();
-
-        Ok(())
-    }
-
-    pub fn exists(&self, rtxn: &RoTxn, name: &str) -> Result<bool> {
-        Ok(self.index_mapping.get(rtxn, name)?.is_some())
-    }
-
-    /// Resizes the maximum size of the specified index to the double of its current maximum size.
-    ///
-    /// This operation involves closing the underlying environment and so can take a long time to complete.
-    ///
-    /// # Panics
-    ///
-    /// - If the Index corresponding to the passed name is concurrently being deleted/resized or cannot be found in the
-    ///   in memory hash map.
-    pub fn resize_index(&self, rtxn: &RoTxn, name: &str) -> Result<()> {
-        let uuid = self
-            .index_mapping
-            .get(rtxn, name)?
-            .ok_or_else(|| Error::IndexNotFound(name.to_string()))?;
-
-        // We remove the index from the in-memory index map.
-        self.index_map.write().unwrap().close_for_resize(
-            &uuid,
-            self.enable_mdb_writemap,
-            self.index_growth_amount,
-        );
-
-        Ok(())
-    }
-
-    /// Return an index, may open it if it wasn't already opened.
-    pub fn index(&self, rtxn: &RoTxn, name: &str) -> Result<Index> {
-        let uuid = self
-            .index_mapping
-            .get(rtxn, name)?
-            .ok_or_else(|| Error::IndexNotFound(name.to_string()))?;
-
-        let mut tries = 0;
-        // attempts to open the index in a loop.
-        //
-        // If the index is currently being closed, we will wait for it to be closed and retry getting it in a subsequent
-        // loop iteration.
-        //
-        // We make 100 attempts before giving up.
-        // This could happen in the following situations:
-        //
-        // 1. There is a bug preventing the index from being correctly closed, or us from detecting it was.
-        // 2. A user of the index is keeping it open for more than 600 seconds. This could happen e.g. during a long indexation,
-        //    a pathological search, and so on.
-        //
-        // In these situations, reporting the error through a panic is in order.
-        let index = loop {
-            tries += 1;
-            if tries > 100 {
-                panic!("Too many spurious wake ups while trying to open the index {name}");
-            }
-
-            // we get the index here to drop the lock before entering the match
-            let index = self.index_map.read().unwrap().get(&uuid);
-
-            match index {
-                Available(index) => break index,
-                Closing(reopen) => {
-                    // Avoiding deadlocks: no lock taken while doing this operation.
-                    let reopen = if let Some(reopen) = reopen.wait_timeout(Duration::from_secs(6)) {
-                        reopen
-                    } else {
-                        continue;
-                    };
-                    let index_path = self.base_path.join(uuid.to_string());
-                    // take the lock to reopen the environment.
-                    reopen.reopen(&mut self.index_map.write().unwrap(), &index_path)?;
-                    continue;
-                }
-                BeingDeleted => return Err(Error::IndexNotFound(name.to_string())),
-                // since we're lazy, it's possible that the index has not been opened yet.
-                Missing => {
-                    let mut index_map = self.index_map.write().unwrap();
-                    // between the read lock and the write lock it's not impossible
-                    // that someone already opened the index (eg if two searches happen
-                    // at the same time), thus before opening it we check a second time
-                    // if it's not already there.
-                    match index_map.get(&uuid) {
-                        Missing => {
-                            let index_path = self.base_path.join(uuid.to_string());
-
-                            break index_map.create(
-                                &uuid,
-                                &index_path,
-                                None,
-                                self.enable_mdb_writemap,
-                                self.index_base_map_size,
-                            )?;
-                        }
-                        Available(index) => break index,
-                        Closing(_) => {
-                            // the reopening will be handled in the next loop operation
-                            continue;
-                        }
-                        BeingDeleted => return Err(Error::IndexNotFound(name.to_string())),
-                    }
-                }
-            }
-        };
-
-        Ok(index)
-    }
-
-    /// Attempts `f` for each index that exists in the index mapper.
-    ///
-    /// It is preferable to use this function rather than a loop that opens all indexes, as a way to avoid having all indexes opened,
-    /// which is unsupported in general.
-    ///
-    /// Since `f` is allowed to return a result, and `Index` is cloneable, it is still possible to wrongly build e.g. a vector of
-    /// all the indexes, but this function makes it harder and so less likely to do accidentally.
-    pub fn try_for_each_index<U, V>(
-        &self,
-        rtxn: &RoTxn,
-        mut f: impl FnMut(&str, &Index) -> Result<U>,
-    ) -> Result<V>
-    where
-        V: FromIterator<U>,
-    {
-        self.index_mapping
-            .iter(rtxn)?
-            .map(|res| {
-                res.map_err(Error::from)
-                    .and_then(|(name, _)| self.index(rtxn, name).and_then(|index| f(name, &index)))
-            })
-            .collect()
-    }
-
-    /// Return the name of all indexes without opening them.
-    pub fn index_names(&self, rtxn: &RoTxn) -> Result<Vec<String>> {
-        self.index_mapping
-            .iter(rtxn)?
-            .map(|res| res.map_err(Error::from).map(|(name, _)| name.to_string()))
-            .collect()
-    }
-
-    /// Swap two index names.
-    pub fn swap(&self, wtxn: &mut RwTxn, lhs: &str, rhs: &str) -> Result<()> {
-        let lhs_uuid = self
-            .index_mapping
-            .get(wtxn, lhs)?
-            .ok_or_else(|| Error::IndexNotFound(lhs.to_string()))?;
-        let rhs_uuid = self
-            .index_mapping
-            .get(wtxn, rhs)?
-            .ok_or_else(|| Error::IndexNotFound(rhs.to_string()))?;
-
-        self.index_mapping.put(wtxn, lhs, &rhs_uuid)?;
-        self.index_mapping.put(wtxn, rhs, &lhs_uuid)?;
-
-        Ok(())
-    }
-
-    /// The stats of an index.
-    ///
-    /// If available in the cache, they are directly returned.
-    /// Otherwise, the `Index` is opened to compute the stats on the fly (the result is not cached).
-    /// The stats for an index are cached after each `Index` update.
-    pub fn stats_of(&self, rtxn: &RoTxn, index_uid: &str) -> Result<IndexStats> {
-        let uuid = self
-            .index_mapping
-            .get(rtxn, index_uid)?
-            .ok_or_else(|| Error::IndexNotFound(index_uid.to_string()))?;
-
-        match self.index_stats.get(rtxn, &uuid)? {
-            Some(stats) => Ok(stats),
-            None => {
-                let index = self.index(rtxn, index_uid)?;
-                let index_rtxn = index.read_txn()?;
-                IndexStats::new(&index, &index_rtxn)
-            }
-        }
-    }
-
-    /// Stores the new stats for an index.
-    ///
-    /// Expected usage is to compute the stats the index using `IndexStats::new`, the pass it to this function.
-    pub fn store_stats_of(
-        &self,
-        wtxn: &mut RwTxn,
-        index_uid: &str,
-        stats: &IndexStats,
-    ) -> Result<()> {
-        let uuid = self
-            .index_mapping
-            .get(wtxn, index_uid)?
-            .ok_or_else(|| Error::IndexNotFound(index_uid.to_string()))?;
-
-        self.index_stats.put(wtxn, &uuid, stats)?;
-        Ok(())
-    }
-
-    pub fn index_exists(&self, rtxn: &RoTxn, name: &str) -> Result<bool> {
-        Ok(self.index_mapping.get(rtxn, name)?.is_some())
-    }
-
-    pub fn indexer_config(&self) -> &IndexerConfig {
-        &self.indexer_config
-    }
-}

index-scheduler/src/insta_snapshot.rs

@@ -28,9 +28,6 @@ pub fn snapshot_index_scheduler(scheduler: &IndexScheduler) -> String {
         started_at,
         finished_at,
         index_mapper,
-        features: _,
-        max_number_of_tasks: _,
-        puffin_frame: _,
         wake_up: _,
         dumps_path: _,
         snapshots_path: _,
@@ -186,9 +183,6 @@ fn snapshot_details(d: &Details) -> String {
             provided_ids: received_document_ids,
             deleted_documents,
         } => format!("{{ received_document_ids: {received_document_ids}, deleted_documents: {deleted_documents:?} }}"),
-        Details::DocumentDeletionByFilter { original_filter, deleted_documents } => format!(
-           "{{ original_filter: {original_filter}, deleted_documents: {deleted_documents:?} }}"
-        ),
         Details::ClearAll { deleted_documents } => {
             format!("{{ deleted_documents: {deleted_documents:?} }}")
         },
@@ -260,16 +254,6 @@ pub fn snapshot_canceled_by(
     snap
 }
 pub fn snapshot_index_mapper(rtxn: &RoTxn, mapper: &IndexMapper) -> String {
-    let mut s = String::new();
-    let names = mapper.index_names(rtxn).unwrap();
-
-    for name in names {
-        let stats = mapper.stats_of(rtxn, &name).unwrap();
-        s.push_str(&format!(
-            "{name}: {{ number_of_documents: {}, field_distribution: {:?} }}\n",
-            stats.number_of_documents, stats.field_distribution
-        ));
-    }
-
-    s
+    let names = mapper.indexes(rtxn).unwrap().into_iter().map(|(n, _)| n).collect::<Vec<_>>();
+    format!("{names:?}")
 }

index-scheduler/src/lru.rs

@@ -1,203 +0,0 @@
-//! Thread-safe `Vec`-backend LRU cache using [`std::sync::atomic::AtomicU64`] for synchronization.
-
-use std::sync::atomic::{AtomicU64, Ordering};
-
-/// Thread-safe `Vec`-backend LRU cache
-#[derive(Debug)]
-pub struct Lru<T> {
-    data: Vec<(AtomicU64, T)>,
-    generation: AtomicU64,
-    cap: usize,
-}
-
-impl<T> Lru<T> {
-    /// Creates a new LRU cache with the specified capacity.
-    ///
-    /// The capacity is allocated up-front, and will never change through a [`Self::put`] operation.
-    ///
-    /// # Panics
-    ///
-    /// - If the capacity is 0.
-    /// - If the capacity exceeds `isize::MAX` bytes.
-    pub fn new(cap: usize) -> Self {
-        assert_ne!(cap, 0, "The capacity of a cache cannot be 0");
-        Self {
-            // Note: since the element of the vector contains an AtomicU64, it is definitely not zero-sized so cap will never be usize::MAX.
-            data: Vec::with_capacity(cap),
-            generation: AtomicU64::new(0),
-            cap,
-        }
-    }
-
-    /// The capacity of this LRU cache, that is the maximum number of elements it can hold before evicting elements from the cache.
-    ///
-    /// The cache will contain at most this number of elements at any given time.
-    pub fn capacity(&self) -> usize {
-        self.cap
-    }
-
-    fn next_generation(&self) -> u64 {
-        // Acquire so this "happens-before" any potential store to a data cell (with Release ordering)
-        let generation = self.generation.fetch_add(1, Ordering::Acquire);
-        generation + 1
-    }
-
-    fn next_generation_mut(&mut self) -> u64 {
-        let generation = self.generation.get_mut();
-        *generation += 1;
-        *generation
-    }
-
-    /// Add a value in the cache, evicting an older value if necessary.
-    ///
-    /// If a value was evicted from the cache, it is returned.
-    ///
-    /// # Complexity
-    ///
-    /// - If the cache is full, then linear in the capacity.
-    /// - Otherwise constant.
-    pub fn put(&mut self, value: T) -> Option<T> {
-        // no need for a memory fence: we assume that whichever mechanism provides us synchronization
-        // (very probably, a RwLock) takes care of fencing for us.
-
-        let next_generation = self.next_generation_mut();
-        let evicted = if self.is_full() { self.pop() } else { None };
-        self.data.push((AtomicU64::new(next_generation), value));
-        evicted
-    }
-
-    /// Evict the oldest value from the cache.
-    ///
-    /// If the cache is empty, `None` will be returned.
-    ///
-    /// # Complexity
-    ///
-    /// - Linear in the capacity of the cache.
-    pub fn pop(&mut self) -> Option<T> {
-        // Don't use `Iterator::min_by_key` that provides shared references to its elements,
-        // so that we can get an exclusive one.
-        // This allows to handles the `AtomicU64`s as normal integers without using atomic instructions.
-        let mut min_generation_index = None;
-        for (index, (generation, _)) in self.data.iter_mut().enumerate() {
-            let generation = *generation.get_mut();
-            if let Some((_, min_generation)) = min_generation_index {
-                if min_generation > generation {
-                    min_generation_index = Some((index, generation));
-                }
-            } else {
-                min_generation_index = Some((index, generation))
-            }
-        }
-        min_generation_index.map(|(min_index, _)| self.data.swap_remove(min_index).1)
-    }
-
-    /// The current number of elements in the cache.
-    ///
-    /// This value is guaranteed to be less than or equal to [`Self::capacity`].
-    pub fn len(&self) -> usize {
-        self.data.len()
-    }
-
-    /// Returns `true` if putting any additional element in the cache would cause the eviction of an element.
-    pub fn is_full(&self) -> bool {
-        self.len() == self.capacity()
-    }
-}
-
-pub struct LruMap<K, V>(Lru<(K, V)>);
-
-impl<K, V> LruMap<K, V>
-where
-    K: Eq,
-{
-    /// Creates a new LRU cache map with the specified capacity.
-    ///
-    /// The capacity is allocated up-front, and will never change through a [`Self::insert`] operation.
-    ///
-    /// # Panics
-    ///
-    /// - If the capacity is 0.
-    /// - If the capacity exceeds `isize::MAX` bytes.
-    pub fn new(cap: usize) -> Self {
-        Self(Lru::new(cap))
-    }
-
-    /// Gets a value in the cache map by its key.
-    ///
-    /// If no value matches, `None` will be returned.
-    ///
-    /// # Complexity
-    ///
-    /// - Linear in the capacity of the cache.
-    pub fn get(&self, key: &K) -> Option<&V> {
-        for (generation, (candidate, value)) in self.0.data.iter() {
-            if key == candidate {
-                generation.store(self.0.next_generation(), Ordering::Release);
-                return Some(value);
-            }
-        }
-        None
-    }
-
-    /// Gets a value in the cache map by its key.
-    ///
-    /// If no value matches, `None` will be returned.
-    ///
-    /// # Complexity
-    ///
-    /// - Linear in the capacity of the cache.
-    pub fn get_mut(&mut self, key: &K) -> Option<&mut V> {
-        let next_generation = self.0.next_generation_mut();
-        for (generation, (candidate, value)) in self.0.data.iter_mut() {
-            if key == candidate {
-                *generation.get_mut() = next_generation;
-                return Some(value);
-            }
-        }
-        None
-    }
-
-    /// Inserts a value in the cache map by its key, replacing any existing value and returning any evicted value.
-    ///
-    /// # Complexity
-    ///
-    /// - Linear in the capacity of the cache.
-    pub fn insert(&mut self, key: K, mut value: V) -> InsertionOutcome<K, V> {
-        match self.get_mut(&key) {
-            Some(old_value) => {
-                std::mem::swap(old_value, &mut value);
-                InsertionOutcome::Replaced(value)
-            }
-            None => match self.0.put((key, value)) {
-                Some((key, value)) => InsertionOutcome::Evicted(key, value),
-                None => InsertionOutcome::InsertedNew,
-            },
-        }
-    }
-
-    /// Removes an element from the cache map by its key, returning its value.
-    ///
-    /// Returns `None` if there was no element with this key in the cache.
-    ///
-    /// # Complexity
-    ///
-    /// - Linear in the capacity of the cache.
-    pub fn remove(&mut self, key: &K) -> Option<V> {
-        for (index, (_, (candidate, _))) in self.0.data.iter_mut().enumerate() {
-            if key == candidate {
-                return Some(self.0.data.swap_remove(index).1 .1);
-            }
-        }
-        None
-    }
-}
-
-/// The result of an insertion in a LRU map.
-pub enum InsertionOutcome<K, V> {
-    /// The key was not in the cache, the key-value pair has been inserted.
-    InsertedNew,
-    /// The key was not in the cache and an old key-value pair was evicted from the cache to make room for its insertions.
-    Evicted(K, V),
-    /// The key was already in the cache map, its value has been updated.
-    Replaced(V),
-}

index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/cancel_processed.snap

@@ -1,5 +1,6 @@
 ---
 source: index-scheduler/src/lib.rs
+assertion_line: 1755
 ---
 ### Autobatching Enabled = true
 ### Processing Tasks:
@@ -22,7 +23,7 @@ canceled [0,]
 catto [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-
+[]
 ----------------------------------------------------------------------
 ### Canceled By:
 1 [0,]

index-scheduler/src/snapshots/lib.rs/cancel_enqueued_task/initial_tasks_enqueued.snap

@@ -20,7 +20,7 @@ enqueued [0,1,]
 catto [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-
+[]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/aborted_indexation.snap

@@ -25,9 +25,7 @@ catto [0,]
 wolfo [2,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-beavero: { number_of_documents: 0, field_distribution: {} }
-catto: { number_of_documents: 1, field_distribution: {"id": 1} }
-
+["beavero", "catto"]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/cancel_processed.snap

@@ -1,5 +1,6 @@
 ---
 source: index-scheduler/src/lib.rs
+assertion_line: 1859
 ---
 ### Autobatching Enabled = true
 ### Processing Tasks:
@@ -26,9 +27,7 @@ catto [0,]
 wolfo [2,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-beavero: { number_of_documents: 0, field_distribution: {} }
-catto: { number_of_documents: 1, field_distribution: {"id": 1} }
-
+["beavero", "catto"]
 ----------------------------------------------------------------------
 ### Canceled By:
 3 [1,2,]

index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/first_task_processed.snap

@@ -23,8 +23,7 @@ catto [0,]
 wolfo [2,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-catto: { number_of_documents: 1, field_distribution: {"id": 1} }
-
+["catto"]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/cancel_mix_of_tasks/processing_second_task_cancel_enqueued.snap

@@ -25,8 +25,7 @@ catto [0,]
 wolfo [2,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-catto: { number_of_documents: 1, field_distribution: {"id": 1} }
-
+["catto"]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/cancel_processing_task/aborted_indexation.snap

@@ -20,8 +20,7 @@ enqueued [0,1,]
 catto [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-catto: { number_of_documents: 0, field_distribution: {} }
-
+["catto"]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_processed.snap

@@ -1,5 +1,6 @@
 ---
 source: index-scheduler/src/lib.rs
+assertion_line: 1818
 ---
 ### Autobatching Enabled = true
 ### Processing Tasks:
@@ -22,8 +23,7 @@ canceled [0,]
 catto [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-catto: { number_of_documents: 0, field_distribution: {} }
-
+["catto"]
 ----------------------------------------------------------------------
 ### Canceled By:
 1 [0,]

index-scheduler/src/snapshots/lib.rs/cancel_processing_task/cancel_task_registered.snap

@@ -20,7 +20,7 @@ enqueued [0,1,]
 catto [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-
+[]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/cancel_processing_task/initial_task_processing.snap

@@ -18,7 +18,7 @@ enqueued [0,]
 catto [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-
+[]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/cancel_processing_task/registered_the_first_task.snap

@@ -18,7 +18,7 @@ enqueued [0,]
 catto [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-
+[]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/cancel_processed.snap

@@ -21,8 +21,7 @@ succeeded [0,1,]
 catto [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-catto: { number_of_documents: 1, field_distribution: {"id": 1} }
-
+["catto"]
 ----------------------------------------------------------------------
 ### Canceled By:
 1 []

index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/initial_task_processed.snap

@@ -19,8 +19,7 @@ succeeded [0,]
 catto [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-catto: { number_of_documents: 1, field_distribution: {"id": 1} }
-
+["catto"]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/cancel_succeeded_task/registered_the_first_task.snap

@@ -18,7 +18,7 @@ enqueued [0,]
 catto [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-
+[]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/do_not_batch_task_of_different_indexes/all_tasks_processed.snap

@@ -27,10 +27,7 @@ doggos [0,3,]
 girafos [2,5,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-cattos: { number_of_documents: 0, field_distribution: {} }
-doggos: { number_of_documents: 0, field_distribution: {} }
-girafos: { number_of_documents: 0, field_distribution: {} }
-
+["cattos", "doggos", "girafos"]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/document_addition/after_register.snap

@@ -18,7 +18,7 @@ enqueued [0,]
 doggos [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-
+[]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/document_addition/after_the_batch_creation.snap

@@ -18,7 +18,7 @@ enqueued [0,]
 doggos [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-
+[]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/document_addition/once_everything_is_processed.snap

@@ -19,8 +19,7 @@ succeeded [0,]
 doggos [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
-
+["doggos"]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/after_processing_the_batch.snap

@@ -21,8 +21,7 @@ succeeded [0,1,]
 doggos [0,1,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-doggos: { number_of_documents: 1, field_distribution: {"doggo": 1, "id": 1} }
-
+["doggos"]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_first_task.snap

@@ -18,7 +18,7 @@ enqueued [0,]
 doggos [0,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-
+[]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/document_addition_and_document_deletion/registered_the_second_task.snap

@@ -20,7 +20,7 @@ enqueued [0,1,]
 doggos [0,1,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-
+[]
 ----------------------------------------------------------------------
 ### Canceled By:
 

index-scheduler/src/snapshots/lib.rs/document_addition_and_index_deletion/before_index_creation.snap

@@ -23,8 +23,7 @@ succeeded [0,]
 doggos [0,1,2,]
 ----------------------------------------------------------------------
 ### Index Mapper:
-doggos: { number_of_documents: 0, field_distribution: {} }
-
+["doggos"]
 ----------------------------------------------------------------------
 ### Canceled By: