Merge #2633

2633: Fix highlight issue by updating milli to v0.31.2 r=ManyTheFish a=curquiza

Fixes https://github.com/meilisearch/meilisearch/issues/2627

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>

.github/ISSUE_TEMPLATE/bug_report.md

@@ -23,8 +23,8 @@ A clear and concise description of what you expected to happen.
 **Screenshots**
 If applicable, add screenshots to help explain your problem.
 
-**MeiliSearch version:** [e.g. v0.20.0]
+**Meilisearch version:** [e.g. v0.20.0]
 
 **Additional context**
 Additional information that may be relevant to the issue.
-[e.g. architecture, device, OS, browser]
+[e.g. architecture, device, OS, browser]

.github/ISSUE_TEMPLATE/config.yml

@@ -1,10 +1,13 @@
 contact_links:
-  - name: Feature request
+  - name: Language support request & feedback
+    url: https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal?discussions_q=label%3Aproduct%3Acore%3Atokenizer+category%3A%22Feedback+%26+Feature+Proposal%22
+    about:  The requests and feedback regarding Language support are not managed in this repository. Please upvote the related discussion in our dedicated product repository or open a new one if it doesn't exist.
+  - name: Feature request & feedback
     url: https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal
-    about: The feature requests are not managed in this repository, please open a discussion in our dedicated product repository
+    about: The feature requests and feedback regarding the already existing features are not managed in this repository. Please open a discussion in our dedicated product repository
   - name: Documentation issue
     url: https://github.com/meilisearch/documentation/issues/new
     about: For documentation issues, open an issue or a PR in the documentation repository
   - name: Support questions & other
-    url: https://github.com/meilisearch/MeiliSearch/discussions/new
+    url: https://github.com/meilisearch/meilisearch/discussions/new
     about: For any other question, open a discussion in this repository

.github/dependabot.yml

@@ -0,0 +1,13 @@
+# Set update schedule for GitHub Actions only
+
+version: 2
+updates:
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "monthly"
+    labels:
+      - 'skip changelog'
+      - 'dependencies'
+    rebase-strategy: disabled

.github/release-draft-template.yml

@@ -1,13 +0,0 @@
-name-template: 'v$RESOLVED_VERSION'
-tag-template: 'v$RESOLVED_VERSION'
-version-template: '0.21.0-alpha.$PATCH'
-exclude-labels:
-  - 'skip-changelog'
-template: |
-  ## Changes
-
-  $CHANGES
-no-changes-template: 'Changes are coming soon 😎'
-sort-direction: 'ascending'
-version-resolver:
-  default: patch

.github/scripts/check-release.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+# check_tag $current_tag $file_tag $file_name
+function check_tag {
+  if [[ "$1" != "$2" ]]; then
+      echo "Error: the current tag does not match the version in $3: found $2 - expected $1"
+      ret=1
+  fi
+}
+
+ret=0
+current_tag=${GITHUB_REF#'refs/tags/v'}
+
+toml_files='*/Cargo.toml'
+for toml_file in $toml_files;
+do
+    file_tag="$(grep '^version = ' $toml_file | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')"
+    check_tag $current_tag $file_tag $toml_file
+done
+
+lock_file='Cargo.lock'
+lock_tag=$(grep -A 1 'name = "meilisearch-auth"' $lock_file | grep version | cut -d '=' -f 2 | tr -d '"' | tr -d ' ')
+check_tag $current_tag $lock_tag $lock_file
+
+if [[ "$ret" -eq 0 ]] ; then
+  echo 'OK'
+fi
+exit $ret

.github/scripts/is-latest-release.sh

@@ -1,14 +1,14 @@
 #!/bin/sh
 
-# Checks if the current tag should be the latest (in terms of semver and not of release date).
-# Ex: previous tag -> v0.10.1
-#     new tag -> v0.8.12
-#     The new tag should not be the latest
-#     So it returns "false", the CI should not run for the release v0.8.2
-
-# Used in GHA in publish-docker-latest.yml
+# Was used in our CIs to publish the latest docker image. Not used anymore, will be used again when v1 and v2 will be out and we will want to maintain multiple stable versions.
 # Returns "true" or "false" (as a string) to be used in the `if` in GHA
 
+# Checks if the current tag should be the latest (in terms of semver and not of release date).
+# Ex: previous tag -> v2.1.1
+#     new tag -> v1.20.3
+#     The new tag (v1.20.3) should NOT be the latest
+#     So it returns "false", the `latest` tag should not be updated for the release v1.20.3 and still need to correspond to v2.1.1
+
 # GLOBAL
 GREP_SEMVER_REGEXP='v\([0-9]*\)[.]\([0-9]*\)[.]\([0-9]*\)$' # i.e. v[number].[number].[number]
 
@@ -74,7 +74,7 @@ semverLT() {
 # Returns the tag of the latest stable release (in terms of semver and not of release date)
 get_latest() {
     temp_file='temp_file' # temp_file needed because the grep would start before the download is over
-    curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file"
+    curl -s 'https://api.github.com/repos/meilisearch/meilisearch/releases' > "$temp_file"
     releases=$(cat "$temp_file" | \
         grep -E "tag_name|draft|prerelease" \
         | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')

.github/workflows/README.md

@@ -1,20 +0,0 @@
-# GitHub Actions Workflow for MeiliSearch
-
-> **Note:**
-
-> - We do not use [cache](https://github.com/actions/cache) yet but we could use it to speed up CI
-
-## Workflow
-
-- On each pull request, we trigger `cargo test`.
-- On each tag, we build:
-    - the tagged Docker image and publish it to Docker Hub
-    - the binaries for MacOS, Ubuntu, and Windows
-    - the Debian package
-- On each stable release (`v*.*.*` tag):
-    - we build the `latest` Docker image and publish it to Docker Hub
-    - we publish the binary to Hombrew and Gemfury
-
-## Problems
-
-- We do not test on Windows because we are unable to make it work, there is a disk space problem.

.github/workflows/coverage.yml

@@ -1,7 +1,6 @@
 ---
 on:
-  pull_request:
-    types: [review_requested, ready_for_review]
+  workflow_dispatch:
 
 name: Execute code coverage
 
@@ -9,7 +8,7 @@ jobs:
   nightly-coverage:
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly
@@ -26,7 +25,7 @@ jobs:
           RUSTFLAGS: "-Zprofile -Ccodegen-units=1 -Cinline-threshold=0 -Clink-dead-code -Coverflow-checks=off -Cpanic=unwind -Zpanic_abort_tests"
       - uses: actions-rs/grcov@v0.1
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v1
+        uses: codecov/codecov-action@v3
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           file: ${{ steps.coverage.outputs.report }}

.github/workflows/create-issue-dependencies.yml

@@ -0,0 +1,23 @@
+name: Create issue to upgrade dependencies
+on:
+  schedule:
+    - cron: '0 0 1 */3 *'
+  workflow_dispatch:
+  
+jobs:
+  create-issue:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - name: Create an issue
+      uses: actions-ecosystem/action-create-issue@v1
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        title: Upgrade dependencies
+        body: |
+          We need to update the dependencies of the Meilisearch repository, and, if possible, the dependencies of all the core-team repositories that Meilisearch depends on (milli, charabia, heed...).
+
+          ⚠️ This issue should only be done at the beginning of the sprint!     
+        labels: |
+          dependencies
+          maintenance

.github/workflows/flaky.yml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-18.04
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Install cargo-flaky
       run: cargo install cargo-flaky
     - name: Run cargo flaky 100 times

.github/workflows/publish-binaries.yml

@@ -5,10 +5,35 @@ on:
 name: Publish binaries to release
 
 jobs:
+  check-version:
+    name: Check the version validity
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      # Check if the tag has the v<nmumber>.<number>.<number> format.
+      # If yes, it means we are publishing an official release.
+      # If no, we are releasing a RC, so no need to check the version.
+      - name: Check tag format
+        if: github.event_name != 'schedule'
+        id: check-tag-format
+        run: |
+          escaped_tag=$(printf "%q" ${{ github.ref_name }})
+
+          if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo ::set-output name=stable::true
+          else
+            echo ::set-output name=stable::false
+          fi
+      - name: Check release validity
+        if: steps.check-tag-format.outputs.stable == 'true'
+        run: bash .github/scripts/check-release.sh
+
   publish:
-    name: Publish for ${{ matrix.os }}
+    name: Publish binary for ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
+    needs: check-version
     strategy:
+      fail-fast: false
       matrix:
         os: [ubuntu-18.04, macos-latest, windows-latest]
         include:
@@ -26,7 +51,7 @@ jobs:
     - uses: hecrj/setup-rust-action@master
       with:
         rust-version: stable
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Build
       run: cargo build --release --locked
     - name: Upload binaries to release
@@ -37,26 +62,70 @@ jobs:
         asset_name: ${{ matrix.asset_name }}
         tag: ${{ github.ref }}
 
-  publish-armv8:
-    name: Publish for ARMv8
-    runs-on: ubuntu-18.04
+  publish-aarch64:
+    name: Publish binary for aarch64
+    runs-on: ${{ matrix.os }}
+    needs: check-version
+    continue-on-error: false
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - build: aarch64
+            os: ubuntu-18.04
+            target: aarch64-unknown-linux-gnu
+            linker: gcc-aarch64-linux-gnu
+            use-cross: true
+            asset_name: meilisearch-linux-aarch64
+
     steps:
-      - uses: actions/checkout@v2
-      - uses: uraimo/run-on-arch-action@v1.0.7
-        id: runcmd
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Installing Rust toolchain
+        uses: actions-rs/toolchain@v1
         with:
-          architecture: aarch64 # aka ARMv8
-          distribution: ubuntu18.04
-          run: |
-            apt update
-            apt install -y curl gcc make
-            curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal --default-toolchain stable
-            source $HOME/.cargo/env
-            cargo build --release --locked
+          toolchain: stable
+          profile: minimal
+          target: ${{ matrix.target }}
+          override: true
+
+      - name: APT update
+        run: |
+          sudo apt update
+
+      - name: Install target specific tools
+        if: matrix.use-cross
+        run: |
+          sudo apt-get install -y ${{ matrix.linker }}
+
+      - name: Configure target aarch64 GNU
+        if: matrix.target == 'aarch64-unknown-linux-gnu'
+        ## Environment variable is not passed using env:
+        ## LD gold won't work with MUSL
+        # env:
+        #   JEMALLOC_SYS_WITH_LG_PAGE: 16
+        #   RUSTFLAGS: '-Clink-arg=-fuse-ld=gold'
+        run: |
+          echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
+          echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
+          echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
+          echo RUSTFLAGS="-Clink-arg=-fuse-ld=gold" >> $GITHUB_ENV
+
+      - name: Cargo build
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          use-cross: ${{ matrix.use-cross }}
+          args: --release --target ${{ matrix.target }}
+
+      - name: List target output files
+        run: ls -lR ./target
+
       - name: Upload the binary to release
         uses: svenstaro/upload-release-action@v1-release
         with:
           repo_token: ${{ secrets.PUBLISH_TOKEN }}
-          file: target/release/meilisearch
-          asset_name: meilisearch-linux-armv8
+          file: target/${{ matrix.target }}/release/meilisearch
+          asset_name: ${{ matrix.asset_name }}
           tag: ${{ github.ref }}

.github/workflows/publish-deb-brew-pkg.yml

@@ -5,16 +5,25 @@ on:
     types: [released]
 
 jobs:
+  check-version:
+    name: Check the version validity
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check release validity
+        run: bash .github/scripts/check-release.sh
+
   debian:
     name: Publish debian packagge
     runs-on: ubuntu-18.04
+    needs: check-version
     steps:
     - uses: hecrj/setup-rust-action@master
       with:
         rust-version: stable
     - name: Install cargo-deb
       run: cargo install cargo-deb
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Build deb package
       run: cargo deb -p meilisearch-http -o target/debian/meilisearch.deb
     - name: Upload debian pkg to release
@@ -30,6 +39,7 @@ jobs:
   homebrew:
     name: Bump Homebrew formula
     runs-on: ubuntu-18.04
+    needs: check-version
     steps:
       - name: Create PR to Homebrew
         uses: mislav/bump-homebrew-formula-action@v1

.github/workflows/publish-docker-images.yml

@@ -0,0 +1,71 @@
+---
+on:
+  schedule:
+    - cron: '0 4 * * *' # Every day at 4:00am
+  push:
+    tags:
+      - '*'
+
+name: Publish tagged images to Docker Hub
+
+jobs:
+  docker:
+    runs-on: docker
+    steps:
+      - uses: actions/checkout@v2
+
+      # Check if the tag has the v<nmumber>.<number>.<number> format. If yes, it means we are publishing an official release.
+      # In this situation, we need to set `output.stable` to create/update the following tags (additionally to the `vX.Y.Z` Docker tag):
+      # - a `vX.Y` (without patch version) Docker tag
+      # - a `latest` Docker tag
+      - name: Check tag format
+        if: github.event_name != 'schedule'
+        id: check-tag-format
+        run: |
+          escaped_tag=$(printf "%q" ${{ github.ref_name }})
+
+          if [[ $escaped_tag =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo ::set-output name=stable::true
+          else
+            echo ::set-output name=stable::false
+          fi
+
+      # Check only the validity of the tag for official releases (not for pre-releases or other tags)
+      - name: Check release validity
+        if: github.event_name != 'schedule' && steps.check-tag-format.outputs.stable == 'true'
+        run: bash .github/scripts/check-release.sh
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        if: github.event_name != 'schedule'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Docker meta
+        id: meta
+        uses: docker/metadata-action@v4
+        with:
+          images: getmeili/meilisearch
+          # The lastest and `vX.Y` tags are only pushed for the official Meilisearch releases
+          # See https://github.com/docker/metadata-action#latest-tag
+          flavor: latest=false
+          tags: |
+            type=ref,event=tag
+            type=semver,pattern=v{{major}}.{{minor}},enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
+            type=raw,value=latest,enable=${{ steps.check-tag-format.outputs.stable == 'true' }}
+
+      - name: Build and push
+        id: docker_build
+        uses: docker/build-push-action@v3
+        with:
+          # We do not push tags for the cron jobs, this is only for test purposes
+          push: ${{ github.event_name != 'schedule' }}
+          platforms: linux/amd64,linux/arm64
+          tags: ${{ steps.meta.outputs.tags }}

.github/workflows/publish-docker-latest.yml

@@ -1,22 +0,0 @@
----
-on:
-  release:
-    types: [released]
-
-name: Publish latest image to Docker Hub
-
-jobs:
-  build:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v2
-      - name: Check if current release is latest
-        run: echo "##[set-output name=is_latest;]$(sh .github/is-latest-release.sh)"
-        id: release
-      - name: Publish to Registry
-        if: steps.release.outputs.is_latest == 'true'
-        uses: elgohr/Publish-Docker-Github-Action@master
-        with:
-          name: getmeili/meilisearch
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/publish-docker-tag.yml

@@ -1,22 +0,0 @@
----
-on:
-  push:
-    tags:
-      - '*'
-
-name: Publish tagged image to Docker Hub
-
-jobs:
-  build:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v2
-      - name: Publish to Registry
-        uses: elgohr/Publish-Docker-Github-Action@master
-        env:
-          COMMIT_SHA: ${{ github.sha }}
-        with:
-          name: getmeili/meilisearch
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-          tag_names: true

.github/workflows/release-drafter.yml

@@ -1,16 +0,0 @@
-name: Release Drafter
-
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  update_release_draft:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: release-drafter/release-drafter@v5
-        with:
-          config-name: release-draft-template.yml
-        env:
-          GITHUB_TOKEN: ${{ secrets.RELEASE_DRAFTER_TOKEN }}

.github/workflows/rust.yml

@@ -11,6 +11,8 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
+  RUST_BACKTRACE: 1
+  RUSTFLAGS: "-D warnings"
 
 jobs:
   tests:
@@ -21,14 +23,9 @@ jobs:
       matrix:
         os: [ubuntu-18.04, macos-latest, windows-latest]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Cache dependencies
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/.cargo
-          ./target
-        key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
+      uses: Swatinem/rust-cache@v1.4.0
     - name: Run cargo check without any default features
       uses: actions-rs/cargo@v1
       with:
@@ -40,24 +37,38 @@ jobs:
         command: test
         args: --locked --release
 
+  # We run tests in debug also, to make sure that the debug_assertions are hit
+  test-debug:
+    name: Run tests in debug
+    runs-on: ubuntu-18.04
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v1.4.0
+      - name: Run tests in debug
+        uses: actions-rs/cargo@v1
+        with:
+          command: test
+          args: --locked
+
   clippy:
     name: Run Clippy
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
-      - name: Cache dependencies
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo
-            ./target
-          key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
+      - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
           toolchain: stable
           override: true
           components: clippy
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v1.4.0
       - name: Run cargo clippy
         uses: actions-rs/cargo@v1
         with:
@@ -68,19 +79,14 @@ jobs:
     name: Run Rustfmt
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
-      - name: Cache dependencies
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo
-            ./target
-          key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
+      - uses: actions/checkout@v3
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
-          toolchain: nightly
+          toolchain: stable
           override: true
           components: rustfmt
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v1.4.0
       - name: Run cargo fmt
         run: cargo fmt --all -- --check

CONTRIBUTING.md

@@ -1,112 +1,110 @@
 # Contributing
 
-First, thank you for contributing to MeiliSearch! The goal of this document is to
-provide everything you need to start contributing to MeiliSearch. The
-following TOC is sorted progressively, starting with the basics and
-expanding into more specifics.
+First, thank you for contributing to Meilisearch! The goal of this document is to provide everything you need to start contributing to Meilisearch.
 
-<!-- MarkdownTOC autolink="true" style="ordered" indent="   " -->
+Remember that there are many ways to contribute other than writing code: writing [tutorials or blog posts](https://github.com/meilisearch/awesome-meilisearch), improving [the documentation](https://github.com/meilisearch/documentation), submitting [bug reports](https://github.com/meilisearch/meilisearch/issues/new?assignees=&labels=&template=bug_report.md&title=) and [feature requests](https://github.com/meilisearch/product/discussions/categories/feedback-feature-proposal)...
 
-1. [Assumptions](#assumptions)
-1. [Your First Contribution](#your-first-contribution)
-1. [Change Control](#change-control)
-   1. [Git Branches](#git-branches)
-   1. [Git Commits](#git-commits)
-      1. [Style](#style)
-   1. [Github Pull Requests](#github-pull-requests)
-      1. [Reviews & Approvals](#reviews--approvals)
-      1. [Merge Style](#merge-style)
-   1. [CI](#ci)
-1. [Development](#development)
-   1. [Setup](#setup)
-   1. [Testing](#testing)
-   1. [Benchmarking](#benchmarking--profiling)
-1. [Humans](#humans)
-   1. [Documentation](#documentation)
-   1. [Changelog](#changelog)
+## Table of Contents
 
-<!-- /MarkdownTOC -->
+- [Assumptions](#assumptions)
+- [How to Contribute](#how-to-contribute)
+- [Development Workflow](#development-workflow)
+- [Git Guidelines](#git-guidelines)
+- [Release Process (for internal team only)](#release-process-for-internal-team-only)
 
 ## Assumptions
 
-1. **You're familiar with [Github](https://github.com) and the [pull request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)
-   workflow.**
-2. **You've read the MeiliSearch [docs](https://docs.meilisearch.com).**
-3. **You know about the [MeiliSearch community](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html).
+1. **You're familiar with [GitHub](https://github.com) and the [Pull Requests](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)(PR) workflow.**
+2. **You've read the Meilisearch [documentation](https://docs.meilisearch.com).**
+3. **You know about the [Meilisearch community](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html).
    Please use this for help.**
 
-## Your First Contribution
+## How to Contribute
 
 1. Ensure your change has an issue! Find an
    [existing issue](https://github.com/meilisearch/meilisearch/issues/) or [open a new issue](https://github.com/meilisearch/meilisearch/issues/new).
    * This is where you can get a feel if the change will be accepted or not.
-2. Once approved, [fork the MeiliSearch repository](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) in your own
-   Github account.
+2. Once approved, [fork the Meilisearch repository](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) in your own GitHub account.
 3. [Create a new Git branch](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-and-deleting-branches-within-your-repository)
-4. Review the MeiliSearch [workflow](#workflow) and [development](#development).
-5. Make your changes.
-6. [Submit the branch as a pull request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) to the main MeiliSearch
-   repo. A MeiliSearch team member should comment and/or review your pull request
-   with a few days. Although, depending on the circumstances, it may take
-   longer.
+4. Review the [Development Workflow](#development-workflow) section that describes the steps to maintain the repository.
+5. Make your changes on your branch.
+6. [Submit the branch as a Pull Request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) pointing to the `main` branch of the Meilisearch repository. A maintainer should comment and/or review your Pull Request within a few days. Although depending on the circumstances, it may take longer.
 
-## Change Control
+## Development Workflow
+
+### Setup and run Meilisearch
+
+```bash
+cargo run --release
+```
+
+We recommend using the `--release` flag to test the full performance of Meilisearch.
+
+### Test
+
+```bash
+cargo test
+```
+
+This command will be triggered to each PR as a requirement for merging it.
+
+If you get a "Too many open files" error you might want to increase the open file limit using this command:
+
+```bash
+ulimit -Sn 3000
+```
+
+## Git Guidelines
 
 ### Git Branches
 
-_All_ changes must be made in a branch and submitted as [pull requests](#pull-requests).
-MeiliSearch does not adopt any type of branch naming style, but please use something
-descriptive of your changes.
+All changes must be made in a branch and submitted as PR.
+
+We do not enforce any branch naming style, but please use something descriptive of your changes.
 
 ### Git Commits
 
-#### Style
+As minimal requirements, your commit message should:
+- be capitalized
+- not finish by a dot or any other punctuation character (!,?)
+- start with a verb so that we can read your commit message this way: "This commit will ...", where "..." is the commit message.
+  e.g.: "Fix the home page button" or "Add more tests for create_index method"
 
-Please ensure your commits are small and focused; they should tell a story of
-your change. This helps reviewers to follow your changes, especially for more
-complex changes.
+We don't follow any other convention, but if you want to use one, we recommend [the Chris Beams one](https://chris.beams.io/posts/git-commit/).
 
-Familiarise yourself with [How to Write a Git Commit Message](https://chris.beams.io/posts/git-commit/).
+### GitHub Pull Requests
 
-### Github Pull Requests
+Some notes on GitHub PRs:
 
-Once your changes are ready you must submit your branch as a pull request.
+- All PRs must be reviewed and approved by at least one maintainer.
+- The PR title should be accurate and descriptive of the changes.
+- [Convert your PR as a draft](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/changing-the-stage-of-a-pull-request) if your changes are a work in progress: no one will review it until you pass your PR as ready for review.<br>
+  The draft PRs are recommended when you want to show that you are working on something and make your work visible.
+- The branch related to the PR must be **up-to-date with `main`** before merging. Fortunately, this project uses [Bors](https://github.com/bors-ng/bors-ng) to automatically enforce this requirement without the PR author having to rebase manually.
 
-#### Reviews & Approvals
+## Release Process (for internal team only)
 
-All pull requests must be reviewed and approved by at least one MeiliSearch team
-member.
+Meilisearch tools follow the [Semantic Versioning Convention](https://semver.org/).
 
-#### Merge Style
+### Automation to rebase and Merge the PRs
 
-All pull requests are squashed and merged. We generally discourage large pull
-requests that are over 300-500 lines of diff. If you would like to propose
-a change that is larger we suggest coming onto our chat channel and
-discuss it with one of our engineers. This way we can talk through the
-solution and discuss if a change that large is even needed! This overall
-will produce a quicker response to the change and likely produce code that
-aligns better with our process.
+This project integrates a bot that helps us manage pull requests merging.<br>
+_[Read more about this](https://github.com/meilisearch/integration-guides/blob/main/resources/bors.md)._
 
-## Development
+### How to Publish a new Release
 
-### Setup
+The full Meilisearch release process is described in [this guide](https://github.com/meilisearch/core-team/blob/main/resources/meilisearch-release.md). Please follow it carefully before doing any release.
 
-See the [MeiliSearch Docs](https://docs.meilisearch.com/reference/features/installation.html) for how to set up a development environment.
+### Release assets
 
-### Benchmarking & Profiling
+For each release, the following assets are created:
+- Binaries for differents platforms (Linux, MacOS, Windows and ARM architectures) are attached to the GitHub release
+- Binaries are pushed to HomeBrew and APT (not published for RC)
+- Docker tags are created/updated:
+  - `vX.Y.Z`
+  - `vX.Y` (not published for RC)
+  - `latest` (not published for RC)
 
-We do not yet do any benchmarking, nor have we formalised our profiling. If you'd like to work on this please get in touch!
+<hr>
 
-## Humans
-
-After making your change, you'll want to prepare it for MeiliSearch users (mostly humans). This usually entails updating documentation and announcing your feature.
-
-### Documentation
-
-Documentation is very important to MeiliSearch. All contributions that
-alter user-facing behavior MUST include documentation changes. Please see
-[GitHub.com/meilisearch/documentation](https://github.com/meilisearch/documentation) for more info.
-
-### Changelog
-
-Until we have guidelines in place, updating the [`Changelog`](/CHANGELOG.md) is solely the responsibility of MeiliSearch team members.
+Thank you again for reading this through, we can not wait to begin to work with you if you made your way through this contributing guide ❤️

Cargo.toml

@@ -1,8 +1,15 @@
 [workspace]
+resolver = "2"
 members = [
     "meilisearch-http",
-    "meilisearch-error",
+    "meilisearch-types",
+    "meilisearch-lib",
+    "meilisearch-auth",
+    "permissive-json-pointer",
 ]
 
-[profile.release]
-debug = true
+[profile.dev.package.flate2]
+opt-level = 3
+
+[profile.dev.package.milli]
+opt-level = 3

Cross.toml

@@ -0,0 +1,7 @@
+[build.env]
+passthrough = [
+    "RUST_BACKTRACE",
+    "CARGO_TERM_COLOR",
+    "RUSTFLAGS",
+    "JEMALLOC_SYS_WITH_LG_PAGE"
+]

Dockerfile

@@ -1,45 +1,46 @@
 # Compile
-FROM    alpine:3.14 AS compiler
+FROM    rust:alpine3.14 AS compiler
 
-RUN     apk update --quiet
-RUN     apk add curl
-RUN     apk add build-base
-
-RUN     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
+RUN     apk add -q --update-cache --no-cache build-base openssl-dev
 
 WORKDIR /meilisearch
 
-COPY    Cargo.lock .
-COPY    Cargo.toml .
-
-COPY    meilisearch-error/Cargo.toml meilisearch-error/
-COPY    meilisearch-http/Cargo.toml meilisearch-http/
-
-ENV     RUSTFLAGS="-C target-feature=-crt-static"
-
-# Create dummy main.rs files for each workspace member to be able to compile all the dependencies
-RUN     find . -type d -name "meilisearch-*" | xargs -I{} sh -c 'mkdir {}/src; echo "fn main() { }" > {}/src/main.rs;'
-# Use `cargo build` instead of `cargo vendor` because we need to not only download but compile dependencies too
-RUN     $HOME/.cargo/bin/cargo build --release
-# Cleanup dummy main.rs files
-RUN     find . -path "*/src/main.rs" -delete
-
 ARG     COMMIT_SHA
 ARG     COMMIT_DATE
 ENV     COMMIT_SHA=${COMMIT_SHA} COMMIT_DATE=${COMMIT_DATE}
+ENV     RUSTFLAGS="-C target-feature=-crt-static"
 
 COPY    . .
-RUN     $HOME/.cargo/bin/cargo build --release
+RUN     set -eux; \
+        apkArch="$(apk --print-arch)"; \
+        if [ "$apkArch" = "aarch64" ]; then \
+            export JEMALLOC_SYS_WITH_LG_PAGE=16; \
+        fi && \
+        cargo build --release
 
 # Run
 FROM    alpine:3.14
 
-RUN     apk add -q --no-cache libgcc tini
-
-COPY    --from=compiler /meilisearch/target/release/meilisearch .
-
 ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+ENV     MEILI_SERVER_PROVIDER docker
+
+RUN     apk update --quiet \
+        && apk add -q --no-cache libgcc tini curl
+
+# add meilisearch to the `/bin` so you can run it from anywhere and it's easy
+# to find.
+COPY    --from=compiler /meilisearch/target/release/meilisearch /bin/meilisearch
+# To stay compatible with the older version of the container (pre v0.27.0) we're
+# going to symlink the meilisearch binary in the path to `/meilisearch`
+RUN     ln -s /bin/meilisearch /meilisearch
+
+# This directory should hold all the data related to meilisearch so we're going
+# to move our PWD in there.
+# We don't want to put the meilisearch binary
+WORKDIR /meili_data
+
+
 EXPOSE  7700/tcp
 
 ENTRYPOINT ["tini", "--"]
-CMD     ./meilisearch
+CMD     /bin/meilisearch

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2019-2021 Meili SAS
+Copyright (c) 2019-2022 Meili SAS
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -1,8 +1,8 @@
 <p align="center">
-  <img src="assets/logo.svg" alt="MeiliSearch" width="200" height="200" />
+  <img src="assets/logo.svg" alt="Meilisearch" width="200" height="200" />
 </p>
 
-<h1 align="center">MeiliSearch</h1>
+<h1 align="center">Meilisearch</h1>
 
 <h4 align="center">
   <a href="https://www.meilisearch.com">Website</a> |
@@ -15,17 +15,17 @@
 </h4>
 
 <p align="center">
-  <a href="https://github.com/meilisearch/MeiliSearch/actions"><img src="https://github.com/meilisearch/MeiliSearch/workflows/Cargo%20test/badge.svg" alt="Build Status"></a>
-  <a href="https://deps.rs/repo/github/meilisearch/MeiliSearch"><img src="https://deps.rs/repo/github/meilisearch/MeiliSearch/status.svg" alt="Dependency status"></a>
-  <a href="https://github.com/meilisearch/MeiliSearch/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
-  <a href="https://slack.meilisearch.com"><img src="https://img.shields.io/badge/slack-MeiliSearch-blue.svg?logo=slack" alt="Slack"></a>
-  <a href="https://github.com/meilisearch/MeiliSearch/discussions" alt="Discussions"><img src="https://img.shields.io/badge/github-discussions-red" /></a>
+  <a href="https://github.com/meilisearch/meilisearch/actions"><img src="https://github.com/meilisearch/meilisearch/workflows/Cargo%20test/badge.svg" alt="Build Status"></a>
+  <a href="https://deps.rs/repo/github/meilisearch/meilisearch"><img src="https://deps.rs/repo/github/meilisearch/meilisearch/status.svg" alt="Dependency status"></a>
+  <a href="https://github.com/meilisearch/meilisearch/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-informational" alt="License"></a>
+  <a href="https://slack.meilisearch.com"><img src="https://img.shields.io/badge/slack-meilisearch-blue.svg?logo=slack" alt="Slack"></a>
+  <a href="https://github.com/meilisearch/meilisearch/discussions" alt="Discussions"><img src="https://img.shields.io/badge/github-discussions-red" /></a>
   <a href="https://app.bors.tech/repositories/26457"><img src="https://bors.tech/images/badge_small.svg" alt="Bors enabled"></a>
 </p>
 
 <p align="center">⚡ Lightning Fast, Ultra Relevant, and Typo-Tolerant Search Engine 🔍</p>
 
-**MeiliSearch** is a powerful, fast, open-source, easy to use and deploy search engine. Both searching and indexing are highly customizable. Features such as typo-tolerance, filters, and synonyms are provided out-of-the-box.
+**Meilisearch** is a powerful, fast, open-source, easy to use and deploy search engine. Both searching and indexing are highly customizable. Features such as typo-tolerance, filters, and synonyms are provided out-of-the-box.
 For more information about features go to [our documentation](https://docs.meilisearch.com/).
 
 <p align="center">
@@ -58,12 +58,16 @@ meilisearch
 #### Docker
 
 ```bash
-docker run -p 7700:7700 -v "$(pwd)/data.ms:/data.ms" getmeili/meilisearch
+docker run -p 7700:7700 -v "$(pwd)/meili_data:/meili_data" getmeili/meilisearch
 ```
 
-#### Try MeiliSearch in our Sandbox
+#### Announcing a cloud-hosted Meilisearch
 
-Create a MeiliSearch instance in [MeiliSearch Sandbox](https://sandbox.meilisearch.com/). This instance is free, and will be active for 48 hours.
+Join the closed beta by filling out this [form](https://meilisearch.typeform.com/to/FtnzvZfh).
+
+#### Try Meilisearch in our Sandbox
+
+Create a Meilisearch instance in [Meilisearch Sandbox](https://sandbox.meilisearch.com/). This instance is free, and will be active for 48 hours.
 
 #### Run on Digital Ocean
 
@@ -95,8 +99,8 @@ curl -L https://install.meilisearch.com | sh
 If you have the latest stable Rust toolchain installed on your local system, clone the repository and change it to your working directory.
 
 ```bash
-git clone https://github.com/meilisearch/MeiliSearch.git
-cd MeiliSearch
+git clone https://github.com/meilisearch/meilisearch.git
+cd meilisearch
 cargo run --release
 ```
 
@@ -105,7 +109,7 @@ cargo run --release
 Let's create an index! If you need a sample dataset, use [this movie database](https://www.notion.so/meilisearch/A-movies-dataset-to-test-Meili-1cbf7c9cfa4247249c40edfa22d7ca87#b5ae399b81834705ba5420ac70358a65). You can also find it in the `datasets/` directory.
 
 ```bash
-curl -L 'https://bit.ly/2PAcw9l' -o movies.json
+curl -L https://docs.meilisearch.com/movies.json -o movies.json
 ```
 
 Now, you're ready to index some data.
@@ -157,19 +161,19 @@ curl 'http://127.0.0.1:7700/indexes/movies/search?q=botman+robin&limit=2' | jq
 
 #### Use the Web Interface
 
-We also deliver an **out-of-the-box [web interface](https://github.com/meilisearch/mini-dashboard)** in which you can test MeiliSearch interactively.
+We also deliver an **out-of-the-box [web interface](https://github.com/meilisearch/mini-dashboard)** in which you can test Meilisearch interactively.
 
-You can access the web interface in your web browser at the root of the server. The default URL is [http://127.0.0.1:7700](http://127.0.0.1:7700). All you need to do is open your web browser and enter MeiliSearch’s address to visit it. This will lead you to a web page with a search bar that will allow you to search in the selected index.
+You can access the web interface in your web browser at the root of the server. The default URL is [http://127.0.0.1:7700](http://127.0.0.1:7700). All you need to do is open your web browser and enter Meilisearch’s address to visit it. This will lead you to a web page with a search bar that will allow you to search in the selected index.
 
 | [See the gif above](#demo)
 
 ## Documentation
 
-Now that your MeiliSearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com).
+Now that your Meilisearch server is up and running, you can learn more about how to tune your search engine in [the documentation](https://docs.meilisearch.com).
 
 ## Contributing
 
-Hey! We're glad you're thinking about contributing to MeiliSearch! Feel free to pick an [issue labeled as `good first issue`](https://github.com/meilisearch/MeiliSearch/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22), and to ask any question you need. Some points might not be clear and we are available to help you!
+Hey! We're glad you're thinking about contributing to Meilisearch! Feel free to pick an [issue labeled as `good first issue`](https://github.com/meilisearch/meilisearch/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22), and to ask any question you need. Some points might not be clear and we are available to help you!
 
 Also, we recommend following the [CONTRIBUTING](./CONTRIBUTING.md) to create your PR.
 
@@ -180,16 +184,16 @@ The code in this repository is only concerned with managing multiple indexes, ha
 Search and indexation are the domain of our core engine, [`milli`](https://github.com/meilisearch/milli), while tokenization is handled by [our `tokenizer` library](https://github.com/meilisearch/tokenizer/).
 ## Telemetry
 
-MeiliSearch collects anonymous data regarding general usage.
-This helps us better understand developers' usage of MeiliSearch features.
+Meilisearch collects anonymous data regarding general usage.
+This helps us better understand developers' usage of Meilisearch features.
 
-To see what information we're retrieving, please see the complete list [on the dedicated issue](https://github.com/meilisearch/MeiliSearch/issues/720).
+To find out more on what information we're retrieving, please see our documentation on [Telemetry](https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html).
 
 This program is optional, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.
 
 ## Feature request
 
-The feature requests are not managed in this repository. Please visit our [dedicated repository](https://github.com/meilisearch/product) to see our work about the MeiliSearch product.
+The feature requests are not managed in this repository. Please visit our [dedicated repository](https://github.com/meilisearch/product) to see our work about the Meilisearch product.
 
 If you have a feature request or any feedback about an existing feature, please open [a discussion](https://github.com/meilisearch/product/discussions).
 Also, feel free to participate in the current discussions, we are looking forward to reading your comments.
@@ -198,4 +202,4 @@ Also, feel free to participate in the current discussions, we are looking forwar
 
 Please visit [this page](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html#contact-us).
 
-MeiliSearch is developed by [Meili](https://www.meilisearch.com), a young company. To know more about us, you can [read our blog](https://blog.meilisearch.com). Any suggestion or feedback is highly appreciated. Thank you for your support!
+Meilisearch is developed by [Meili](https://www.meilisearch.com), a young company. To know more about us, you can [read our blog](https://blog.meilisearch.com). Any suggestion or feedback is highly appreciated. Thank you for your support!

SECURITY.md

@@ -0,0 +1,33 @@
+# Security
+
+Meilisearch takes the security of our software products and services seriously.
+
+If you believe you have found a security vulnerability in any Meilisearch-owned repository, please report it to us as described below.
+
+## Supported versions
+
+As long as we are pre-v1.0, only the latest version of Meilisearch will be supported with security updates.
+
+## Reporting security issues
+
+⚠️ Please do not report security vulnerabilities through public GitHub issues. ⚠️
+
+Instead, please kindly email us at security@meilisearch.com
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+- Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+- Full paths of source file(s) related to the manifestation of the issue
+- The location of the affected source code (tag/branch/commit or direct URL)
+- Any special configuration required to reproduce the issue
+- Step-by-step instructions to reproduce the issue
+- Proof-of-concept or exploit code (if possible)
+- Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+You will receive a response from us within 72 hours. If the issue is confirmed, we will release a patch as soon as possible depending on complexity.
+
+## Preferred languages
+
+We prefer all communications to be in English.

assets/logo.svg

@@ -1,17 +1,19 @@
-<svg width="360" height="360" viewBox="0 0 360 360" fill="none" xmlns="http://www.w3.org/2000/svg">
-<g id="logo_main">
-<rect id="Rectangle" x="107.333" y="0.150146" width="274.315" height="274.315" rx="98.8334" transform="rotate(23 107.333 0.150146)" fill="url(#paint0_linear)"/>
-<path id="Rectangle_2" fill-rule="evenodd" clip-rule="evenodd" d="M61.3296 230.199C46.2224 194.608 38.6688 176.813 38.208 160.329C37.5286 136.025 47.0175 112.539 64.3891 95.5282C76.1718 83.9904 93.9669 76.4368 129.557 61.3296C165.147 46.2224 182.943 38.6688 199.427 38.208C223.731 37.5286 247.217 47.0175 264.228 64.3891C275.766 76.1718 283.319 93.9669 298.426 129.557C313.534 165.147 321.087 182.943 321.548 199.427C322.227 223.731 312.738 247.217 295.367 264.228C283.584 275.766 265.789 283.319 230.199 298.426C194.608 313.534 176.813 321.087 160.329 321.548C136.025 322.227 112.539 312.738 95.5282 295.367C83.9903 283.584 76.4368 265.789 61.3296 230.199Z" fill="url(#paint1_linear)"/>
-<path id="m" fill-rule="evenodd" clip-rule="evenodd" d="M219.568 130.748C242.363 130.748 259.263 147.451 259.263 174.569V229.001H227.232V179.678C227.232 166.119 220.747 159.634 210.136 159.634C205.223 159.634 200.311 161.796 195.595 167.494C195.791 169.852 195.988 172.21 195.988 174.569V229.001H164.154V179.678C164.154 166.119 157.472 159.634 147.057 159.634C142.145 159.634 137.429 161.992 132.712 168.084V229.001H100.878V133.695H132.712V139.394C139.197 133.892 145.878 130.748 156.49 130.748C168.477 130.748 178.695 135.267 185.769 143.52C195.791 134.678 205.42 130.748 219.568 130.748Z" fill="white"/>
-</g>
+<svg width="300" height="300" viewBox="0 0 300 300" fill="none" xmlns="http://www.w3.org/2000/svg">
+<path d="M0 237L55.426 96.7678C63.2367 77.0063 82.499 64 103.955 64H137.371L81.9447 204.232C74.1341 223.993 54.8717 237 33.4156 237H0Z" fill="url(#paint0_linear_1_898)"/>
+<path d="M81.3123 237L136.738 96.7682C144.549 77.0067 163.811 64.0004 185.267 64.0004H218.683L163.257 204.232C155.446 223.994 136.184 237 114.728 237H81.3123Z" fill="url(#paint1_linear_1_898)"/>
+<path d="M162.629 237L218.055 96.7682C225.866 77.0067 245.128 64.0004 266.584 64.0004H300L244.574 204.232C236.763 223.994 217.501 237 196.045 237H162.629Z" fill="url(#paint2_linear_1_898)"/>
 <defs>
-<linearGradient id="paint0_linear" x1="-13.6248" y1="129.208" x2="244.49" y2="403.522" gradientUnits="userSpaceOnUse">
-<stop stop-color="#E41359"/>
-<stop offset="1" stop-color="#F23C79"/>
+<linearGradient id="paint0_linear_1_898" x1="300.001" y1="50.7858" x2="1.63474" y2="221.244" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FF5CAA"/>
+<stop offset="1" stop-color="#FF4E62"/>
 </linearGradient>
-<linearGradient id="paint1_linear" x1="11.0088" y1="111.65" x2="111.65" y2="348.747" gradientUnits="userSpaceOnUse">
-<stop stop-color="#24222F"/>
-<stop offset="1" stop-color="#2B2937"/>
+<linearGradient id="paint1_linear_1_898" x1="300.001" y1="50.7858" x2="1.63474" y2="221.244" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FF5CAA"/>
+<stop offset="1" stop-color="#FF4E62"/>
+</linearGradient>
+<linearGradient id="paint2_linear_1_898" x1="300.001" y1="50.7858" x2="1.63474" y2="221.244" gradientUnits="userSpaceOnUse">
+<stop stop-color="#FF5CAA"/>
+<stop offset="1" stop-color="#FF4E62"/>
 </linearGradient>
 </defs>
 </svg>

bors.toml

@@ -3,7 +3,9 @@ status = [
     'Tests on macos-latest',
     'Tests on windows-latest',
     'Run Clippy',
-    'Run Rustfmt'
+    'Run Rustfmt',
+    'Run tests in debug',
 ]
+pr_status = ['Milestone Check']
 # 3 hours timeout
 timeout-sec = 10800

datasets/movies/README.md

@@ -1 +0,0 @@
-_datas in movies.json are from https://www.themoviedb.org/_

download-latest.sh

@@ -51,13 +51,13 @@ semverLT() {
     if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
         return 0
     fi
-    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+    if [ "_$SPECIAL_A"  == '_' ] && [ "_$SPECIAL_B"  == '_' ] ; then
         return 1
     fi
-    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  != "_" ] ; then
+    if [ "_$SPECIAL_A"  == '_' ] && [ "_$SPECIAL_B"  != '_' ] ; then
         return 1
     fi
-    if [ "_$SPECIAL_A"  != "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+    if [ "_$SPECIAL_A"  != '_' ] && [ "_$SPECIAL_B"  == '_' ] ; then
         return 0
     fi
     if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
@@ -67,39 +67,47 @@ semverLT() {
     return 1
 }
 
+# Get a token from https://github.com/settings/tokens to increase rate limit (from 60 to 5000), make sure the token scope is set to 'public_repo'
+# Create GITHUB_PAT environment variable once you acquired the token to start using it
 # Returns the tag of the latest stable release (in terms of semver and not of release date)
 get_latest() {
     temp_file='temp_file' # temp_file needed because the grep would start before the download is over
-    curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file" || return 1
+
+    if [ -z "$GITHUB_PAT" ]; then
+        curl -s 'https://api.github.com/repos/meilisearch/meilisearch/releases' > "$temp_file" || return 1
+    else
+        curl -H "Authorization: token $GITHUB_PAT" -s 'https://api.github.com/repos/meilisearch/meilisearch/releases' > "$temp_file" || return 1
+    fi
+
     releases=$(cat "$temp_file" | \
-        grep -E "tag_name|draft|prerelease" \
+        grep -E '"tag_name":|"draft":|"prerelease":' \
         | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
         # Returns a list of [tag_name draft_boolean prerelease_boolean ...]
         # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
 
     i=0
-    latest=""
-    current_tag=""
+    latest=''
+    current_tag=''
     for release_info in $releases; do
-        if [ $i -eq 0 ]; then # Cheking tag_name
+        if [ $i -eq 0 ]; then # Checking tag_name
             if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
                 current_tag=$release_info
             else
-                current_tag=""
+                current_tag=''
             fi
             i=1
         elif [ $i -eq 1 ]; then # Checking draft boolean
-            if [ "$release_info" = "true" ]; then
-                current_tag=""
+            if [ "$release_info" = 'true' ]; then
+                current_tag=''
             fi
             i=2
         elif [ $i -eq 2 ]; then # Checking prerelease boolean
-            if [ "$release_info" = "true" ]; then
-                current_tag=""
+            if [ "$release_info" = 'true' ]; then
+                current_tag=''
             fi
             i=0
-            if [ "$current_tag" != "" ]; then # If the current_tag is valid
-                if [ "$latest" = "" ]; then # If there is no latest yet
+            if [ "$current_tag" != '' ]; then # If the current_tag is valid
+                if [ "$latest" = '' ]; then # If there is no latest yet
                     latest="$current_tag"
                 else
                     semverLT $current_tag $latest # Comparing latest and the current tag
@@ -112,7 +120,7 @@ get_latest() {
     done
 
     rm -f "$temp_file"
-    echo $latest
+    return 0
 }
 
 # Gets the OS by setting the $os variable
@@ -140,11 +148,18 @@ get_os() {
 get_archi() {
     architecture=$(uname -m)
     case "$architecture" in
-    'x86_64' | 'amd64')
+    'x86_64' | 'amd64' )
         archi='amd64'
         ;;
+    'arm64')
+        if [ $os = 'macos' ]; then # MacOS M1
+            archi='amd64'
+        else
+            archi='aarch64'
+        fi
+        ;;
     'aarch64')
-        archi='armv8'
+        archi='aarch64'
         ;;
     *)
         return 1
@@ -153,7 +168,7 @@ get_archi() {
 }
 
 success_usage() {
-    printf "$GREEN%s\n$DEFAULT" "MeiliSearch binary successfully downloaded as '$BINARY_NAME' file."
+    printf "$GREEN%s\n$DEFAULT" "Meilisearch $latest binary successfully downloaded as '$binary_name' file."
     echo ''
     echo 'Run it:'
     echo '    $ ./meilisearch'
@@ -161,40 +176,65 @@ success_usage() {
     echo '    $ ./meilisearch --help'
 }
 
-failure_usage() {
-    printf "$RED%s\n$DEFAULT" 'ERROR: MeiliSearch binary is not available for your OS distribution or your architecture yet.'
+not_available_failure_usage() {
+    printf "$RED%s\n$DEFAULT" 'ERROR: Meilisearch binary is not available for your OS distribution or your architecture yet.'
     echo ''
     echo 'However, you can easily compile the binary from the source files.'
     echo 'Follow the steps at the page ("Source" tab): https://docs.meilisearch.com/learn/getting_started/installation.html'
 }
 
+fetch_release_failure_usage() {
+    echo ''
+    printf "$RED%s\n$DEFAULT" 'ERROR: Impossible to get the latest stable version of Meilisearch.'
+    echo 'Please let us know about this issue: https://github.com/meilisearch/meilisearch/issues/new/choose'
+}
+
 # MAIN
-latest="$(get_latest)"
 
+# Fill $latest variable
+if ! get_latest; then
+    fetch_release_failure_usage # TO CHANGE
+    exit 1
+fi
+
+if [ "$latest" = '' ]; then
+    fetch_release_failure_usage
+    exit 1
+fi
+
+# Fill $os variable
 if ! get_os; then
-    failure_usage
+    not_available_failure_usage
     exit 1
 fi
 
+# Fill $archi variable
 if ! get_archi; then
-    failure_usage
+    not_available_failure_usage
     exit 1
 fi
 
-echo "Downloading MeiliSearch binary $latest for $os, architecture $archi..."
+echo "Downloading Meilisearch binary $latest for $os, architecture $archi..."
 case "$os" in
     'windows')
         release_file="meilisearch-$os-$archi.exe"
-		BINARY_NAME='meilisearch.exe'
+        binary_name='meilisearch.exe'
 
         ;;
-	*) 
-		release_file="meilisearch-$os-$archi"
-		BINARY_NAME='meilisearch'
+    *)
+        release_file="meilisearch-$os-$archi"
+        binary_name='meilisearch'
 
 esac
-link="https://github.com/meilisearch/MeiliSearch/releases/download/$latest/$release_file"
-curl -OL "$link"
-mv "$release_file" "$BINARY_NAME"
-chmod 744 "$BINARY_NAME"
+
+# Fetch the Meilisearch binary
+link="https://github.com/meilisearch/meilisearch/releases/download/$latest/$release_file"
+curl --fail -OL "$link"
+if [ $? -ne 0 ]; then
+    fetch_release_failure_usage
+    exit 1
+fi
+
+mv "$release_file" "$binary_name"
+chmod 744 "$binary_name"
 success_usage

meilisearch-auth/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "meilisearch-auth"
+version = "0.28.1"
+edition = "2021"
+
+[dependencies]
+enum-iterator = "0.7.0"
+hmac = "0.12.1"
+meilisearch-types = { path = "../meilisearch-types" }
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.31.2" }
+rand = "0.8.4"
+serde = { version = "1.0.136", features = ["derive"] }
+serde_json = { version = "1.0.79", features = ["preserve_order"] }
+sha2 = "0.10.2"
+thiserror = "1.0.30"
+time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+uuid = { version = "1.1.2", features = ["serde", "v4"] }

meilisearch-auth/src/action.rs

@@ -0,0 +1,151 @@
+use enum_iterator::IntoEnumIterator;
+use serde::{Deserialize, Serialize};
+use std::hash::Hash;
+
+#[derive(IntoEnumIterator, Copy, Clone, Serialize, Deserialize, Debug, Eq, PartialEq, Hash)]
+#[repr(u8)]
+pub enum Action {
+    #[serde(rename = "*")]
+    All = actions::ALL,
+    #[serde(rename = "search")]
+    Search = actions::SEARCH,
+    #[serde(rename = "documents.*")]
+    DocumentsAll = actions::DOCUMENTS_ALL,
+    #[serde(rename = "documents.add")]
+    DocumentsAdd = actions::DOCUMENTS_ADD,
+    #[serde(rename = "documents.get")]
+    DocumentsGet = actions::DOCUMENTS_GET,
+    #[serde(rename = "documents.delete")]
+    DocumentsDelete = actions::DOCUMENTS_DELETE,
+    #[serde(rename = "indexes.*")]
+    IndexesAll = actions::INDEXES_ALL,
+    #[serde(rename = "indexes.create")]
+    IndexesAdd = actions::INDEXES_CREATE,
+    #[serde(rename = "indexes.get")]
+    IndexesGet = actions::INDEXES_GET,
+    #[serde(rename = "indexes.update")]
+    IndexesUpdate = actions::INDEXES_UPDATE,
+    #[serde(rename = "indexes.delete")]
+    IndexesDelete = actions::INDEXES_DELETE,
+    #[serde(rename = "tasks.*")]
+    TasksAll = actions::TASKS_ALL,
+    #[serde(rename = "tasks.get")]
+    TasksGet = actions::TASKS_GET,
+    #[serde(rename = "settings.*")]
+    SettingsAll = actions::SETTINGS_ALL,
+    #[serde(rename = "settings.get")]
+    SettingsGet = actions::SETTINGS_GET,
+    #[serde(rename = "settings.update")]
+    SettingsUpdate = actions::SETTINGS_UPDATE,
+    #[serde(rename = "stats.*")]
+    StatsAll = actions::STATS_ALL,
+    #[serde(rename = "stats.get")]
+    StatsGet = actions::STATS_GET,
+    #[serde(rename = "dumps.*")]
+    DumpsAll = actions::DUMPS_ALL,
+    #[serde(rename = "dumps.create")]
+    DumpsCreate = actions::DUMPS_CREATE,
+    #[serde(rename = "version")]
+    Version = actions::VERSION,
+    #[serde(rename = "keys.create")]
+    KeysAdd = actions::KEYS_CREATE,
+    #[serde(rename = "keys.get")]
+    KeysGet = actions::KEYS_GET,
+    #[serde(rename = "keys.update")]
+    KeysUpdate = actions::KEYS_UPDATE,
+    #[serde(rename = "keys.delete")]
+    KeysDelete = actions::KEYS_DELETE,
+}
+
+impl Action {
+    pub fn from_repr(repr: u8) -> Option<Self> {
+        use actions::*;
+        match repr {
+            ALL => Some(Self::All),
+            SEARCH => Some(Self::Search),
+            DOCUMENTS_ALL => Some(Self::DocumentsAll),
+            DOCUMENTS_ADD => Some(Self::DocumentsAdd),
+            DOCUMENTS_GET => Some(Self::DocumentsGet),
+            DOCUMENTS_DELETE => Some(Self::DocumentsDelete),
+            INDEXES_ALL => Some(Self::IndexesAll),
+            INDEXES_CREATE => Some(Self::IndexesAdd),
+            INDEXES_GET => Some(Self::IndexesGet),
+            INDEXES_UPDATE => Some(Self::IndexesUpdate),
+            INDEXES_DELETE => Some(Self::IndexesDelete),
+            TASKS_ALL => Some(Self::TasksAll),
+            TASKS_GET => Some(Self::TasksGet),
+            SETTINGS_ALL => Some(Self::SettingsAll),
+            SETTINGS_GET => Some(Self::SettingsGet),
+            SETTINGS_UPDATE => Some(Self::SettingsUpdate),
+            STATS_ALL => Some(Self::StatsAll),
+            STATS_GET => Some(Self::StatsGet),
+            DUMPS_ALL => Some(Self::DumpsAll),
+            DUMPS_CREATE => Some(Self::DumpsCreate),
+            VERSION => Some(Self::Version),
+            KEYS_CREATE => Some(Self::KeysAdd),
+            KEYS_GET => Some(Self::KeysGet),
+            KEYS_UPDATE => Some(Self::KeysUpdate),
+            KEYS_DELETE => Some(Self::KeysDelete),
+            _otherwise => None,
+        }
+    }
+
+    pub fn repr(&self) -> u8 {
+        use actions::*;
+        match self {
+            Self::All => ALL,
+            Self::Search => SEARCH,
+            Self::DocumentsAll => DOCUMENTS_ALL,
+            Self::DocumentsAdd => DOCUMENTS_ADD,
+            Self::DocumentsGet => DOCUMENTS_GET,
+            Self::DocumentsDelete => DOCUMENTS_DELETE,
+            Self::IndexesAll => INDEXES_ALL,
+            Self::IndexesAdd => INDEXES_CREATE,
+            Self::IndexesGet => INDEXES_GET,
+            Self::IndexesUpdate => INDEXES_UPDATE,
+            Self::IndexesDelete => INDEXES_DELETE,
+            Self::TasksAll => TASKS_ALL,
+            Self::TasksGet => TASKS_GET,
+            Self::SettingsAll => SETTINGS_ALL,
+            Self::SettingsGet => SETTINGS_GET,
+            Self::SettingsUpdate => SETTINGS_UPDATE,
+            Self::StatsAll => STATS_ALL,
+            Self::StatsGet => STATS_GET,
+            Self::DumpsAll => DUMPS_ALL,
+            Self::DumpsCreate => DUMPS_CREATE,
+            Self::Version => VERSION,
+            Self::KeysAdd => KEYS_CREATE,
+            Self::KeysGet => KEYS_GET,
+            Self::KeysUpdate => KEYS_UPDATE,
+            Self::KeysDelete => KEYS_DELETE,
+        }
+    }
+}
+
+pub mod actions {
+    pub(crate) const ALL: u8 = 0;
+    pub const SEARCH: u8 = 1;
+    pub const DOCUMENTS_ALL: u8 = 2;
+    pub const DOCUMENTS_ADD: u8 = 3;
+    pub const DOCUMENTS_GET: u8 = 4;
+    pub const DOCUMENTS_DELETE: u8 = 5;
+    pub const INDEXES_ALL: u8 = 6;
+    pub const INDEXES_CREATE: u8 = 7;
+    pub const INDEXES_GET: u8 = 8;
+    pub const INDEXES_UPDATE: u8 = 9;
+    pub const INDEXES_DELETE: u8 = 10;
+    pub const TASKS_ALL: u8 = 11;
+    pub const TASKS_GET: u8 = 12;
+    pub const SETTINGS_ALL: u8 = 13;
+    pub const SETTINGS_GET: u8 = 14;
+    pub const SETTINGS_UPDATE: u8 = 15;
+    pub const STATS_ALL: u8 = 16;
+    pub const STATS_GET: u8 = 17;
+    pub const DUMPS_ALL: u8 = 18;
+    pub const DUMPS_CREATE: u8 = 19;
+    pub const VERSION: u8 = 20;
+    pub const KEYS_CREATE: u8 = 21;
+    pub const KEYS_GET: u8 = 22;
+    pub const KEYS_UPDATE: u8 = 23;
+    pub const KEYS_DELETE: u8 = 24;
+}

meilisearch-auth/src/dump.rs

@@ -0,0 +1,47 @@
+use serde_json::Deserializer;
+
+use std::fs::File;
+use std::io::BufReader;
+use std::io::Write;
+use std::path::Path;
+
+use crate::{AuthController, HeedAuthStore, Result};
+
+const KEYS_PATH: &str = "keys";
+
+impl AuthController {
+    pub fn dump(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> Result<()> {
+        let mut store = HeedAuthStore::new(&src)?;
+
+        // do not attempt to close the database on drop!
+        store.set_drop_on_close(false);
+
+        let keys_file_path = dst.as_ref().join(KEYS_PATH);
+
+        let keys = store.list_api_keys()?;
+        let mut keys_file = File::create(&keys_file_path)?;
+        for key in keys {
+            serde_json::to_writer(&mut keys_file, &key)?;
+            keys_file.write_all(b"\n")?;
+        }
+
+        Ok(())
+    }
+
+    pub fn load_dump(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> Result<()> {
+        let store = HeedAuthStore::new(&dst)?;
+
+        let keys_file_path = src.as_ref().join(KEYS_PATH);
+
+        if !keys_file_path.exists() {
+            return Ok(());
+        }
+
+        let reader = BufReader::new(File::open(&keys_file_path)?);
+        for key in Deserializer::from_reader(reader).into_iter() {
+            store.put_api_key(key?)?;
+        }
+
+        Ok(())
+    }
+}

meilisearch-auth/src/error.rs

@@ -0,0 +1,60 @@
+use std::error::Error;
+
+use meilisearch_types::error::{Code, ErrorCode};
+use meilisearch_types::internal_error;
+use serde_json::Value;
+
+pub type Result<T> = std::result::Result<T, AuthControllerError>;
+
+#[derive(Debug, thiserror::Error)]
+pub enum AuthControllerError {
+    #[error("`{0}` field is mandatory.")]
+    MissingParameter(&'static str),
+    #[error("`actions` field value `{0}` is invalid. It should be an array of string representing action names.")]
+    InvalidApiKeyActions(Value),
+    #[error("`indexes` field value `{0}` is invalid. It should be an array of string representing index names.")]
+    InvalidApiKeyIndexes(Value),
+    #[error("`expiresAt` field value `{0}` is invalid. It should follow the RFC 3339 format to represents a date or datetime in the future or specified as a null value. e.g. 'YYYY-MM-DD' or 'YYYY-MM-DD HH:MM:SS'.")]
+    InvalidApiKeyExpiresAt(Value),
+    #[error("`description` field value `{0}` is invalid. It should be a string or specified as a null value.")]
+    InvalidApiKeyDescription(Value),
+    #[error(
+        "`name` field value `{0}` is invalid. It should be a string or specified as a null value."
+    )]
+    InvalidApiKeyName(Value),
+    #[error("`uid` field value `{0}` is invalid. It should be a valid UUID v4 string or omitted.")]
+    InvalidApiKeyUid(Value),
+    #[error("API key `{0}` not found.")]
+    ApiKeyNotFound(String),
+    #[error("`uid` field value `{0}` is already an existing API key.")]
+    ApiKeyAlreadyExists(String),
+    #[error("The `{0}` field cannot be modified for the given resource.")]
+    ImmutableField(String),
+    #[error("Internal error: {0}")]
+    Internal(Box<dyn Error + Send + Sync + 'static>),
+}
+
+internal_error!(
+    AuthControllerError: milli::heed::Error,
+    std::io::Error,
+    serde_json::Error,
+    std::str::Utf8Error
+);
+
+impl ErrorCode for AuthControllerError {
+    fn error_code(&self) -> Code {
+        match self {
+            Self::MissingParameter(_) => Code::MissingParameter,
+            Self::InvalidApiKeyActions(_) => Code::InvalidApiKeyActions,
+            Self::InvalidApiKeyIndexes(_) => Code::InvalidApiKeyIndexes,
+            Self::InvalidApiKeyExpiresAt(_) => Code::InvalidApiKeyExpiresAt,
+            Self::InvalidApiKeyDescription(_) => Code::InvalidApiKeyDescription,
+            Self::InvalidApiKeyName(_) => Code::InvalidApiKeyName,
+            Self::ApiKeyNotFound(_) => Code::ApiKeyNotFound,
+            Self::InvalidApiKeyUid(_) => Code::InvalidApiKeyUid,
+            Self::ApiKeyAlreadyExists(_) => Code::ApiKeyAlreadyExists,
+            Self::ImmutableField(_) => Code::ImmutableField,
+            Self::Internal(_) => Code::Internal,
+        }
+    }
+}

meilisearch-auth/src/key.rs

@@ -0,0 +1,201 @@
+use crate::action::Action;
+use crate::error::{AuthControllerError, Result};
+use crate::store::KeyId;
+
+use meilisearch_types::index_uid::IndexUid;
+use meilisearch_types::star_or::StarOr;
+use serde::{Deserialize, Serialize};
+use serde_json::{from_value, Value};
+use time::format_description::well_known::Rfc3339;
+use time::macros::{format_description, time};
+use time::{Date, OffsetDateTime, PrimitiveDateTime};
+use uuid::Uuid;
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Key {
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub description: Option<String>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub name: Option<String>,
+    pub uid: KeyId,
+    pub actions: Vec<Action>,
+    pub indexes: Vec<StarOr<IndexUid>>,
+    #[serde(with = "time::serde::rfc3339::option")]
+    pub expires_at: Option<OffsetDateTime>,
+    #[serde(with = "time::serde::rfc3339")]
+    pub created_at: OffsetDateTime,
+    #[serde(with = "time::serde::rfc3339")]
+    pub updated_at: OffsetDateTime,
+}
+
+impl Key {
+    pub fn create_from_value(value: Value) -> Result<Self> {
+        let name = match value.get("name") {
+            None | Some(Value::Null) => None,
+            Some(des) => from_value(des.clone())
+                .map(Some)
+                .map_err(|_| AuthControllerError::InvalidApiKeyName(des.clone()))?,
+        };
+
+        let description = match value.get("description") {
+            None | Some(Value::Null) => None,
+            Some(des) => from_value(des.clone())
+                .map(Some)
+                .map_err(|_| AuthControllerError::InvalidApiKeyDescription(des.clone()))?,
+        };
+
+        let uid = value.get("uid").map_or_else(
+            || Ok(Uuid::new_v4()),
+            |uid| {
+                from_value(uid.clone())
+                    .map_err(|_| AuthControllerError::InvalidApiKeyUid(uid.clone()))
+            },
+        )?;
+
+        let actions = value
+            .get("actions")
+            .map(|act| {
+                from_value(act.clone())
+                    .map_err(|_| AuthControllerError::InvalidApiKeyActions(act.clone()))
+            })
+            .ok_or(AuthControllerError::MissingParameter("actions"))??;
+
+        let indexes = value
+            .get("indexes")
+            .map(|ind| {
+                from_value(ind.clone())
+                    .map_err(|_| AuthControllerError::InvalidApiKeyIndexes(ind.clone()))
+            })
+            .ok_or(AuthControllerError::MissingParameter("indexes"))??;
+
+        let expires_at = value
+            .get("expiresAt")
+            .map(parse_expiration_date)
+            .ok_or(AuthControllerError::MissingParameter("expiresAt"))??;
+
+        let created_at = OffsetDateTime::now_utc();
+        let updated_at = created_at;
+
+        Ok(Self {
+            name,
+            description,
+            uid,
+            actions,
+            indexes,
+            expires_at,
+            created_at,
+            updated_at,
+        })
+    }
+
+    pub fn update_from_value(&mut self, value: Value) -> Result<()> {
+        if let Some(des) = value.get("description") {
+            let des = from_value(des.clone())
+                .map_err(|_| AuthControllerError::InvalidApiKeyDescription(des.clone()));
+            self.description = des?;
+        }
+
+        if let Some(des) = value.get("name") {
+            let des = from_value(des.clone())
+                .map_err(|_| AuthControllerError::InvalidApiKeyName(des.clone()));
+            self.name = des?;
+        }
+
+        if value.get("uid").is_some() {
+            return Err(AuthControllerError::ImmutableField("uid".to_string()));
+        }
+
+        if value.get("actions").is_some() {
+            return Err(AuthControllerError::ImmutableField("actions".to_string()));
+        }
+
+        if value.get("indexes").is_some() {
+            return Err(AuthControllerError::ImmutableField("indexes".to_string()));
+        }
+
+        if value.get("expiresAt").is_some() {
+            return Err(AuthControllerError::ImmutableField("expiresAt".to_string()));
+        }
+
+        if value.get("createdAt").is_some() {
+            return Err(AuthControllerError::ImmutableField("createdAt".to_string()));
+        }
+
+        if value.get("updatedAt").is_some() {
+            return Err(AuthControllerError::ImmutableField("updatedAt".to_string()));
+        }
+
+        self.updated_at = OffsetDateTime::now_utc();
+
+        Ok(())
+    }
+
+    pub(crate) fn default_admin() -> Self {
+        let now = OffsetDateTime::now_utc();
+        let uid = Uuid::new_v4();
+        Self {
+            name: Some("Default Admin API Key".to_string()),
+            description: Some("Use it for anything that is not a search operation. Caution! Do not expose it on a public frontend".to_string()),
+            uid,
+            actions: vec![Action::All],
+            indexes: vec![StarOr::Star],
+            expires_at: None,
+            created_at: now,
+            updated_at: now,
+        }
+    }
+
+    pub(crate) fn default_search() -> Self {
+        let now = OffsetDateTime::now_utc();
+        let uid = Uuid::new_v4();
+        Self {
+            name: Some("Default Search API Key".to_string()),
+            description: Some("Use it to search from the frontend".to_string()),
+            uid,
+            actions: vec![Action::Search],
+            indexes: vec![StarOr::Star],
+            expires_at: None,
+            created_at: now,
+            updated_at: now,
+        }
+    }
+}
+
+fn parse_expiration_date(value: &Value) -> Result<Option<OffsetDateTime>> {
+    match value {
+        Value::String(string) => OffsetDateTime::parse(string, &Rfc3339)
+            .or_else(|_| {
+                PrimitiveDateTime::parse(
+                    string,
+                    format_description!(
+                        "[year repr:full base:calendar]-[month repr:numerical]-[day]T[hour]:[minute]:[second]"
+                    ),
+                ).map(|datetime| datetime.assume_utc())
+            })
+            .or_else(|_| {
+                PrimitiveDateTime::parse(
+                    string,
+                    format_description!(
+                        "[year repr:full base:calendar]-[month repr:numerical]-[day] [hour]:[minute]:[second]"
+                    ),
+                ).map(|datetime| datetime.assume_utc())
+            })
+            .or_else(|_| {
+                    Date::parse(string, format_description!(
+                        "[year repr:full base:calendar]-[month repr:numerical]-[day]"
+                    )).map(|date| PrimitiveDateTime::new(date, time!(00:00)).assume_utc())
+            })
+            .map_err(|_| AuthControllerError::InvalidApiKeyExpiresAt(value.clone()))
+            // check if the key is already expired.
+            .and_then(|d| {
+                if d > OffsetDateTime::now_utc() {
+                    Ok(d)
+                } else {
+                    Err(AuthControllerError::InvalidApiKeyExpiresAt(value.clone()))
+                }
+            })
+            .map(Option::Some),
+        Value::Null => Ok(None),
+        _otherwise => Err(AuthControllerError::InvalidApiKeyExpiresAt(value.clone())),
+    }
+}

meilisearch-auth/src/lib.rs

@@ -0,0 +1,258 @@
+mod action;
+mod dump;
+pub mod error;
+mod key;
+mod store;
+
+use std::collections::{HashMap, HashSet};
+use std::ops::Deref;
+use std::path::Path;
+use std::sync::Arc;
+
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+pub use action::{actions, Action};
+use error::{AuthControllerError, Result};
+pub use key::Key;
+use meilisearch_types::star_or::StarOr;
+use store::generate_key_as_hexa;
+pub use store::open_auth_store_env;
+use store::HeedAuthStore;
+
+#[derive(Clone)]
+pub struct AuthController {
+    store: Arc<HeedAuthStore>,
+    master_key: Option<String>,
+}
+
+impl AuthController {
+    pub fn new(db_path: impl AsRef<Path>, master_key: &Option<String>) -> Result<Self> {
+        let store = HeedAuthStore::new(db_path)?;
+
+        if store.is_empty()? {
+            generate_default_keys(&store)?;
+        }
+
+        Ok(Self {
+            store: Arc::new(store),
+            master_key: master_key.clone(),
+        })
+    }
+
+    pub fn create_key(&self, value: Value) -> Result<Key> {
+        let key = Key::create_from_value(value)?;
+        match self.store.get_api_key(key.uid)? {
+            Some(_) => Err(AuthControllerError::ApiKeyAlreadyExists(
+                key.uid.to_string(),
+            )),
+            None => self.store.put_api_key(key),
+        }
+    }
+
+    pub fn update_key(&self, uid: Uuid, value: Value) -> Result<Key> {
+        let mut key = self.get_key(uid)?;
+        key.update_from_value(value)?;
+        self.store.put_api_key(key)
+    }
+
+    pub fn get_key(&self, uid: Uuid) -> Result<Key> {
+        self.store
+            .get_api_key(uid)?
+            .ok_or_else(|| AuthControllerError::ApiKeyNotFound(uid.to_string()))
+    }
+
+    pub fn get_optional_uid_from_encoded_key(&self, encoded_key: &[u8]) -> Result<Option<Uuid>> {
+        match &self.master_key {
+            Some(master_key) => self
+                .store
+                .get_uid_from_encoded_key(encoded_key, master_key.as_bytes()),
+            None => Ok(None),
+        }
+    }
+
+    pub fn get_uid_from_encoded_key(&self, encoded_key: &str) -> Result<Uuid> {
+        self.get_optional_uid_from_encoded_key(encoded_key.as_bytes())?
+            .ok_or_else(|| AuthControllerError::ApiKeyNotFound(encoded_key.to_string()))
+    }
+
+    pub fn get_key_filters(
+        &self,
+        uid: Uuid,
+        search_rules: Option<SearchRules>,
+    ) -> Result<AuthFilter> {
+        let mut filters = AuthFilter::default();
+        let key = self
+            .store
+            .get_api_key(uid)?
+            .ok_or_else(|| AuthControllerError::ApiKeyNotFound(uid.to_string()))?;
+
+        if !key.indexes.iter().any(|i| i == &StarOr::Star) {
+            filters.search_rules = match search_rules {
+                // Intersect search_rules with parent key authorized indexes.
+                Some(search_rules) => SearchRules::Map(
+                    key.indexes
+                        .into_iter()
+                        .filter_map(|index| {
+                            search_rules.get_index_search_rules(index.deref()).map(
+                                |index_search_rules| {
+                                    (String::from(index), Some(index_search_rules))
+                                },
+                            )
+                        })
+                        .collect(),
+                ),
+                None => SearchRules::Set(key.indexes.into_iter().map(String::from).collect()),
+            };
+        } else if let Some(search_rules) = search_rules {
+            filters.search_rules = search_rules;
+        }
+
+        filters.allow_index_creation = key
+            .actions
+            .iter()
+            .any(|&action| action == Action::IndexesAdd || action == Action::All);
+
+        Ok(filters)
+    }
+
+    pub fn list_keys(&self) -> Result<Vec<Key>> {
+        self.store.list_api_keys()
+    }
+
+    pub fn delete_key(&self, uid: Uuid) -> Result<()> {
+        if self.store.delete_api_key(uid)? {
+            Ok(())
+        } else {
+            Err(AuthControllerError::ApiKeyNotFound(uid.to_string()))
+        }
+    }
+
+    pub fn get_master_key(&self) -> Option<&String> {
+        self.master_key.as_ref()
+    }
+
+    /// Generate a valid key from a key id using the current master key.
+    /// Returns None if no master key has been set.
+    pub fn generate_key(&self, uid: Uuid) -> Option<String> {
+        self.master_key
+            .as_ref()
+            .map(|master_key| generate_key_as_hexa(uid, master_key.as_bytes()))
+    }
+
+    /// Check if the provided key is authorized to make a specific action
+    /// without checking if the key is valid.
+    pub fn is_key_authorized(
+        &self,
+        uid: Uuid,
+        action: Action,
+        index: Option<&str>,
+    ) -> Result<bool> {
+        match self
+            .store
+            // check if the key has access to all indexes.
+            .get_expiration_date(uid, action, None)?
+            .or(match index {
+                // else check if the key has access to the requested index.
+                Some(index) => {
+                    self.store
+                        .get_expiration_date(uid, action, Some(index.as_bytes()))?
+                }
+                // or to any index if no index has been requested.
+                None => self.store.prefix_first_expiration_date(uid, action)?,
+            }) {
+            // check expiration date.
+            Some(Some(exp)) => Ok(OffsetDateTime::now_utc() < exp),
+            // no expiration date.
+            Some(None) => Ok(true),
+            // action or index forbidden.
+            None => Ok(false),
+        }
+    }
+}
+
+pub struct AuthFilter {
+    pub search_rules: SearchRules,
+    pub allow_index_creation: bool,
+}
+
+impl Default for AuthFilter {
+    fn default() -> Self {
+        Self {
+            search_rules: SearchRules::default(),
+            allow_index_creation: true,
+        }
+    }
+}
+
+/// Transparent wrapper around a list of allowed indexes with the search rules to apply for each.
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(untagged)]
+pub enum SearchRules {
+    Set(HashSet<String>),
+    Map(HashMap<String, Option<IndexSearchRules>>),
+}
+
+impl Default for SearchRules {
+    fn default() -> Self {
+        Self::Set(Some("*".to_string()).into_iter().collect())
+    }
+}
+
+impl SearchRules {
+    pub fn is_index_authorized(&self, index: &str) -> bool {
+        match self {
+            Self::Set(set) => set.contains("*") || set.contains(index),
+            Self::Map(map) => map.contains_key("*") || map.contains_key(index),
+        }
+    }
+
+    pub fn get_index_search_rules(&self, index: &str) -> Option<IndexSearchRules> {
+        match self {
+            Self::Set(set) => {
+                if set.contains("*") || set.contains(index) {
+                    Some(IndexSearchRules::default())
+                } else {
+                    None
+                }
+            }
+            Self::Map(map) => map
+                .get(index)
+                .or_else(|| map.get("*"))
+                .map(|isr| isr.clone().unwrap_or_default()),
+        }
+    }
+}
+
+impl IntoIterator for SearchRules {
+    type Item = (String, IndexSearchRules);
+    type IntoIter = Box<dyn Iterator<Item = Self::Item>>;
+
+    fn into_iter(self) -> Self::IntoIter {
+        match self {
+            Self::Set(array) => {
+                Box::new(array.into_iter().map(|i| (i, IndexSearchRules::default())))
+            }
+            Self::Map(map) => {
+                Box::new(map.into_iter().map(|(i, isr)| (i, isr.unwrap_or_default())))
+            }
+        }
+    }
+}
+
+/// Contains the rules to apply on the top of the search query for a specific index.
+///
+/// filter: search filter to apply in addition to query filters.
+#[derive(Debug, Serialize, Deserialize, Default, Clone)]
+pub struct IndexSearchRules {
+    pub filter: Option<serde_json::Value>,
+}
+
+fn generate_default_keys(store: &HeedAuthStore) -> Result<()> {
+    store.put_api_key(Key::default_admin())?;
+    store.put_api_key(Key::default_search())?;
+
+    Ok(())
+}

meilisearch-auth/src/store.rs

@@ -0,0 +1,319 @@
+use std::borrow::Cow;
+use std::cmp::Reverse;
+use std::collections::HashSet;
+use std::convert::TryFrom;
+use std::convert::TryInto;
+use std::fs::create_dir_all;
+use std::ops::Deref;
+use std::path::Path;
+use std::str;
+use std::sync::Arc;
+
+use enum_iterator::IntoEnumIterator;
+use hmac::{Hmac, Mac};
+use meilisearch_types::star_or::StarOr;
+use milli::heed::types::{ByteSlice, DecodeIgnore, SerdeJson};
+use milli::heed::{Database, Env, EnvOpenOptions, RwTxn};
+use sha2::Sha256;
+use time::OffsetDateTime;
+use uuid::fmt::Hyphenated;
+use uuid::Uuid;
+
+use super::error::Result;
+use super::{Action, Key};
+
+const AUTH_STORE_SIZE: usize = 1_073_741_824; //1GiB
+const AUTH_DB_PATH: &str = "auth";
+const KEY_DB_NAME: &str = "api-keys";
+const KEY_ID_ACTION_INDEX_EXPIRATION_DB_NAME: &str = "keyid-action-index-expiration";
+
+pub type KeyId = Uuid;
+
+#[derive(Clone)]
+pub struct HeedAuthStore {
+    env: Arc<Env>,
+    keys: Database<ByteSlice, SerdeJson<Key>>,
+    action_keyid_index_expiration: Database<KeyIdActionCodec, SerdeJson<Option<OffsetDateTime>>>,
+    should_close_on_drop: bool,
+}
+
+impl Drop for HeedAuthStore {
+    fn drop(&mut self) {
+        if self.should_close_on_drop && Arc::strong_count(&self.env) == 1 {
+            self.env.as_ref().clone().prepare_for_closing();
+        }
+    }
+}
+
+pub fn open_auth_store_env(path: &Path) -> milli::heed::Result<milli::heed::Env> {
+    let mut options = EnvOpenOptions::new();
+    options.map_size(AUTH_STORE_SIZE); // 1GB
+    options.max_dbs(2);
+    options.open(path)
+}
+
+impl HeedAuthStore {
+    pub fn new(path: impl AsRef<Path>) -> Result<Self> {
+        let path = path.as_ref().join(AUTH_DB_PATH);
+        create_dir_all(&path)?;
+        let env = Arc::new(open_auth_store_env(path.as_ref())?);
+        let keys = env.create_database(Some(KEY_DB_NAME))?;
+        let action_keyid_index_expiration =
+            env.create_database(Some(KEY_ID_ACTION_INDEX_EXPIRATION_DB_NAME))?;
+        Ok(Self {
+            env,
+            keys,
+            action_keyid_index_expiration,
+            should_close_on_drop: true,
+        })
+    }
+
+    pub fn set_drop_on_close(&mut self, v: bool) {
+        self.should_close_on_drop = v;
+    }
+
+    pub fn is_empty(&self) -> Result<bool> {
+        let rtxn = self.env.read_txn()?;
+
+        Ok(self.keys.len(&rtxn)? == 0)
+    }
+
+    pub fn put_api_key(&self, key: Key) -> Result<Key> {
+        let uid = key.uid;
+        let mut wtxn = self.env.write_txn()?;
+
+        self.keys.put(&mut wtxn, uid.as_bytes(), &key)?;
+
+        // delete key from inverted database before refilling it.
+        self.delete_key_from_inverted_db(&mut wtxn, &uid)?;
+        // create inverted database.
+        let db = self.action_keyid_index_expiration;
+
+        let mut actions = HashSet::new();
+        for action in &key.actions {
+            match action {
+                Action::All => actions.extend(Action::into_enum_iter()),
+                Action::DocumentsAll => {
+                    actions.extend(
+                        [
+                            Action::DocumentsGet,
+                            Action::DocumentsDelete,
+                            Action::DocumentsAdd,
+                        ]
+                        .iter(),
+                    );
+                }
+                Action::IndexesAll => {
+                    actions.extend(
+                        [
+                            Action::IndexesAdd,
+                            Action::IndexesDelete,
+                            Action::IndexesGet,
+                            Action::IndexesUpdate,
+                        ]
+                        .iter(),
+                    );
+                }
+                Action::SettingsAll => {
+                    actions.extend([Action::SettingsGet, Action::SettingsUpdate].iter());
+                }
+                Action::DumpsAll => {
+                    actions.insert(Action::DumpsCreate);
+                }
+                Action::TasksAll => {
+                    actions.insert(Action::TasksGet);
+                }
+                Action::StatsAll => {
+                    actions.insert(Action::StatsGet);
+                }
+                other => {
+                    actions.insert(*other);
+                }
+            }
+        }
+
+        let no_index_restriction = key.indexes.contains(&StarOr::Star);
+        for action in actions {
+            if no_index_restriction {
+                // If there is no index restriction we put None.
+                db.put(&mut wtxn, &(&uid, &action, None), &key.expires_at)?;
+            } else {
+                // else we create a key for each index.
+                for index in key.indexes.iter() {
+                    db.put(
+                        &mut wtxn,
+                        &(&uid, &action, Some(index.deref().as_bytes())),
+                        &key.expires_at,
+                    )?;
+                }
+            }
+        }
+
+        wtxn.commit()?;
+
+        Ok(key)
+    }
+
+    pub fn get_api_key(&self, uid: Uuid) -> Result<Option<Key>> {
+        let rtxn = self.env.read_txn()?;
+        self.keys.get(&rtxn, uid.as_bytes()).map_err(|e| e.into())
+    }
+
+    pub fn get_uid_from_encoded_key(
+        &self,
+        encoded_key: &[u8],
+        master_key: &[u8],
+    ) -> Result<Option<Uuid>> {
+        let rtxn = self.env.read_txn()?;
+        let uid = self
+            .keys
+            .remap_data_type::<DecodeIgnore>()
+            .iter(&rtxn)?
+            .filter_map(|res| match res {
+                Ok((uid, _)) => {
+                    let (uid, _) = try_split_array_at(uid)?;
+                    let uid = Uuid::from_bytes(*uid);
+                    if generate_key_as_hexa(uid, master_key).as_bytes() == encoded_key {
+                        Some(uid)
+                    } else {
+                        None
+                    }
+                }
+                Err(_) => None,
+            })
+            .next();
+
+        Ok(uid)
+    }
+
+    pub fn delete_api_key(&self, uid: Uuid) -> Result<bool> {
+        let mut wtxn = self.env.write_txn()?;
+        let existing = self.keys.delete(&mut wtxn, uid.as_bytes())?;
+        self.delete_key_from_inverted_db(&mut wtxn, &uid)?;
+        wtxn.commit()?;
+
+        Ok(existing)
+    }
+
+    pub fn list_api_keys(&self) -> Result<Vec<Key>> {
+        let mut list = Vec::new();
+        let rtxn = self.env.read_txn()?;
+        for result in self.keys.remap_key_type::<DecodeIgnore>().iter(&rtxn)? {
+            let (_, content) = result?;
+            list.push(content);
+        }
+        list.sort_unstable_by_key(|k| Reverse(k.created_at));
+        Ok(list)
+    }
+
+    pub fn get_expiration_date(
+        &self,
+        uid: Uuid,
+        action: Action,
+        index: Option<&[u8]>,
+    ) -> Result<Option<Option<OffsetDateTime>>> {
+        let rtxn = self.env.read_txn()?;
+        let tuple = (&uid, &action, index);
+        Ok(self.action_keyid_index_expiration.get(&rtxn, &tuple)?)
+    }
+
+    pub fn prefix_first_expiration_date(
+        &self,
+        uid: Uuid,
+        action: Action,
+    ) -> Result<Option<Option<OffsetDateTime>>> {
+        let rtxn = self.env.read_txn()?;
+        let tuple = (&uid, &action, None);
+        let exp = self
+            .action_keyid_index_expiration
+            .prefix_iter(&rtxn, &tuple)?
+            .next()
+            .transpose()?
+            .map(|(_, expiration)| expiration);
+
+        Ok(exp)
+    }
+
+    fn delete_key_from_inverted_db(&self, wtxn: &mut RwTxn, key: &KeyId) -> Result<()> {
+        let mut iter = self
+            .action_keyid_index_expiration
+            .remap_types::<ByteSlice, DecodeIgnore>()
+            .prefix_iter_mut(wtxn, key.as_bytes())?;
+        while iter.next().transpose()?.is_some() {
+            // safety: we don't keep references from inside the LMDB database.
+            unsafe { iter.del_current()? };
+        }
+
+        Ok(())
+    }
+}
+
+/// Codec allowing to retrieve the expiration date of an action,
+/// optionally on a specific index, for a given key.
+pub struct KeyIdActionCodec;
+
+impl<'a> milli::heed::BytesDecode<'a> for KeyIdActionCodec {
+    type DItem = (KeyId, Action, Option<&'a [u8]>);
+
+    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
+        let (key_id_bytes, action_bytes) = try_split_array_at(bytes)?;
+        let (action_bytes, index) = match try_split_array_at(action_bytes)? {
+            (action, []) => (action, None),
+            (action, index) => (action, Some(index)),
+        };
+        let key_id = Uuid::from_bytes(*key_id_bytes);
+        let action = Action::from_repr(u8::from_be_bytes(*action_bytes))?;
+
+        Some((key_id, action, index))
+    }
+}
+
+impl<'a> milli::heed::BytesEncode<'a> for KeyIdActionCodec {
+    type EItem = (&'a KeyId, &'a Action, Option<&'a [u8]>);
+
+    fn bytes_encode((key_id, action, index): &Self::EItem) -> Option<Cow<[u8]>> {
+        let mut bytes = Vec::new();
+
+        bytes.extend_from_slice(key_id.as_bytes());
+        let action_bytes = u8::to_be_bytes(action.repr());
+        bytes.extend_from_slice(&action_bytes);
+        if let Some(index) = index {
+            bytes.extend_from_slice(index);
+        }
+
+        Some(Cow::Owned(bytes))
+    }
+}
+
+pub fn generate_key_as_hexa(uid: Uuid, master_key: &[u8]) -> String {
+    // format uid as hyphenated allowing user to generate their own keys.
+    let mut uid_buffer = [0; Hyphenated::LENGTH];
+    let uid = uid.hyphenated().encode_lower(&mut uid_buffer);
+
+    // new_from_slice function never fail.
+    let mut mac = Hmac::<Sha256>::new_from_slice(master_key).unwrap();
+    mac.update(uid.as_bytes());
+
+    let result = mac.finalize();
+    format!("{:x}", result.into_bytes())
+}
+
+/// Divides one slice into two at an index, returns `None` if mid is out of bounds.
+pub fn try_split_at<T>(slice: &[T], mid: usize) -> Option<(&[T], &[T])> {
+    if mid <= slice.len() {
+        Some(slice.split_at(mid))
+    } else {
+        None
+    }
+}
+
+/// Divides one slice into an array and the tail at an index,
+/// returns `None` if `N` is out of bounds.
+pub fn try_split_array_at<T, const N: usize>(slice: &[T]) -> Option<(&[T; N], &[T])>
+where
+    [T; N]: for<'a> TryFrom<&'a [T]>,
+{
+    let (head, tail) = try_split_at(slice, N)?;
+    let head = head.try_into().ok()?;
+    Some((head, tail))
+}

meilisearch-error/Cargo.toml

@@ -1,8 +0,0 @@
-[package]
-name = "meilisearch-error"
-version = "0.22.0"
-authors = ["marin <postma.marin@protonmail.com>"]
-edition = "2018"
-
-[dependencies]
-actix-http = "=3.0.0-beta.6"

meilisearch-error/src/lib.rs

@@ -1,203 +0,0 @@
-use std::fmt;
-
-use actix_http::http::StatusCode;
-
-pub trait ErrorCode: std::error::Error {
-    fn error_code(&self) -> Code;
-
-    /// returns the HTTP status code ascociated with the error
-    fn http_status(&self) -> StatusCode {
-        self.error_code().http()
-    }
-
-    /// returns the doc url ascociated with the error
-    fn error_url(&self) -> String {
-        self.error_code().url()
-    }
-
-    /// returns error name, used as error code
-    fn error_name(&self) -> String {
-        self.error_code().name()
-    }
-
-    /// return the error type
-    fn error_type(&self) -> String {
-        self.error_code().type_()
-    }
-}
-
-#[allow(clippy::enum_variant_names)]
-enum ErrorType {
-    InternalError,
-    InvalidRequestError,
-    AuthenticationError,
-}
-
-impl fmt::Display for ErrorType {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        use ErrorType::*;
-
-        match self {
-            InternalError => write!(f, "internal_error"),
-            InvalidRequestError => write!(f, "invalid_request_error"),
-            AuthenticationError => write!(f, "authentication_error"),
-        }
-    }
-}
-
-pub enum Code {
-    // index related error
-    CreateIndex,
-    IndexAlreadyExists,
-    IndexNotFound,
-    InvalidIndexUid,
-    OpenIndex,
-
-    // invalid state error
-    InvalidState,
-    MissingPrimaryKey,
-    PrimaryKeyAlreadyPresent,
-
-    MaxFieldsLimitExceeded,
-    MissingDocumentId,
-
-    Facet,
-    Filter,
-    Sort,
-
-    BadParameter,
-    BadRequest,
-    DocumentNotFound,
-    Internal,
-    InvalidToken,
-    MissingAuthorizationHeader,
-    NotFound,
-    PayloadTooLarge,
-    RetrieveDocument,
-    SearchDocuments,
-    UnsupportedMediaType,
-
-    DumpAlreadyInProgress,
-    DumpProcessFailed,
-}
-
-impl Code {
-    /// ascociate a `Code` variant to the actual ErrCode
-    fn err_code(&self) -> ErrCode {
-        use Code::*;
-
-        match self {
-            // index related errors
-            // create index is thrown on internal error while creating an index.
-            CreateIndex => ErrCode::internal("index_creation_failed", StatusCode::BAD_REQUEST),
-            IndexAlreadyExists => ErrCode::invalid("index_already_exists", StatusCode::BAD_REQUEST),
-            // thrown when requesting an unexisting index
-            IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND),
-            InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST),
-            OpenIndex => {
-                ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR)
-            }
-
-            // invalid state error
-            InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR),
-            // thrown when no primary key has been set
-            MissingPrimaryKey => ErrCode::invalid("missing_primary_key", StatusCode::BAD_REQUEST),
-            // error thrown when trying to set an already existing primary key
-            PrimaryKeyAlreadyPresent => {
-                ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST)
-            }
-
-            // invalid document
-            MaxFieldsLimitExceeded => {
-                ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST)
-            }
-            MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST),
-
-            // error related to facets
-            Facet => ErrCode::invalid("invalid_facet", StatusCode::BAD_REQUEST),
-            // error related to filters
-            Filter => ErrCode::invalid("invalid_filter", StatusCode::BAD_REQUEST),
-            // error related to sorts
-            Sort => ErrCode::invalid("invalid_sort", StatusCode::BAD_REQUEST),
-
-            BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST),
-            BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST),
-            DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND),
-            Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR),
-            InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN),
-            MissingAuthorizationHeader => {
-                ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED)
-            }
-            NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND),
-            PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE),
-            RetrieveDocument => {
-                ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST)
-            }
-            SearchDocuments => ErrCode::internal("search_error", StatusCode::BAD_REQUEST),
-            UnsupportedMediaType => {
-                ErrCode::invalid("unsupported_media_type", StatusCode::UNSUPPORTED_MEDIA_TYPE)
-            }
-
-            // error related to dump
-            DumpAlreadyInProgress => {
-                ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT)
-            }
-            DumpProcessFailed => {
-                ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR)
-            }
-        }
-    }
-
-    /// return the HTTP status code ascociated with the `Code`
-    fn http(&self) -> StatusCode {
-        self.err_code().status_code
-    }
-
-    /// return error name, used as error code
-    fn name(&self) -> String {
-        self.err_code().error_name.to_string()
-    }
-
-    /// return the error type
-    fn type_(&self) -> String {
-        self.err_code().error_type.to_string()
-    }
-
-    /// return the doc url ascociated with the error
-    fn url(&self) -> String {
-        format!("https://docs.meilisearch.com/errors#{}", self.name())
-    }
-}
-
-/// Internal structure providing a convenient way to create error codes
-struct ErrCode {
-    status_code: StatusCode,
-    error_type: ErrorType,
-    error_name: &'static str,
-}
-
-impl ErrCode {
-    fn authentication(error_name: &'static str, status_code: StatusCode) -> ErrCode {
-        ErrCode {
-            status_code,
-            error_name,
-            error_type: ErrorType::AuthenticationError,
-        }
-    }
-
-    fn internal(error_name: &'static str, status_code: StatusCode) -> ErrCode {
-        ErrCode {
-            status_code,
-            error_name,
-            error_type: ErrorType::InternalError,
-        }
-    }
-
-    fn invalid(error_name: &'static str, status_code: StatusCode) -> ErrCode {
-        ErrCode {
-            status_code,
-            error_name,
-            error_type: ErrorType::InvalidRequestError,
-        }
-    }
-}

meilisearch-http/Cargo.toml

@@ -1,95 +1,97 @@
 [package]
 authors = ["Quentin de Quelen <quentin@dequelen.me>", "Clément Renault <clement@meilisearch.com>"]
-description = "MeiliSearch HTTP server"
-edition = "2018"
+description = "Meilisearch HTTP server"
+edition = "2021"
 license = "MIT"
 name = "meilisearch-http"
-version = "0.22.0"
+version = "0.28.1"
 
 [[bin]]
 name = "meilisearch"
 path = "src/main.rs"
 
 [build-dependencies]
-actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true }
-anyhow = { version = "*", optional = true }
-cargo_toml = { version = "0.9.0", optional = true }
+anyhow = { version = "1.0.56", optional = true }
+cargo_toml = { version = "0.11.4", optional = true }
 hex = { version = "0.4.3", optional = true }
-reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false, optional = true }
-sha-1 = { version = "0.9.4", optional = true }
-tempfile = { version = "3.1.0", optional = true }
-vergen = { version = "5.1.15", default-features = false, features = ["git"] }
-zip = { version = "0.5.12", optional = true }
+reqwest = { version = "0.11.9", features = ["blocking", "rustls-tls"], default-features = false, optional = true }
+sha-1 = { version = "0.10.0", optional = true }
+static-files = { version = "0.2.3", optional = true }
+tempfile = { version = "3.3.0", optional = true }
+vergen = { version = "7.0.0", default-features = false, features = ["git"] }
+zip = { version = "0.5.13", optional = true }
 
 [dependencies]
-actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "2dac1a4"}
-actix-http = { version = "=3.0.0-beta.6" }
-actix-service = "2.0.0"
-actix-web = { version = "=4.0.0-beta.6", features = ["rustls"] }
-actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true }
-anyhow = "1.0.36"
-async-stream = "0.3.0"
-async-trait = "0.1.42"
-arc-swap = "1.2.0"
-byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
-bytes = "0.6.0"
-chrono = { version = "0.4.19", features = ["serde"] }
-crossbeam-channel = "0.5.0"
+actix-cors = "0.6.1"
+actix-web = { version = "4.0.1", default-features = false, features = ["macros", "compress-brotli", "compress-gzip", "cookies", "rustls"] }
+actix-web-static-files = { git = "https://github.com/kilork/actix-web-static-files.git", rev = "2d3b6160", optional = true }
+anyhow = { version = "1.0.56", features = ["backtrace"] }
+async-stream = "0.3.3"
+async-trait = "0.1.52"
+bstr = "0.2.17"
+byte-unit = { version = "4.0.14", default-features = false, features = ["std", "serde"] }
+bytes = "1.1.0"
+clap = { version = "3.1.6", features = ["derive", "env"] }
+crossbeam-channel = "0.5.2"
 either = "1.6.1"
-env_logger = "0.8.2"
-flate2 = "1.0.19"
-fst = "0.4.5"
-futures = "0.3.7"
-futures-util = "0.3.8"
-heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
-http = "0.2.1"
-indexmap = { version = "1.3.2", features = ["serde-1"] }
-itertools = "0.10.0"
-log = "0.4.8"
-main_error = "0.1.0"
-meilisearch-error = { path = "../meilisearch-error" }
-meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" }
-memmap = "0.7.0"
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.13.1" }
+env_logger = "0.9.0"
+flate2 = "1.0.22"
+fst = "0.4.7"
+futures = "0.3.21"
+futures-util = "0.3.21"
+http = "0.2.6"
+indexmap = { version = "1.8.0", features = ["serde-1"] }
+itertools = "0.10.3"
+jsonwebtoken = "8.0.1"
+log = "0.4.14"
+meilisearch-auth = { path = "../meilisearch-auth" }
+meilisearch-types = { path = "../meilisearch-types" }
+meilisearch-lib = { path = "../meilisearch-lib" }
 mime = "0.3.16"
-num_cpus = "1.13.0"
-once_cell = "1.5.2"
-parking_lot = "0.11.1"
-rand = "0.7.3"
-rayon = "1.5.0"
-regex = "1.4.2"
-rustls = "0.19"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = { version = "1.0.59", features = ["preserve_order"] }
-sha2 = "0.9.1"
-siphasher = "0.3.2"
-slice-group-by = "0.2.6"
-structopt = "0.3.20"
-tar = "0.4.29"
-tempfile = "3.1.0"
-thiserror = "1.0.24"
-tokio = { version = "1", features = ["full"] }
-uuid = { version =  "0.8.2", features = ["serde"] }
-walkdir = "2.3.2"
+num_cpus = "1.13.1"
 obkv = "0.2.0"
-pin-project = "1.0.7"
-whoami = { version = "1.1.2", optional = true }
-reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true }
-serdeval = "0.1.0"
-sysinfo = "0.20.0"
+once_cell = "1.10.0"
+parking_lot = "0.12.0"
+pin-project-lite = "0.2.8"
+platform-dirs = "0.3.0"
+rand = "0.8.5"
+rayon = "1.5.1"
+regex = "1.5.5"
+reqwest = { version = "0.11.4", features = ["rustls-tls", "json"], default-features = false }
+rustls = "0.20.4"
+rustls-pemfile = "0.3.0"
+segment = { version = "0.2.0", optional = true }
+serde = { version = "1.0.136", features = ["derive"] }
+serde-cs = "0.2.3"
+serde_json = { version = "1.0.79", features = ["preserve_order"] }
+sha2 = "0.10.2"
+siphasher = "0.3.10"
+slice-group-by = "0.3.0"
+static-files = { version = "0.2.3", optional = true }
+sysinfo = "0.23.5"
+tar = "0.4.38"
+tempfile = "3.3.0"
+thiserror = "1.0.30"
+time = { version = "0.3.7", features = ["serde-well-known", "formatting", "parsing", "macros"] }
+tokio = { version = "1.17.0", features = ["full"] }
+tokio-stream = "0.1.8"
+uuid = { version = "1.1.2", features = ["serde", "v4"] }
+walkdir = "2.3.2"
 
 [dev-dependencies]
-actix-rt = "2.1.0"
-assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" }
-mockall = "0.9.1"
-paste = "1.0.5"
-serde_url_params = "0.2.1"
-tempdir = "0.3.7"
-urlencoding = "1.1.1"
+actix-rt = "2.7.0"
+assert-json-diff = "2.0.1"
+manifest-dir-macros = "0.1.14"
+maplit = "1.0.2"
+urlencoding = "2.1.0"
+yaup = "0.2.0"
 
 [features]
+default = ["analytics", "mini-dashboard"]
+analytics = ["segment"]
 mini-dashboard = [
     "actix-web-static-files",
+    "static-files",
     "anyhow",
     "cargo_toml",
     "hex",
@@ -98,12 +100,10 @@ mini-dashboard = [
     "tempfile",
     "zip",
 ]
-analytics = ["whoami", "reqwest"]
-default = ["analytics", "mini-dashboard"]
 
 [target.'cfg(target_os = "linux")'.dependencies]
-jemallocator = "0.3.2"
+tikv-jemallocator = "0.4.3"
 
 [package.metadata.mini-dashboard]
-assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.4/build.zip"
-sha1 = "750e8a8e56cfa61fbf9ead14b08a5f17ad3f3d37"
+assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.2.1/build.zip"
+sha1 = "05a02ff13c3982091884a3f81d28bf53e72607b2"

meilisearch-http/build.rs

@@ -16,11 +16,11 @@ mod mini_dashboard {
     use std::io::{Cursor, Read, Write};
     use std::path::PathBuf;
 
-    use actix_web_static_files::resource_dir;
     use anyhow::Context;
     use cargo_toml::Manifest;
     use reqwest::blocking::get;
     use sha1::{Digest, Sha1};
+    use static_files::resource_dir;
 
     pub fn setup_mini_dashboard() -> anyhow::Result<()> {
         let cargo_manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap());

meilisearch-http/src/analytics.rs

@@ -1,126 +0,0 @@
-use std::hash::{Hash, Hasher};
-use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
-
-use log::debug;
-use serde::Serialize;
-use siphasher::sip::SipHasher;
-
-use crate::Data;
-use crate::Opt;
-
-const AMPLITUDE_API_KEY: &str = "f7fba398780e06d8fe6666a9be7e3d47";
-
-#[derive(Debug, Serialize)]
-struct EventProperties {
-    database_size: u64,
-    last_update_timestamp: Option<i64>, //timestamp
-    number_of_documents: Vec<u64>,
-}
-
-impl EventProperties {
-    async fn from(data: Data) -> anyhow::Result<EventProperties> {
-        let stats = data.index_controller.get_all_stats().await?;
-
-        let database_size = stats.database_size;
-        let last_update_timestamp = stats.last_update.map(|u| u.timestamp());
-        let number_of_documents = stats
-            .indexes
-            .values()
-            .map(|index| index.number_of_documents)
-            .collect();
-
-        Ok(EventProperties {
-            database_size,
-            last_update_timestamp,
-            number_of_documents,
-        })
-    }
-}
-
-#[derive(Debug, Serialize)]
-struct UserProperties<'a> {
-    env: &'a str,
-    start_since_days: u64,
-    user_email: Option<String>,
-    server_provider: Option<String>,
-}
-
-#[derive(Debug, Serialize)]
-struct Event<'a> {
-    user_id: &'a str,
-    event_type: &'a str,
-    device_id: &'a str,
-    time: u64,
-    app_version: &'a str,
-    user_properties: UserProperties<'a>,
-    event_properties: Option<EventProperties>,
-}
-
-#[derive(Debug, Serialize)]
-struct AmplitudeRequest<'a> {
-    api_key: &'a str,
-    events: Vec<Event<'a>>,
-}
-
-pub async fn analytics_sender(data: Data, opt: Opt) {
-    let username = whoami::username();
-    let hostname = whoami::hostname();
-    let platform = whoami::platform();
-
-    let uid = username + &hostname + &platform.to_string();
-
-    let mut hasher = SipHasher::new();
-    uid.hash(&mut hasher);
-    let hash = hasher.finish();
-
-    let uid = format!("{:X}", hash);
-    let platform = platform.to_string();
-    let first_start = Instant::now();
-
-    loop {
-        let n = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
-        let user_id = &uid;
-        let device_id = &platform;
-        let time = n.as_secs();
-        let event_type = "runtime_tick";
-        let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day
-        let event_properties = EventProperties::from(data.clone()).await.ok();
-        let app_version = env!("CARGO_PKG_VERSION").to_string();
-        let app_version = app_version.as_str();
-        let user_email = std::env::var("MEILI_USER_EMAIL").ok();
-        let server_provider = std::env::var("MEILI_SERVER_PROVIDER").ok();
-        let user_properties = UserProperties {
-            env: &opt.env,
-            start_since_days: elapsed_since_start,
-            user_email,
-            server_provider,
-        };
-
-        let event = Event {
-            user_id,
-            event_type,
-            device_id,
-            time,
-            app_version,
-            user_properties,
-            event_properties,
-        };
-
-        let request = AmplitudeRequest {
-            api_key: AMPLITUDE_API_KEY,
-            events: vec![event],
-        };
-
-        let response = reqwest::Client::new()
-            .post("https://api2.amplitude.com/2/httpapi")
-            .timeout(Duration::from_secs(60)) // 1 minute max
-            .json(&request)
-            .send()
-            .await;
-        if let Err(e) = response {
-            debug!("Unsuccessful call to Amplitude: {}", e);
-        }
-
-        tokio::time::sleep(Duration::from_secs(3600)).await;
-    }
-}

meilisearch-http/src/analytics/mock_analytics.rs

@@ -0,0 +1,51 @@
+use std::{any::Any, sync::Arc};
+
+use actix_web::HttpRequest;
+use serde_json::Value;
+
+use crate::{routes::indexes::documents::UpdateDocumentsQuery, Opt};
+
+use super::{find_user_id, Analytics};
+
+pub struct MockAnalytics;
+
+#[derive(Default)]
+pub struct SearchAggregator {}
+
+#[allow(dead_code)]
+impl SearchAggregator {
+    pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self {
+        Self::default()
+    }
+
+    pub fn succeed(&mut self, _: &dyn Any) {}
+}
+
+impl MockAnalytics {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(opt: &Opt) -> (Arc<dyn Analytics>, String) {
+        let user = find_user_id(&opt.db_path).unwrap_or_default();
+        (Arc::new(Self), user)
+    }
+}
+
+impl Analytics for MockAnalytics {
+    // These methods are noop and should be optimized out
+    fn publish(&self, _event_name: String, _send: Value, _request: Option<&HttpRequest>) {}
+    fn get_search(&self, _aggregate: super::SearchAggregator) {}
+    fn post_search(&self, _aggregate: super::SearchAggregator) {}
+    fn add_documents(
+        &self,
+        _documents_query: &UpdateDocumentsQuery,
+        _index_creation: bool,
+        _request: &HttpRequest,
+    ) {
+    }
+    fn update_documents(
+        &self,
+        _documents_query: &UpdateDocumentsQuery,
+        _index_creation: bool,
+        _request: &HttpRequest,
+    ) {
+    }
+}

meilisearch-http/src/analytics/mod.rs

@@ -0,0 +1,84 @@
+mod mock_analytics;
+// if we are in release mode and the feature analytics was enabled
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+mod segment_analytics;
+
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use actix_web::HttpRequest;
+use once_cell::sync::Lazy;
+use platform_dirs::AppDirs;
+use serde_json::Value;
+
+use crate::routes::indexes::documents::UpdateDocumentsQuery;
+
+pub use mock_analytics::MockAnalytics;
+
+// if we are in debug mode OR the analytics feature is disabled
+// the `SegmentAnalytics` point to the mock instead of the real analytics
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
+pub type SegmentAnalytics = mock_analytics::MockAnalytics;
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
+pub type SearchAggregator = mock_analytics::SearchAggregator;
+
+// if we are in release mode and the feature analytics was enabled
+// we use the real analytics
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+pub type SegmentAnalytics = segment_analytics::SegmentAnalytics;
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+pub type SearchAggregator = segment_analytics::SearchAggregator;
+
+/// The Meilisearch config dir:
+/// `~/.config/Meilisearch` on *NIX or *BSD.
+/// `~/Library/ApplicationSupport` on macOS.
+/// `%APPDATA` (= `C:\Users%USERNAME%\AppData\Roaming`) on windows.
+static MEILISEARCH_CONFIG_PATH: Lazy<Option<PathBuf>> =
+    Lazy::new(|| AppDirs::new(Some("Meilisearch"), false).map(|appdir| appdir.config_dir));
+
+fn config_user_id_path(db_path: &Path) -> Option<PathBuf> {
+    db_path
+        .canonicalize()
+        .ok()
+        .map(|path| {
+            path.join("instance-uid")
+                .display()
+                .to_string()
+                .replace('/', "-")
+        })
+        .zip(MEILISEARCH_CONFIG_PATH.as_ref())
+        .map(|(filename, config_path)| config_path.join(filename.trim_start_matches('-')))
+}
+
+/// Look for the instance-uid in the `data.ms` or in `~/.config/Meilisearch/path-to-db-instance-uid`
+fn find_user_id(db_path: &Path) -> Option<String> {
+    fs::read_to_string(db_path.join("instance-uid"))
+        .ok()
+        .or_else(|| fs::read_to_string(&config_user_id_path(db_path)?).ok())
+}
+
+pub trait Analytics: Sync + Send {
+    /// The method used to publish most analytics that do not need to be batched every hours
+    fn publish(&self, event_name: String, send: Value, request: Option<&HttpRequest>);
+
+    /// This method should be called to aggregate a get search
+    fn get_search(&self, aggregate: SearchAggregator);
+
+    /// This method should be called to aggregate a post search
+    fn post_search(&self, aggregate: SearchAggregator);
+
+    // this method should be called to aggregate a add documents request
+    fn add_documents(
+        &self,
+        documents_query: &UpdateDocumentsQuery,
+        index_creation: bool,
+        request: &HttpRequest,
+    );
+    // this method should be called to batch a update documents request
+    fn update_documents(
+        &self,
+        documents_query: &UpdateDocumentsQuery,
+        index_creation: bool,
+        request: &HttpRequest,
+    );
+}

meilisearch-http/src/analytics/segment_analytics.rs

@@ -0,0 +1,626 @@
+use std::collections::{BinaryHeap, HashMap, HashSet};
+use std::fs;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use actix_web::http::header::USER_AGENT;
+use actix_web::HttpRequest;
+use http::header::CONTENT_TYPE;
+use meilisearch_auth::SearchRules;
+use meilisearch_lib::index::{
+    SearchQuery, SearchResult, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER,
+    DEFAULT_HIGHLIGHT_POST_TAG, DEFAULT_HIGHLIGHT_PRE_TAG,
+};
+use meilisearch_lib::index_controller::Stats;
+use meilisearch_lib::MeiliSearch;
+use once_cell::sync::Lazy;
+use regex::Regex;
+use segment::message::{Identify, Track, User};
+use segment::{AutoBatcher, Batcher, HttpClient};
+use serde_json::{json, Value};
+use sysinfo::{DiskExt, System, SystemExt};
+use time::OffsetDateTime;
+use tokio::select;
+use tokio::sync::mpsc::{self, Receiver, Sender};
+use uuid::Uuid;
+
+use crate::analytics::Analytics;
+use crate::routes::indexes::documents::UpdateDocumentsQuery;
+use crate::Opt;
+
+use super::{config_user_id_path, MEILISEARCH_CONFIG_PATH};
+
+const ANALYTICS_HEADER: &str = "X-Meilisearch-Client";
+
+/// Write the instance-uid in the `data.ms` and in `~/.config/MeiliSearch/path-to-db-instance-uid`. Ignore the errors.
+fn write_user_id(db_path: &Path, user_id: &str) {
+    let _ = fs::write(db_path.join("instance-uid"), user_id.as_bytes());
+    if let Some((meilisearch_config_path, user_id_path)) = MEILISEARCH_CONFIG_PATH
+        .as_ref()
+        .zip(config_user_id_path(db_path))
+    {
+        let _ = fs::create_dir_all(&meilisearch_config_path);
+        let _ = fs::write(user_id_path, user_id.as_bytes());
+    }
+}
+
+const SEGMENT_API_KEY: &str = "P3FWhhEsJiEDCuEHpmcN9DHcK4hVfBvb";
+
+pub fn extract_user_agents(request: &HttpRequest) -> Vec<String> {
+    request
+        .headers()
+        .get(ANALYTICS_HEADER)
+        .or_else(|| request.headers().get(USER_AGENT))
+        .map(|header| header.to_str().ok())
+        .flatten()
+        .unwrap_or("unknown")
+        .split(';')
+        .map(str::trim)
+        .map(ToString::to_string)
+        .collect()
+}
+
+pub enum AnalyticsMsg {
+    BatchMessage(Track),
+    AggregateGetSearch(SearchAggregator),
+    AggregatePostSearch(SearchAggregator),
+    AggregateAddDocuments(DocumentsAggregator),
+    AggregateUpdateDocuments(DocumentsAggregator),
+}
+
+pub struct SegmentAnalytics {
+    sender: Sender<AnalyticsMsg>,
+    user: User,
+}
+
+impl SegmentAnalytics {
+    pub async fn new(opt: &Opt, meilisearch: &MeiliSearch) -> (Arc<dyn Analytics>, String) {
+        let user_id = super::find_user_id(&opt.db_path);
+        let first_time_run = user_id.is_none();
+        let user_id = user_id.unwrap_or_else(|| Uuid::new_v4().to_string());
+        write_user_id(&opt.db_path, &user_id);
+
+        let client = reqwest::Client::builder()
+            .connect_timeout(Duration::from_secs(10))
+            .build();
+
+        // if reqwest throws an error we won't be able to send analytics
+        if client.is_err() {
+            return super::MockAnalytics::new(opt);
+        }
+
+        let client = HttpClient::new(
+            client.unwrap(),
+            "https://telemetry.meilisearch.com".to_string(),
+        );
+        let user = User::UserId { user_id };
+        let mut batcher = AutoBatcher::new(client, Batcher::new(None), SEGMENT_API_KEY.to_string());
+
+        // If Meilisearch is Launched for the first time:
+        // 1. Send an event Launched associated to the user `total_launch`.
+        // 2. Batch an event Launched with the real instance-id and send it in one hour.
+        if first_time_run {
+            let _ = batcher
+                .push(Track {
+                    user: User::UserId {
+                        user_id: "total_launch".to_string(),
+                    },
+                    event: "Launched".to_string(),
+                    ..Default::default()
+                })
+                .await;
+            let _ = batcher.flush().await;
+            let _ = batcher
+                .push(Track {
+                    user: user.clone(),
+                    event: "Launched".to_string(),
+                    ..Default::default()
+                })
+                .await;
+        }
+
+        let (sender, inbox) = mpsc::channel(100); // How many analytics can we bufferize
+
+        let segment = Box::new(Segment {
+            inbox,
+            user: user.clone(),
+            opt: opt.clone(),
+            batcher,
+            post_search_aggregator: SearchAggregator::default(),
+            get_search_aggregator: SearchAggregator::default(),
+            add_documents_aggregator: DocumentsAggregator::default(),
+            update_documents_aggregator: DocumentsAggregator::default(),
+        });
+        tokio::spawn(segment.run(meilisearch.clone()));
+
+        let this = Self {
+            sender,
+            user: user.clone(),
+        };
+
+        (Arc::new(this), user.to_string())
+    }
+}
+
+impl super::Analytics for SegmentAnalytics {
+    fn publish(&self, event_name: String, mut send: Value, request: Option<&HttpRequest>) {
+        let user_agent = request.map(|req| extract_user_agents(req));
+
+        send["user-agent"] = json!(user_agent);
+        let event = Track {
+            user: self.user.clone(),
+            event: event_name.clone(),
+            properties: send,
+            ..Default::default()
+        };
+        let _ = self
+            .sender
+            .try_send(AnalyticsMsg::BatchMessage(event.into()));
+    }
+
+    fn get_search(&self, aggregate: SearchAggregator) {
+        let _ = self
+            .sender
+            .try_send(AnalyticsMsg::AggregateGetSearch(aggregate));
+    }
+
+    fn post_search(&self, aggregate: SearchAggregator) {
+        let _ = self
+            .sender
+            .try_send(AnalyticsMsg::AggregatePostSearch(aggregate));
+    }
+
+    fn add_documents(
+        &self,
+        documents_query: &UpdateDocumentsQuery,
+        index_creation: bool,
+        request: &HttpRequest,
+    ) {
+        let aggregate = DocumentsAggregator::from_query(documents_query, index_creation, request);
+        let _ = self
+            .sender
+            .try_send(AnalyticsMsg::AggregateAddDocuments(aggregate));
+    }
+
+    fn update_documents(
+        &self,
+        documents_query: &UpdateDocumentsQuery,
+        index_creation: bool,
+        request: &HttpRequest,
+    ) {
+        let aggregate = DocumentsAggregator::from_query(documents_query, index_creation, request);
+        let _ = self
+            .sender
+            .try_send(AnalyticsMsg::AggregateUpdateDocuments(aggregate));
+    }
+}
+
+pub struct Segment {
+    inbox: Receiver<AnalyticsMsg>,
+    user: User,
+    opt: Opt,
+    batcher: AutoBatcher,
+    get_search_aggregator: SearchAggregator,
+    post_search_aggregator: SearchAggregator,
+    add_documents_aggregator: DocumentsAggregator,
+    update_documents_aggregator: DocumentsAggregator,
+}
+
+impl Segment {
+    fn compute_traits(opt: &Opt, stats: Stats) -> Value {
+        static FIRST_START_TIMESTAMP: Lazy<Instant> = Lazy::new(Instant::now);
+        static SYSTEM: Lazy<Value> = Lazy::new(|| {
+            let mut sys = System::new_all();
+            sys.refresh_all();
+            let kernel_version = sys
+                .kernel_version()
+                .map(|k| k.split_once("-").map(|(k, _)| k.to_string()))
+                .flatten();
+            json!({
+                    "distribution": sys.name(),
+                    "kernel_version": kernel_version,
+                    "cores": sys.processors().len(),
+                    "ram_size": sys.total_memory(),
+                    "disk_size": sys.disks().iter().map(|disk| disk.total_space()).max(),
+                    "server_provider": std::env::var("MEILI_SERVER_PROVIDER").ok(),
+            })
+        });
+        // The infos are all cli option except every option containing sensitive information.
+        // We consider an information as sensible if it contains a path, an address or a key.
+        let infos = {
+            // First we see if any sensitive fields were used.
+            let db_path = opt.db_path != PathBuf::from("./data.ms");
+            let import_dump = opt.import_dump.is_some();
+            let dumps_dir = opt.dumps_dir != PathBuf::from("dumps/");
+            let import_snapshot = opt.import_snapshot.is_some();
+            let snapshots_dir = opt.snapshot_dir != PathBuf::from("snapshots/");
+            let http_addr = opt.http_addr != "127.0.0.1:7700";
+
+            let mut infos = serde_json::to_value(opt).unwrap();
+
+            // Then we overwrite all sensitive field with a boolean representing if
+            // the feature was used or not.
+            infos["db_path"] = json!(db_path);
+            infos["import_dump"] = json!(import_dump);
+            infos["dumps_dir"] = json!(dumps_dir);
+            infos["import_snapshot"] = json!(import_snapshot);
+            infos["snapshot_dir"] = json!(snapshots_dir);
+            infos["http_addr"] = json!(http_addr);
+
+            infos
+        };
+
+        let number_of_documents = stats
+            .indexes
+            .values()
+            .map(|index| index.number_of_documents)
+            .collect::<Vec<u64>>();
+
+        json!({
+            "start_since_days": FIRST_START_TIMESTAMP.elapsed().as_secs() / (60 * 60 * 24), // one day
+            "system": *SYSTEM,
+            "stats": {
+                "database_size": stats.database_size,
+                "indexes_number": stats.indexes.len(),
+                "documents_number": number_of_documents,
+            },
+            "infos": infos,
+        })
+    }
+
+    async fn run(mut self, meilisearch: MeiliSearch) {
+        const INTERVAL: Duration = Duration::from_secs(60 * 60); // one hour
+                                                                 // The first batch must be sent after one hour.
+        let mut interval =
+            tokio::time::interval_at(tokio::time::Instant::now() + INTERVAL, INTERVAL);
+
+        loop {
+            select! {
+                _ = interval.tick() => {
+                    self.tick(meilisearch.clone()).await;
+                },
+                msg = self.inbox.recv() => {
+                    match msg {
+                        Some(AnalyticsMsg::BatchMessage(msg)) => drop(self.batcher.push(msg).await),
+                        Some(AnalyticsMsg::AggregateGetSearch(agreg)) => self.get_search_aggregator.aggregate(agreg),
+                        Some(AnalyticsMsg::AggregatePostSearch(agreg)) => self.post_search_aggregator.aggregate(agreg),
+                        Some(AnalyticsMsg::AggregateAddDocuments(agreg)) => self.add_documents_aggregator.aggregate(agreg),
+                        Some(AnalyticsMsg::AggregateUpdateDocuments(agreg)) => self.update_documents_aggregator.aggregate(agreg),
+                        None => (),
+                    }
+                }
+            }
+        }
+    }
+
+    async fn tick(&mut self, meilisearch: MeiliSearch) {
+        if let Ok(stats) = meilisearch.get_all_stats(&SearchRules::default()).await {
+            let _ = self
+                .batcher
+                .push(Identify {
+                    context: Some(json!({
+                        "app": {
+                            "version": env!("CARGO_PKG_VERSION").to_string(),
+                        },
+                    })),
+                    user: self.user.clone(),
+                    traits: Self::compute_traits(&self.opt, stats),
+                    ..Default::default()
+                })
+                .await;
+        }
+        let get_search = std::mem::take(&mut self.get_search_aggregator)
+            .into_event(&self.user, "Documents Searched GET");
+        let post_search = std::mem::take(&mut self.post_search_aggregator)
+            .into_event(&self.user, "Documents Searched POST");
+        let add_documents = std::mem::take(&mut self.add_documents_aggregator)
+            .into_event(&self.user, "Documents Added");
+        let update_documents = std::mem::take(&mut self.update_documents_aggregator)
+            .into_event(&self.user, "Documents Updated");
+
+        if let Some(get_search) = get_search {
+            let _ = self.batcher.push(get_search).await;
+        }
+        if let Some(post_search) = post_search {
+            let _ = self.batcher.push(post_search).await;
+        }
+        if let Some(add_documents) = add_documents {
+            let _ = self.batcher.push(add_documents).await;
+        }
+        if let Some(update_documents) = update_documents {
+            let _ = self.batcher.push(update_documents).await;
+        }
+        let _ = self.batcher.flush().await;
+    }
+}
+
+#[derive(Default)]
+pub struct SearchAggregator {
+    timestamp: Option<OffsetDateTime>,
+
+    // context
+    user_agents: HashSet<String>,
+
+    // requests
+    total_received: usize,
+    total_succeeded: usize,
+    time_spent: BinaryHeap<usize>,
+
+    // sort
+    sort_with_geo_point: bool,
+    // everytime a request has a filter, this field must be incremented by the number of terms it contains
+    sort_sum_of_criteria_terms: usize,
+    // everytime a request has a filter, this field must be incremented by one
+    sort_total_number_of_criteria: usize,
+
+    // filter
+    filter_with_geo_radius: bool,
+    // everytime a request has a filter, this field must be incremented by the number of terms it contains
+    filter_sum_of_criteria_terms: usize,
+    // everytime a request has a filter, this field must be incremented by one
+    filter_total_number_of_criteria: usize,
+    used_syntax: HashMap<String, usize>,
+
+    // q
+    // The maximum number of terms in a q request
+    max_terms_number: usize,
+
+    // pagination
+    max_limit: usize,
+    max_offset: usize,
+
+    // formatting
+    highlight_pre_tag: bool,
+    highlight_post_tag: bool,
+    crop_marker: bool,
+    show_matches_position: bool,
+    crop_length: bool,
+}
+
+impl SearchAggregator {
+    pub fn from_query(query: &SearchQuery, request: &HttpRequest) -> Self {
+        let mut ret = Self::default();
+        ret.timestamp = Some(OffsetDateTime::now_utc());
+
+        ret.total_received = 1;
+        ret.user_agents = extract_user_agents(request).into_iter().collect();
+
+        if let Some(ref sort) = query.sort {
+            ret.sort_total_number_of_criteria = 1;
+            ret.sort_with_geo_point = sort.iter().any(|s| s.contains("_geoPoint("));
+            ret.sort_sum_of_criteria_terms = sort.len();
+        }
+
+        if let Some(ref filter) = query.filter {
+            static RE: Lazy<Regex> = Lazy::new(|| Regex::new("AND | OR").unwrap());
+            ret.filter_total_number_of_criteria = 1;
+
+            let syntax = match filter {
+                Value::String(_) => "string".to_string(),
+                Value::Array(values) => {
+                    if values
+                        .iter()
+                        .map(|v| v.to_string())
+                        .any(|s| RE.is_match(&s))
+                    {
+                        "mixed".to_string()
+                    } else {
+                        "array".to_string()
+                    }
+                }
+                _ => "none".to_string(),
+            };
+            // convert the string to a HashMap
+            ret.used_syntax.insert(syntax, 1);
+
+            let stringified_filters = filter.to_string();
+            ret.filter_with_geo_radius = stringified_filters.contains("_geoRadius(");
+            ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count();
+        }
+
+        if let Some(ref q) = query.q {
+            ret.max_terms_number = q.split_whitespace().count();
+        }
+
+        ret.max_limit = query.limit;
+        ret.max_offset = query.offset.unwrap_or_default();
+
+        ret.highlight_pre_tag = query.highlight_pre_tag != DEFAULT_HIGHLIGHT_PRE_TAG();
+        ret.highlight_post_tag = query.highlight_post_tag != DEFAULT_HIGHLIGHT_POST_TAG();
+        ret.crop_marker = query.crop_marker != DEFAULT_CROP_MARKER();
+        ret.crop_length = query.crop_length != DEFAULT_CROP_LENGTH();
+        ret.show_matches_position = query.show_matches_position;
+
+        ret
+    }
+
+    pub fn succeed(&mut self, result: &SearchResult) {
+        self.total_succeeded = self.total_succeeded.saturating_add(1);
+        self.time_spent.push(result.processing_time_ms as usize);
+    }
+
+    /// Aggregate one [SearchAggregator] into another.
+    pub fn aggregate(&mut self, mut other: Self) {
+        if self.timestamp.is_none() {
+            self.timestamp = other.timestamp;
+        }
+
+        // context
+        for user_agent in other.user_agents.into_iter() {
+            self.user_agents.insert(user_agent);
+        }
+        // request
+        self.total_received = self.total_received.saturating_add(other.total_received);
+        self.total_succeeded = self.total_succeeded.saturating_add(other.total_succeeded);
+        self.time_spent.append(&mut other.time_spent);
+        // sort
+        self.sort_with_geo_point |= other.sort_with_geo_point;
+        self.sort_sum_of_criteria_terms = self
+            .sort_sum_of_criteria_terms
+            .saturating_add(other.sort_sum_of_criteria_terms);
+        self.sort_total_number_of_criteria = self
+            .sort_total_number_of_criteria
+            .saturating_add(other.sort_total_number_of_criteria);
+        // filter
+        self.filter_with_geo_radius |= other.filter_with_geo_radius;
+        self.filter_sum_of_criteria_terms = self
+            .filter_sum_of_criteria_terms
+            .saturating_add(other.filter_sum_of_criteria_terms);
+        self.filter_total_number_of_criteria = self
+            .filter_total_number_of_criteria
+            .saturating_add(other.filter_total_number_of_criteria);
+        for (key, value) in other.used_syntax.into_iter() {
+            let used_syntax = self.used_syntax.entry(key).or_insert(0);
+            *used_syntax = used_syntax.saturating_add(value);
+        }
+        // q
+        self.max_terms_number = self.max_terms_number.max(other.max_terms_number);
+        // pagination
+        self.max_limit = self.max_limit.max(other.max_limit);
+        self.max_offset = self.max_offset.max(other.max_offset);
+
+        self.highlight_pre_tag |= other.highlight_pre_tag;
+        self.highlight_post_tag |= other.highlight_post_tag;
+        self.crop_marker |= other.crop_marker;
+        self.show_matches_position |= other.show_matches_position;
+        self.crop_length |= other.crop_length;
+    }
+
+    pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
+        if self.total_received == 0 {
+            None
+        } else {
+            // the index of the 99th percentage of value
+            let percentile_99th = 0.99 * (self.total_succeeded as f64 - 1.) + 1.;
+            // we get all the values in a sorted manner
+            let time_spent = self.time_spent.into_sorted_vec();
+            // We are only interested by the slowest value of the 99th fastest results
+            let time_spent = time_spent.get(percentile_99th as usize);
+
+            let properties = json!({
+                "user-agent": self.user_agents,
+                "requests": {
+                    "99th_response_time":  time_spent.map(|t| format!("{:.2}", t)),
+                    "total_succeeded": self.total_succeeded,
+                    "total_failed": self.total_received.saturating_sub(self.total_succeeded), // just to be sure we never panics
+                    "total_received": self.total_received,
+                },
+                "sort": {
+                    "with_geoPoint": self.sort_with_geo_point,
+                    "avg_criteria_number": format!("{:.2}", self.sort_sum_of_criteria_terms as f64 / self.sort_total_number_of_criteria as f64),
+                },
+                "filter": {
+                   "with_geoRadius": self.filter_with_geo_radius,
+                   "avg_criteria_number": format!("{:.2}", self.filter_sum_of_criteria_terms as f64 / self.filter_total_number_of_criteria as f64),
+                   "most_used_syntax": self.used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)),
+                },
+                "q": {
+                   "max_terms_number": self.max_terms_number,
+                },
+                "pagination": {
+                   "max_limit": self.max_limit,
+                   "max_offset": self.max_offset,
+                },
+                "formatting": {
+                    "highlight_pre_tag": self.highlight_pre_tag,
+                    "highlight_post_tag": self.highlight_post_tag,
+                    "crop_marker": self.crop_marker,
+                    "show_matches_position": self.show_matches_position,
+                    "crop_length": self.crop_length,
+                },
+            });
+
+            Some(Track {
+                timestamp: self.timestamp,
+                user: user.clone(),
+                event: event_name.to_string(),
+                properties,
+                ..Default::default()
+            })
+        }
+    }
+}
+
+#[derive(Default)]
+pub struct DocumentsAggregator {
+    timestamp: Option<OffsetDateTime>,
+
+    // set to true when at least one request was received
+    updated: bool,
+
+    // context
+    user_agents: HashSet<String>,
+
+    content_types: HashSet<String>,
+    primary_keys: HashSet<String>,
+    index_creation: bool,
+}
+
+impl DocumentsAggregator {
+    pub fn from_query(
+        documents_query: &UpdateDocumentsQuery,
+        index_creation: bool,
+        request: &HttpRequest,
+    ) -> Self {
+        let mut ret = Self::default();
+        ret.timestamp = Some(OffsetDateTime::now_utc());
+
+        ret.updated = true;
+        ret.user_agents = extract_user_agents(request).into_iter().collect();
+        if let Some(primary_key) = documents_query.primary_key.clone() {
+            ret.primary_keys.insert(primary_key);
+        }
+        let content_type = request
+            .headers()
+            .get(CONTENT_TYPE)
+            .and_then(|s| s.to_str().ok())
+            .unwrap_or("unknown")
+            .to_string();
+        ret.content_types.insert(content_type);
+        ret.index_creation = index_creation;
+
+        ret
+    }
+
+    /// Aggregate one [DocumentsAggregator] into another.
+    pub fn aggregate(&mut self, other: Self) {
+        if self.timestamp.is_none() {
+            self.timestamp = other.timestamp;
+        }
+
+        self.updated |= other.updated;
+        // we can't create a union because there is no `into_union` method
+        for user_agent in other.user_agents {
+            self.user_agents.insert(user_agent);
+        }
+        for primary_key in other.primary_keys {
+            self.primary_keys.insert(primary_key);
+        }
+        for content_type in other.content_types {
+            self.content_types.insert(content_type);
+        }
+        self.index_creation |= other.index_creation;
+    }
+
+    pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
+        if !self.updated {
+            None
+        } else {
+            let properties = json!({
+                "user-agent": self.user_agents,
+                "payload_type": self.content_types,
+                "primary_key": self.primary_keys,
+                "index_creation": self.index_creation,
+            });
+
+            Some(Track {
+                timestamp: self.timestamp,
+                user: user.clone(),
+                event: event_name.to_string(),
+                properties,
+                ..Default::default()
+            })
+        }
+    }
+}

meilisearch-http/src/data/mod.rs

@@ -1,133 +0,0 @@
-use std::ops::Deref;
-use std::sync::Arc;
-
-use sha2::Digest;
-
-use crate::index::{Checked, Settings};
-use crate::index_controller::{
-    error::Result, DumpInfo, IndexController, IndexMetadata, IndexSettings, IndexStats, Stats,
-};
-use crate::option::Opt;
-
-pub mod search;
-mod updates;
-
-#[derive(Clone)]
-pub struct Data {
-    inner: Arc<DataInner>,
-}
-
-impl Deref for Data {
-    type Target = DataInner;
-
-    fn deref(&self) -> &Self::Target {
-        &self.inner
-    }
-}
-
-pub struct DataInner {
-    pub index_controller: IndexController,
-    pub api_keys: ApiKeys,
-    options: Opt,
-}
-
-#[derive(Clone)]
-pub struct ApiKeys {
-    pub public: Option<String>,
-    pub private: Option<String>,
-    pub master: Option<String>,
-}
-
-impl ApiKeys {
-    pub fn generate_missing_api_keys(&mut self) {
-        if let Some(master_key) = &self.master {
-            if self.private.is_none() {
-                let key = format!("{}-private", master_key);
-                let sha = sha2::Sha256::digest(key.as_bytes());
-                self.private = Some(format!("{:x}", sha));
-            }
-            if self.public.is_none() {
-                let key = format!("{}-public", master_key);
-                let sha = sha2::Sha256::digest(key.as_bytes());
-                self.public = Some(format!("{:x}", sha));
-            }
-        }
-    }
-}
-
-impl Data {
-    pub fn new(options: Opt) -> anyhow::Result<Data> {
-        let path = options.db_path.clone();
-
-        let index_controller = IndexController::new(&path, &options)?;
-
-        let mut api_keys = ApiKeys {
-            master: options.clone().master_key,
-            private: None,
-            public: None,
-        };
-
-        api_keys.generate_missing_api_keys();
-
-        let inner = DataInner {
-            index_controller,
-            api_keys,
-            options,
-        };
-        let inner = Arc::new(inner);
-
-        Ok(Data { inner })
-    }
-
-    pub async fn settings(&self, uid: String) -> Result<Settings<Checked>> {
-        self.index_controller.settings(uid).await
-    }
-
-    pub async fn list_indexes(&self) -> Result<Vec<IndexMetadata>> {
-        self.index_controller.list_indexes().await
-    }
-
-    pub async fn index(&self, uid: String) -> Result<IndexMetadata> {
-        self.index_controller.get_index(uid).await
-    }
-
-    pub async fn create_index(
-        &self,
-        uid: String,
-        primary_key: Option<String>,
-    ) -> Result<IndexMetadata> {
-        let settings = IndexSettings {
-            uid: Some(uid),
-            primary_key,
-        };
-
-        let meta = self.index_controller.create_index(settings).await?;
-        Ok(meta)
-    }
-
-    pub async fn get_index_stats(&self, uid: String) -> Result<IndexStats> {
-        Ok(self.index_controller.get_index_stats(uid).await?)
-    }
-
-    pub async fn get_all_stats(&self) -> Result<Stats> {
-        Ok(self.index_controller.get_all_stats().await?)
-    }
-
-    pub async fn create_dump(&self) -> Result<DumpInfo> {
-        Ok(self.index_controller.create_dump().await?)
-    }
-
-    pub async fn dump_status(&self, uid: String) -> Result<DumpInfo> {
-        Ok(self.index_controller.dump_info(uid).await?)
-    }
-
-    #[inline]
-    pub fn http_payload_size_limit(&self) -> usize {
-        self.options.http_payload_size_limit.get_bytes() as usize
-    }
-
-    #[inline]
-    pub fn api_keys(&self) -> &ApiKeys {
-        &self.api_keys
-    }
-}

meilisearch-http/src/data/search.rs

@@ -1,34 +0,0 @@
-use serde_json::{Map, Value};
-
-use super::Data;
-use crate::index::{SearchQuery, SearchResult};
-use crate::index_controller::error::Result;
-
-impl Data {
-    pub async fn search(&self, index: String, search_query: SearchQuery) -> Result<SearchResult> {
-        self.index_controller.search(index, search_query).await
-    }
-
-    pub async fn retrieve_documents(
-        &self,
-        index: String,
-        offset: usize,
-        limit: usize,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Vec<Map<String, Value>>> {
-        self.index_controller
-            .documents(index, offset, limit, attributes_to_retrieve)
-            .await
-    }
-
-    pub async fn retrieve_document(
-        &self,
-        index: String,
-        document_id: String,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Map<String, Value>> {
-        self.index_controller
-            .document(index, document_id, attributes_to_retrieve)
-            .await
-    }
-}

meilisearch-http/src/data/updates.rs

@@ -1,80 +0,0 @@
-use milli::update::{IndexDocumentsMethod, UpdateFormat};
-
-use crate::extractors::payload::Payload;
-use crate::index::{Checked, Settings};
-use crate::index_controller::{error::Result, IndexMetadata, IndexSettings, UpdateStatus};
-use crate::Data;
-
-impl Data {
-    pub async fn add_documents(
-        &self,
-        index: String,
-        method: IndexDocumentsMethod,
-        format: UpdateFormat,
-        stream: Payload,
-        primary_key: Option<String>,
-    ) -> Result<UpdateStatus> {
-        let update_status = self
-            .index_controller
-            .add_documents(index, method, format, stream, primary_key)
-            .await?;
-        Ok(update_status)
-    }
-
-    pub async fn update_settings(
-        &self,
-        index: String,
-        settings: Settings<Checked>,
-        create: bool,
-    ) -> Result<UpdateStatus> {
-        let update = self
-            .index_controller
-            .update_settings(index, settings, create)
-            .await?;
-        Ok(update)
-    }
-
-    pub async fn clear_documents(&self, index: String) -> Result<UpdateStatus> {
-        let update = self.index_controller.clear_documents(index).await?;
-        Ok(update)
-    }
-
-    pub async fn delete_documents(
-        &self,
-        index: String,
-        document_ids: Vec<String>,
-    ) -> Result<UpdateStatus> {
-        let update = self
-            .index_controller
-            .delete_documents(index, document_ids)
-            .await?;
-        Ok(update)
-    }
-
-    pub async fn delete_index(&self, index: String) -> Result<()> {
-        self.index_controller.delete_index(index).await?;
-        Ok(())
-    }
-
-    pub async fn get_update_status(&self, index: String, uid: u64) -> Result<UpdateStatus> {
-        self.index_controller.update_status(index, uid).await
-    }
-
-    pub async fn get_updates_status(&self, index: String) -> Result<Vec<UpdateStatus>> {
-        self.index_controller.all_update_status(index).await
-    }
-
-    pub async fn update_index(
-        &self,
-        uid: String,
-        primary_key: Option<String>,
-        new_uid: Option<String>,
-    ) -> Result<IndexMetadata> {
-        let settings = IndexSettings {
-            uid: new_uid,
-            primary_key,
-        };
-
-        self.index_controller.update_index(uid, settings).await
-    }
-}

meilisearch-http/src/error.rs

@@ -1,145 +1,57 @@
-use std::error::Error;
-use std::fmt;
-
 use actix_web as aweb;
-use actix_web::body::Body;
-use actix_web::dev::BaseHttpResponseBuilder;
-use actix_web::http::StatusCode;
 use aweb::error::{JsonPayloadError, QueryPayloadError};
-use meilisearch_error::{Code, ErrorCode};
-use milli::UserError;
-use serde::{Deserialize, Serialize};
+use meilisearch_types::error::{Code, ErrorCode, ResponseError};
 
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct ResponseError {
-    #[serde(skip)]
-    code: StatusCode,
-    message: String,
-    error_code: String,
-    error_type: String,
-    error_link: String,
+#[derive(Debug, thiserror::Error)]
+pub enum MeilisearchHttpError {
+    #[error("A Content-Type header is missing. Accepted values for the Content-Type header are: {}",
+            .0.iter().map(|s| format!("`{}`", s)).collect::<Vec<_>>().join(", "))]
+    MissingContentType(Vec<String>),
+    #[error(
+        "The Content-Type `{0}` is invalid. Accepted values for the Content-Type header are: {}",
+        .1.iter().map(|s| format!("`{}`", s)).collect::<Vec<_>>().join(", ")
+    )]
+    InvalidContentType(String, Vec<String>),
 }
 
-impl fmt::Display for ResponseError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.message.fmt(f)
-    }
-}
-
-impl<T> From<T> for ResponseError
-where
-    T: ErrorCode,
-{
-    fn from(other: T) -> Self {
-        Self {
-            code: other.http_status(),
-            message: other.to_string(),
-            error_code: other.error_name(),
-            error_type: other.error_type(),
-            error_link: other.error_url(),
-        }
-    }
-}
-
-impl aweb::error::ResponseError for ResponseError {
-    fn error_response(&self) -> aweb::BaseHttpResponse<Body> {
-        let json = serde_json::to_vec(self).unwrap();
-        BaseHttpResponseBuilder::new(self.status_code())
-            .content_type("application/json")
-            .body(json)
-    }
-
-    fn status_code(&self) -> StatusCode {
-        self.code
-    }
-}
-
-macro_rules! internal_error {
-    ($target:ty : $($other:path), *) => {
-        $(
-            impl From<$other> for $target {
-                fn from(other: $other) -> Self {
-                    Self::Internal(Box::new(other))
-                }
-            }
-        )*
-    }
-}
-
-#[derive(Debug)]
-pub struct MilliError<'a>(pub &'a milli::Error);
-
-impl Error for MilliError<'_> {}
-
-impl fmt::Display for MilliError<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
-impl ErrorCode for MilliError<'_> {
+impl ErrorCode for MeilisearchHttpError {
     fn error_code(&self) -> Code {
-        match self.0 {
-            milli::Error::InternalError(_) => Code::Internal,
-            milli::Error::IoError(_) => Code::Internal,
-            milli::Error::UserError(ref error) => {
-                match error {
-                    // TODO: wait for spec for new error codes.
-                    UserError::Csv(_)
-                    | UserError::SerdeJson(_)
-                    | UserError::MaxDatabaseSizeReached
-                    | UserError::InvalidCriterionName { .. }
-                    | UserError::InvalidDocumentId { .. }
-                    | UserError::InvalidStoreFile
-                    | UserError::NoSpaceLeftOnDevice
-                    | UserError::InvalidAscDescSyntax { .. }
-                    | UserError::DocumentLimitReached => Code::Internal,
-                    UserError::AttributeLimitReached => Code::MaxFieldsLimitExceeded,
-                    UserError::InvalidFilter(_) => Code::Filter,
-                    UserError::InvalidFilterAttribute(_) => Code::Filter,
-                    UserError::InvalidSortName { .. } => Code::Sort,
-                    UserError::MissingDocumentId { .. } => Code::MissingDocumentId,
-                    UserError::MissingPrimaryKey => Code::MissingPrimaryKey,
-                    UserError::PrimaryKeyCannotBeChanged => Code::PrimaryKeyAlreadyPresent,
-                    UserError::PrimaryKeyCannotBeReset => Code::PrimaryKeyAlreadyPresent,
-                    UserError::SortRankingRuleMissing => Code::Sort,
-                    UserError::UnknownInternalDocumentId { .. } => Code::DocumentNotFound,
-                    UserError::InvalidFacetsDistribution { .. } => Code::BadRequest,
-                    UserError::InvalidSortableAttribute { .. } => Code::Sort,
-                }
-            }
-        }
-    }
-}
-
-impl fmt::Display for PayloadError {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
-            PayloadError::Json(e) => e.fmt(f),
-            PayloadError::Query(e) => e.fmt(f),
+            MeilisearchHttpError::MissingContentType(_) => Code::MissingContentType,
+            MeilisearchHttpError::InvalidContentType(_, _) => Code::InvalidContentType,
         }
     }
 }
 
-#[derive(Debug)]
-pub enum PayloadError {
-    Json(JsonPayloadError),
-    Query(QueryPayloadError),
+impl From<MeilisearchHttpError> for aweb::Error {
+    fn from(other: MeilisearchHttpError) -> Self {
+        aweb::Error::from(ResponseError::from(other))
+    }
 }
 
-impl Error for PayloadError {}
+#[derive(Debug, thiserror::Error)]
+pub enum PayloadError {
+    #[error("{0}")]
+    Json(JsonPayloadError),
+    #[error("{0}")]
+    Query(QueryPayloadError),
+    #[error("The json payload provided is malformed. `{0}`.")]
+    MalformedPayload(serde_json::error::Error),
+    #[error("A json payload is missing.")]
+    MissingPayload,
+}
 
 impl ErrorCode for PayloadError {
     fn error_code(&self) -> Code {
         match self {
             PayloadError::Json(err) => match err {
-                JsonPayloadError::Overflow => Code::PayloadTooLarge,
+                JsonPayloadError::Overflow { .. } => Code::PayloadTooLarge,
                 JsonPayloadError::ContentType => Code::UnsupportedMediaType,
                 JsonPayloadError::Payload(aweb::error::PayloadError::Overflow) => {
                     Code::PayloadTooLarge
                 }
-                JsonPayloadError::Deserialize(_) | JsonPayloadError::Payload(_) => Code::BadRequest,
+                JsonPayloadError::Payload(_) => Code::BadRequest,
+                JsonPayloadError::Deserialize(_) => Code::BadRequest,
                 JsonPayloadError::Serialize(_) => Code::Internal,
                 _ => Code::Internal,
             },
@@ -147,13 +59,29 @@ impl ErrorCode for PayloadError {
                 QueryPayloadError::Deserialize(_) => Code::BadRequest,
                 _ => Code::Internal,
             },
+            PayloadError::MissingPayload => Code::MissingPayload,
+            PayloadError::MalformedPayload(_) => Code::MalformedPayload,
         }
     }
 }
 
 impl From<JsonPayloadError> for PayloadError {
     fn from(other: JsonPayloadError) -> Self {
-        Self::Json(other)
+        match other {
+            JsonPayloadError::Deserialize(e)
+                if e.classify() == serde_json::error::Category::Eof
+                    && e.line() == 1
+                    && e.column() == 0 =>
+            {
+                Self::MissingPayload
+            }
+            JsonPayloadError::Deserialize(e)
+                if e.classify() != serde_json::error::Category::Data =>
+            {
+                Self::MalformedPayload(e)
+            }
+            _ => Self::Json(other),
+        }
     }
 }
 
@@ -163,9 +91,8 @@ impl From<QueryPayloadError> for PayloadError {
     }
 }
 
-pub fn payload_error_handler<E>(err: E) -> ResponseError
-where
-    E: Into<PayloadError>,
-{
-    err.into().into()
+impl From<PayloadError> for aweb::Error {
+    fn from(other: PayloadError) -> Self {
+        aweb::Error::from(ResponseError::from(other))
+    }
 }

meilisearch-http/src/extractors/authentication/error.rs

@@ -1,25 +1,22 @@
-use meilisearch_error::{Code, ErrorCode};
+use meilisearch_types::error::{Code, ErrorCode};
 
 #[derive(Debug, thiserror::Error)]
 pub enum AuthenticationError {
-    #[error("You must have an authorization token")]
+    #[error("The Authorization header is missing. It must use the bearer authorization method.")]
     MissingAuthorizationHeader,
-    #[error("Invalid API key")]
-    InvalidToken(String),
+    #[error("The provided API key is invalid.")]
+    InvalidToken,
     // Triggered on configuration error.
-    #[error("Irretrievable state")]
+    #[error("An internal error has occurred. `Irretrievable state`.")]
     IrretrievableState,
-    #[error("Unknown authentication policy")]
-    UnknownPolicy,
 }
 
 impl ErrorCode for AuthenticationError {
     fn error_code(&self) -> Code {
         match self {
             AuthenticationError::MissingAuthorizationHeader => Code::MissingAuthorizationHeader,
-            AuthenticationError::InvalidToken(_) => Code::InvalidToken,
+            AuthenticationError::InvalidToken => Code::InvalidToken,
             AuthenticationError::IrretrievableState => Code::Internal,
-            AuthenticationError::UnknownPolicy => Code::Internal,
         }
     }
 }

meilisearch-http/src/extractors/authentication/mod.rs

@@ -1,84 +1,81 @@
 mod error;
 
-use std::any::{Any, TypeId};
-use std::collections::HashMap;
 use std::marker::PhantomData;
 use std::ops::Deref;
+use std::pin::Pin;
 
 use actix_web::FromRequest;
-use futures::future::err;
-use futures::future::{ok, Ready};
-
-use crate::error::ResponseError;
 use error::AuthenticationError;
+use futures::future::err;
+use futures::Future;
+use meilisearch_auth::{AuthController, AuthFilter};
+use meilisearch_types::error::{Code, ResponseError};
 
-macro_rules! create_policies {
-    ($($name:ident), *) => {
-        pub mod policies {
-            use std::collections::HashSet;
-            use crate::extractors::authentication::Policy;
-
-            $(
-                #[derive(Debug, Default)]
-                pub struct $name {
-                    inner: HashSet<Vec<u8>>
-                }
-
-                impl $name {
-                    pub fn new() -> Self {
-                        Self { inner: HashSet::new() }
-                    }
-
-                    pub fn add(&mut self, token: Vec<u8>) {
-                        self.inner.insert(token);
-                    }
-                }
-
-                impl Policy for $name {
-                    fn authenticate(&self, token: &[u8]) -> bool {
-                        self.inner.contains(token)
-                    }
-                }
-            )*
-        }
-    };
-}
-
-create_policies!(Public, Private, Admin);
-
-/// Instanciate a `Policies`, filled with the given policies.
-macro_rules! init_policies {
-    ($($name:ident), *) => {
-        {
-            let mut policies = crate::extractors::authentication::Policies::new();
-            $(
-                let policy = $name::new();
-                policies.insert(policy);
-            )*
-            policies
-        }
-    };
-}
-
-/// Adds user to all specified policies.
-macro_rules! create_users {
-    ($policies:ident, $($user:expr => { $($policy:ty), * }), *) => {
-        {
-            $(
-                $(
-                    $policies.get_mut::<$policy>().map(|p| p.add($user.to_owned()));
-                )*
-            )*
-        }
-    };
-}
-
-pub struct GuardedData<T, D> {
+pub struct GuardedData<P, D> {
     data: D,
-    _marker: PhantomData<T>,
+    filters: AuthFilter,
+    _marker: PhantomData<P>,
 }
 
-impl<T, D> Deref for GuardedData<T, D> {
+impl<P, D> GuardedData<P, D> {
+    pub fn filters(&self) -> &AuthFilter {
+        &self.filters
+    }
+
+    async fn auth_bearer(
+        auth: AuthController,
+        token: String,
+        index: Option<String>,
+        data: Option<D>,
+    ) -> Result<Self, ResponseError>
+    where
+        P: Policy + 'static,
+    {
+        match Self::authenticate(auth, token, index).await? {
+            Some(filters) => match data {
+                Some(data) => Ok(Self {
+                    data,
+                    filters,
+                    _marker: PhantomData,
+                }),
+                None => Err(AuthenticationError::IrretrievableState.into()),
+            },
+            None => Err(AuthenticationError::InvalidToken.into()),
+        }
+    }
+
+    async fn auth_token(auth: AuthController, data: Option<D>) -> Result<Self, ResponseError>
+    where
+        P: Policy + 'static,
+    {
+        match Self::authenticate(auth, String::new(), None).await? {
+            Some(filters) => match data {
+                Some(data) => Ok(Self {
+                    data,
+                    filters,
+                    _marker: PhantomData,
+                }),
+                None => Err(AuthenticationError::IrretrievableState.into()),
+            },
+            None => Err(AuthenticationError::MissingAuthorizationHeader.into()),
+        }
+    }
+
+    async fn authenticate(
+        auth: AuthController,
+        token: String,
+        index: Option<String>,
+    ) -> Result<Option<AuthFilter>, ResponseError>
+    where
+        P: Policy + 'static,
+    {
+        tokio::task::spawn_blocking(move || P::authenticate(auth, token.as_ref(), index.as_deref()))
+            .await
+            .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))
+    }
+}
+
+impl<P, D> Deref for GuardedData<P, D> {
     type Target = D;
 
     fn deref(&self) -> &Self::Target {
@@ -86,97 +83,172 @@ impl<T, D> Deref for GuardedData<T, D> {
     }
 }
 
-pub trait Policy {
-    fn authenticate(&self, token: &[u8]) -> bool;
-}
-
-#[derive(Debug)]
-pub struct Policies {
-    inner: HashMap<TypeId, Box<dyn Any>>,
-}
-
-impl Policies {
-    pub fn new() -> Self {
-        Self {
-            inner: HashMap::new(),
-        }
-    }
-
-    pub fn insert<S: Policy + 'static>(&mut self, policy: S) {
-        self.inner.insert(TypeId::of::<S>(), Box::new(policy));
-    }
-
-    pub fn get<S: Policy + 'static>(&self) -> Option<&S> {
-        self.inner
-            .get(&TypeId::of::<S>())
-            .and_then(|p| p.downcast_ref::<S>())
-    }
-
-    pub fn get_mut<S: Policy + 'static>(&mut self) -> Option<&mut S> {
-        self.inner
-            .get_mut(&TypeId::of::<S>())
-            .and_then(|p| p.downcast_mut::<S>())
-    }
-}
-
-impl Default for Policies {
-    fn default() -> Self {
-        Self::new()
-    }
-}
-
-pub enum AuthConfig {
-    NoAuth,
-    Auth(Policies),
-}
-
-impl Default for AuthConfig {
-    fn default() -> Self {
-        Self::NoAuth
-    }
-}
-
 impl<P: Policy + 'static, D: 'static + Clone> FromRequest for GuardedData<P, D> {
-    type Config = AuthConfig;
-
     type Error = ResponseError;
 
-    type Future = Ready<Result<Self, Self::Error>>;
+    type Future = Pin<Box<dyn Future<Output = Result<Self, Self::Error>>>>;
 
     fn from_request(
         req: &actix_web::HttpRequest,
-        _payload: &mut actix_http::Payload,
+        _payload: &mut actix_web::dev::Payload,
     ) -> Self::Future {
-        match req.app_data::<Self::Config>() {
-            Some(config) => match config {
-                AuthConfig::NoAuth => match req.app_data::<D>().cloned() {
-                    Some(data) => ok(Self {
-                        data,
-                        _marker: PhantomData,
-                    }),
-                    None => err(AuthenticationError::IrretrievableState.into()),
-                },
-                AuthConfig::Auth(policies) => match policies.get::<P>() {
-                    Some(policy) => match req.headers().get("x-meili-api-key") {
-                        Some(token) => {
-                            if policy.authenticate(token.as_bytes()) {
-                                match req.app_data::<D>().cloned() {
-                                    Some(data) => ok(Self {
-                                        data,
-                                        _marker: PhantomData,
-                                    }),
-                                    None => err(AuthenticationError::IrretrievableState.into()),
-                                }
-                            } else {
-                                err(AuthenticationError::InvalidToken(String::from("hello")).into())
-                            }
+        match req.app_data::<AuthController>().cloned() {
+            Some(auth) => match req
+                .headers()
+                .get("Authorization")
+                .map(|type_token| type_token.to_str().unwrap_or_default().splitn(2, ' '))
+            {
+                Some(mut type_token) => match type_token.next() {
+                    Some("Bearer") => {
+                        // TODO: find a less hardcoded way?
+                        let index = req.match_info().get("index_uid");
+                        match type_token.next() {
+                            Some(token) => Box::pin(Self::auth_bearer(
+                                auth,
+                                token.to_string(),
+                                index.map(String::from),
+                                req.app_data::<D>().cloned(),
+                            )),
+                            None => Box::pin(err(AuthenticationError::InvalidToken.into())),
                         }
-                        None => err(AuthenticationError::MissingAuthorizationHeader.into()),
-                    },
-                    None => err(AuthenticationError::UnknownPolicy.into()),
+                    }
+                    _otherwise => {
+                        Box::pin(err(AuthenticationError::MissingAuthorizationHeader.into()))
+                    }
                 },
+                None => Box::pin(Self::auth_token(auth, req.app_data::<D>().cloned())),
             },
-            None => err(AuthenticationError::IrretrievableState.into()),
+            None => Box::pin(err(AuthenticationError::IrretrievableState.into())),
         }
     }
 }
+
+pub trait Policy {
+    fn authenticate(auth: AuthController, token: &str, index: Option<&str>) -> Option<AuthFilter>;
+}
+
+pub mod policies {
+    use jsonwebtoken::{decode, Algorithm, DecodingKey, Validation};
+    use serde::{Deserialize, Serialize};
+    use time::OffsetDateTime;
+    use uuid::Uuid;
+
+    use crate::extractors::authentication::Policy;
+    use meilisearch_auth::{Action, AuthController, AuthFilter, SearchRules};
+    // reexport actions in policies in order to be used in routes configuration.
+    pub use meilisearch_auth::actions;
+
+    fn tenant_token_validation() -> Validation {
+        let mut validation = Validation::default();
+        validation.validate_exp = false;
+        validation.required_spec_claims.remove("exp");
+        validation.algorithms = vec![Algorithm::HS256, Algorithm::HS384, Algorithm::HS512];
+        validation
+    }
+
+    /// Extracts the key id used to sign the payload, without performing any validation.
+    fn extract_key_id(token: &str) -> Option<Uuid> {
+        let mut validation = tenant_token_validation();
+        validation.insecure_disable_signature_validation();
+        let dummy_key = DecodingKey::from_secret(b"secret");
+        let token_data = decode::<Claims>(token, &dummy_key, &validation).ok()?;
+
+        // get token fields without validating it.
+        let Claims { api_key_uid, .. } = token_data.claims;
+        Some(api_key_uid)
+    }
+
+    fn is_keys_action(action: u8) -> bool {
+        use actions::*;
+        matches!(action, KEYS_GET | KEYS_CREATE | KEYS_UPDATE | KEYS_DELETE)
+    }
+
+    pub struct ActionPolicy<const A: u8>;
+
+    impl<const A: u8> Policy for ActionPolicy<A> {
+        fn authenticate(
+            auth: AuthController,
+            token: &str,
+            index: Option<&str>,
+        ) -> Option<AuthFilter> {
+            // authenticate if token is the master key.
+            // master key can only have access to keys routes.
+            // if master key is None only keys routes are inaccessible.
+            if auth
+                .get_master_key()
+                .map_or_else(|| !is_keys_action(A), |mk| mk == token)
+            {
+                return Some(AuthFilter::default());
+            }
+
+            // Tenant token
+            if let Some(filters) = ActionPolicy::<A>::authenticate_tenant_token(&auth, token, index)
+            {
+                return Some(filters);
+            } else if let Some(action) = Action::from_repr(A) {
+                // API key
+                if let Ok(Some(uid)) = auth.get_optional_uid_from_encoded_key(token.as_bytes()) {
+                    if let Ok(true) = auth.is_key_authorized(uid, action, index) {
+                        return auth.get_key_filters(uid, None).ok();
+                    }
+                }
+            }
+
+            None
+        }
+    }
+
+    impl<const A: u8> ActionPolicy<A> {
+        fn authenticate_tenant_token(
+            auth: &AuthController,
+            token: &str,
+            index: Option<&str>,
+        ) -> Option<AuthFilter> {
+            // Only search action can be accessed by a tenant token.
+            if A != actions::SEARCH {
+                return None;
+            }
+
+            let uid = extract_key_id(token)?;
+            // check if parent key is authorized to do the action.
+            if auth.is_key_authorized(uid, Action::Search, index).ok()? {
+                // Check if tenant token is valid.
+                let key = auth.generate_key(uid)?;
+                let data = decode::<Claims>(
+                    token,
+                    &DecodingKey::from_secret(key.as_bytes()),
+                    &tenant_token_validation(),
+                )
+                .ok()?;
+
+                // Check index access if an index restriction is provided.
+                if let Some(index) = index {
+                    if !data.claims.search_rules.is_index_authorized(index) {
+                        return None;
+                    }
+                }
+
+                // Check if token is expired.
+                if let Some(exp) = data.claims.exp {
+                    if OffsetDateTime::now_utc().unix_timestamp() > exp {
+                        return None;
+                    }
+                }
+
+                return auth
+                    .get_key_filters(uid, Some(data.claims.search_rules))
+                    .ok();
+            }
+
+            None
+        }
+    }
+
+    #[derive(Debug, Serialize, Deserialize)]
+    #[serde(rename_all = "camelCase")]
+    struct Claims {
+        search_rules: SearchRules,
+        exp: Option<i64>,
+        api_key_uid: Uuid,
+    }
+}

meilisearch-http/src/extractors/mod.rs

@@ -1,3 +1,4 @@
 pub mod payload;
 #[macro_use]
 pub mod authentication;
+pub mod sequential_extractor;

meilisearch-http/src/extractors/payload.rs

@@ -1,7 +1,7 @@
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
-use actix_http::error::PayloadError;
+use actix_web::error::PayloadError;
 use actix_web::{dev, web, FromRequest, HttpRequest};
 use futures::future::{ready, Ready};
 use futures::Stream;
@@ -28,8 +28,6 @@ impl Default for PayloadConfig {
 }
 
 impl FromRequest for Payload {
-    type Config = PayloadConfig;
-
     type Error = PayloadError;
 
     type Future = Ready<Result<Payload, Self::Error>>;
@@ -39,7 +37,7 @@ impl FromRequest for Payload {
         let limit = req
             .app_data::<PayloadConfig>()
             .map(|c| c.limit)
-            .unwrap_or(Self::Config::default().limit);
+            .unwrap_or(PayloadConfig::default().limit);
         ready(Ok(Payload {
             payload: payload.take(),
             limit,

meilisearch-http/src/extractors/sequential_extractor.rs

@@ -0,0 +1,148 @@
+#![allow(non_snake_case)]
+use std::{future::Future, pin::Pin, task::Poll};
+
+use actix_web::{dev::Payload, FromRequest, Handler, HttpRequest};
+use pin_project_lite::pin_project;
+
+/// `SeqHandler` is an actix `Handler` that enforces that extractors errors are returned in the
+/// same order as they are defined in the wrapped handler. This is needed because, by default, actix
+/// resolves the extractors concurrently, whereas we always need the authentication extractor to
+/// throw first.
+#[derive(Clone)]
+pub struct SeqHandler<H>(pub H);
+
+pub struct SeqFromRequest<T>(T);
+
+/// This macro implements `FromRequest` for arbitrary arity handler, except for one, which is
+/// useless anyway.
+macro_rules! gen_seq {
+    ($ty:ident; $($T:ident)+) => {
+        pin_project! {
+            pub struct $ty<$($T: FromRequest), +> {
+                $(
+                #[pin]
+                $T: ExtractFuture<$T::Future, $T, $T::Error>,
+                )+
+            }
+        }
+
+        impl<$($T: FromRequest), +> Future for $ty<$($T),+> {
+            type Output = Result<SeqFromRequest<($($T),+)>, actix_web::Error>;
+
+            fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
+                let mut this = self.project();
+
+                let mut count_fut = 0;
+                let mut count_finished = 0;
+
+                $(
+                count_fut += 1;
+                match this.$T.as_mut().project() {
+                    ExtractProj::Future { fut } => match fut.poll(cx) {
+                        Poll::Ready(Ok(output)) => {
+                            count_finished += 1;
+                            let _ = this
+                                .$T
+                                .as_mut()
+                                .project_replace(ExtractFuture::Done { output });
+                        }
+                        Poll::Ready(Err(error)) => {
+                            count_finished += 1;
+                            let _ = this
+                                .$T
+                                .as_mut()
+                                .project_replace(ExtractFuture::Error { error });
+                        }
+                        Poll::Pending => (),
+                    },
+                    ExtractProj::Done { .. } => count_finished += 1,
+                    ExtractProj::Error { .. } => {
+                        // short circuit if all previous are finished and we had an error.
+                        if count_finished == count_fut {
+                            match this.$T.project_replace(ExtractFuture::Empty) {
+                                ExtractReplaceProj::Error { error } => {
+                                    return Poll::Ready(Err(error.into()))
+                                }
+                                _ => unreachable!("Invalid future state"),
+                            }
+                        } else {
+                            count_finished += 1;
+                        }
+                    }
+                    ExtractProj::Empty => unreachable!("From request polled after being finished. {}", stringify!($T)),
+                }
+                )+
+
+                if count_fut == count_finished {
+                    let result = (
+                        $(
+                            match this.$T.project_replace(ExtractFuture::Empty) {
+                                ExtractReplaceProj::Done { output } => output,
+                                ExtractReplaceProj::Error { error } => return Poll::Ready(Err(error.into())),
+                                _ => unreachable!("Invalid future state"),
+                            },
+                        )+
+                    );
+
+                    Poll::Ready(Ok(SeqFromRequest(result)))
+                } else {
+                    Poll::Pending
+                }
+            }
+        }
+
+        impl<$($T: FromRequest,)+> FromRequest for SeqFromRequest<($($T,)+)> {
+            type Error = actix_web::Error;
+
+            type Future = $ty<$($T),+>;
+
+            fn from_request(req: &HttpRequest, payload: &mut Payload) -> Self::Future {
+                $ty {
+                $(
+                    $T: ExtractFuture::Future {
+                        fut: $T::from_request(req, payload),
+                    },
+                )+
+                }
+            }
+        }
+
+    impl<Han, $($T: FromRequest),+> Handler<SeqFromRequest<($($T),+)>> for SeqHandler<Han>
+    where
+        Han: Handler<($($T),+)>,
+    {
+        type Output = Han::Output;
+        type Future = Han::Future;
+
+        fn call(&self, args: SeqFromRequest<($($T),+)>) -> Self::Future {
+            self.0.call(args.0)
+        }
+    }
+    };
+}
+
+// Not working for a single argument, but then, it is not really necessary.
+// gen_seq! { SeqFromRequestFut1; A }
+gen_seq! { SeqFromRequestFut2; A B }
+gen_seq! { SeqFromRequestFut3; A B C }
+gen_seq! { SeqFromRequestFut4; A B C D }
+gen_seq! { SeqFromRequestFut5; A B C D E }
+gen_seq! { SeqFromRequestFut6; A B C D E F }
+
+pin_project! {
+    #[project = ExtractProj]
+    #[project_replace = ExtractReplaceProj]
+    enum ExtractFuture<Fut, Res, Err> {
+        Future {
+            #[pin]
+            fut: Fut,
+        },
+        Done {
+            output: Res,
+        },
+        Error {
+            error: Err,
+        },
+        Empty,
+    }
+}

meilisearch-http/src/helpers/env.rs

@@ -1,10 +1,11 @@
+use meilisearch_lib::heed::Env;
 use walkdir::WalkDir;
 
 pub trait EnvSizer {
     fn size(&self) -> u64;
 }
 
-impl EnvSizer for heed::Env {
+impl EnvSizer for Env {
     fn size(&self) -> u64 {
         WalkDir::new(self.path())
             .into_iter()

meilisearch-http/src/helpers/mod.rs

@@ -1,4 +1,3 @@
-pub mod compression;
 mod env;
 
 pub use env::EnvSizer;

meilisearch-http/src/index/dump.rs

@@ -1,170 +0,0 @@
-use std::fs::{create_dir_all, File};
-use std::io::{BufRead, BufReader, Write};
-use std::path::Path;
-use std::sync::Arc;
-
-use anyhow::{bail, Context};
-use heed::RoTxn;
-use indexmap::IndexMap;
-use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream};
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-
-use crate::index_controller::{asc_ranking_rule, desc_ranking_rule};
-use crate::option::IndexerOpts;
-
-use super::error::Result;
-use super::{update_handler::UpdateHandler, Index, Settings, Unchecked};
-
-#[derive(Serialize, Deserialize)]
-struct DumpMeta {
-    settings: Settings<Unchecked>,
-    primary_key: Option<String>,
-}
-
-const META_FILE_NAME: &str = "meta.json";
-const DATA_FILE_NAME: &str = "documents.jsonl";
-
-impl Index {
-    pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
-        // acquire write txn make sure any ongoing write is finished before we start.
-        let txn = self.env.write_txn()?;
-
-        self.dump_documents(&txn, &path)?;
-        self.dump_meta(&txn, &path)?;
-
-        Ok(())
-    }
-
-    fn dump_documents(&self, txn: &RoTxn, path: impl AsRef<Path>) -> Result<()> {
-        let document_file_path = path.as_ref().join(DATA_FILE_NAME);
-        let mut document_file = File::create(&document_file_path)?;
-
-        let documents = self.all_documents(txn)?;
-        let fields_ids_map = self.fields_ids_map(txn)?;
-
-        // dump documents
-        let mut json_map = IndexMap::new();
-        for document in documents {
-            let (_, reader) = document?;
-
-            for (fid, bytes) in reader.iter() {
-                if let Some(name) = fields_ids_map.name(fid) {
-                    json_map.insert(name, serde_json::from_slice::<serde_json::Value>(bytes)?);
-                }
-            }
-
-            serde_json::to_writer(&mut document_file, &json_map)?;
-            document_file.write_all(b"\n")?;
-
-            json_map.clear();
-        }
-
-        Ok(())
-    }
-
-    fn dump_meta(&self, txn: &RoTxn, path: impl AsRef<Path>) -> Result<()> {
-        let meta_file_path = path.as_ref().join(META_FILE_NAME);
-        let mut meta_file = File::create(&meta_file_path)?;
-
-        let settings = self.settings_txn(txn)?.into_unchecked();
-        let primary_key = self.primary_key(txn)?.map(String::from);
-        let meta = DumpMeta {
-            settings,
-            primary_key,
-        };
-
-        serde_json::to_writer(&mut meta_file, &meta)?;
-
-        Ok(())
-    }
-
-    pub fn load_dump(
-        src: impl AsRef<Path>,
-        dst: impl AsRef<Path>,
-        size: usize,
-        indexing_options: &IndexerOpts,
-    ) -> anyhow::Result<()> {
-        let dir_name = src
-            .as_ref()
-            .file_name()
-            .with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?;
-
-        let dst_dir_path = dst.as_ref().join("indexes").join(dir_name);
-        create_dir_all(&dst_dir_path)?;
-
-        let meta_path = src.as_ref().join(META_FILE_NAME);
-        let mut meta_file = File::open(meta_path)?;
-
-        // We first deserialize the dump meta into a serde_json::Value and change
-        // the custom ranking rules settings from the old format to the new format.
-        let mut meta: Value = serde_json::from_reader(&mut meta_file)?;
-        if let Some(ranking_rules) = meta.pointer_mut("/settings/rankingRules") {
-            convert_custom_ranking_rules(ranking_rules);
-        }
-
-        // Then we serialize it back into a vec to deserialize it
-        // into a `DumpMeta` struct with the newly patched `rankingRules` format.
-        let patched_meta = serde_json::to_vec(&meta)?;
-
-        let DumpMeta {
-            settings,
-            primary_key,
-        } = serde_json::from_slice(&patched_meta)?;
-        let settings = settings.check();
-        let index = Self::open(&dst_dir_path, size)?;
-        let mut txn = index.write_txn()?;
-
-        let handler = UpdateHandler::new(indexing_options)?;
-
-        index.update_settings_txn(&mut txn, &settings, handler.update_builder(0))?;
-
-        let document_file_path = src.as_ref().join(DATA_FILE_NAME);
-        let reader = File::open(&document_file_path)?;
-        let mut reader = BufReader::new(reader);
-        reader.fill_buf()?;
-        // If the document file is empty, we don't perform the document addition, to prevent
-        // a primary key error to be thrown.
-        if !reader.buffer().is_empty() {
-            index.update_documents_txn(
-                &mut txn,
-                JsonStream,
-                IndexDocumentsMethod::UpdateDocuments,
-                Some(reader),
-                handler.update_builder(0),
-                primary_key.as_deref(),
-            )?;
-        }
-
-        txn.commit()?;
-
-        match Arc::try_unwrap(index.0) {
-            Ok(inner) => inner.prepare_for_closing().wait(),
-            Err(_) => bail!("Could not close index properly."),
-        }
-
-        Ok(())
-    }
-}
-
-/// Converts the ranking rules from the format `asc(_)`, `desc(_)` to the format `_:asc`, `_:desc`.
-///
-/// This is done for compatibility reasons, and to avoid a new dump version,
-/// since the new syntax was introduced soon after the new dump version.
-fn convert_custom_ranking_rules(ranking_rules: &mut Value) {
-    *ranking_rules = match ranking_rules.take() {
-        Value::Array(values) => values
-            .into_iter()
-            .filter_map(|value| match value {
-                Value::String(s) if s.starts_with("asc") => asc_ranking_rule(&s)
-                    .map(|f| format!("{}:asc", f))
-                    .map(Value::String),
-                Value::String(s) if s.starts_with("desc") => desc_ranking_rule(&s)
-                    .map(|f| format!("{}:desc", f))
-                    .map(Value::String),
-                otherwise => Some(otherwise),
-            })
-            .collect(),
-        otherwise => otherwise,
-    }
-}

meilisearch-http/src/index/mod.rs

@@ -1,198 +0,0 @@
-use std::collections::{BTreeSet, HashSet};
-use std::fs::create_dir_all;
-use std::marker::PhantomData;
-use std::ops::Deref;
-use std::path::Path;
-use std::sync::Arc;
-
-use heed::{EnvOpenOptions, RoTxn};
-use milli::update::Setting;
-use milli::{obkv_to_json, FieldId};
-use serde_json::{Map, Value};
-
-use error::Result;
-pub use search::{default_crop_length, SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT};
-pub use updates::{Checked, Facets, Settings, Unchecked};
-
-use crate::helpers::EnvSizer;
-
-use self::error::IndexError;
-
-pub mod error;
-pub mod update_handler;
-
-mod dump;
-mod search;
-mod updates;
-
-pub type Document = Map<String, Value>;
-
-#[derive(Clone)]
-pub struct Index(pub Arc<milli::Index>);
-
-impl Deref for Index {
-    type Target = milli::Index;
-
-    fn deref(&self) -> &Self::Target {
-        self.0.as_ref()
-    }
-}
-
-impl Index {
-    pub fn open(path: impl AsRef<Path>, size: usize) -> Result<Self> {
-        create_dir_all(&path)?;
-        let mut options = EnvOpenOptions::new();
-        options.map_size(size);
-        let index = milli::Index::new(options, &path)?;
-        Ok(Index(Arc::new(index)))
-    }
-
-    pub fn settings(&self) -> Result<Settings<Checked>> {
-        let txn = self.read_txn()?;
-        self.settings_txn(&txn)
-    }
-
-    pub fn settings_txn(&self, txn: &RoTxn) -> Result<Settings<Checked>> {
-        let displayed_attributes = self
-            .displayed_fields(txn)?
-            .map(|fields| fields.into_iter().map(String::from).collect());
-
-        let searchable_attributes = self
-            .searchable_fields(txn)?
-            .map(|fields| fields.into_iter().map(String::from).collect());
-
-        let filterable_attributes = self.filterable_fields(txn)?.into_iter().collect();
-
-        let sortable_attributes = self.sortable_fields(txn)?.into_iter().collect();
-
-        let criteria = self
-            .criteria(txn)?
-            .into_iter()
-            .map(|c| c.to_string())
-            .collect();
-
-        let stop_words = self
-            .stop_words(txn)?
-            .map(|stop_words| -> Result<BTreeSet<_>> {
-                Ok(stop_words.stream().into_strs()?.into_iter().collect())
-            })
-            .transpose()?
-            .unwrap_or_else(BTreeSet::new);
-        let distinct_field = self.distinct_field(txn)?.map(String::from);
-
-        // in milli each word in the synonyms map were split on their separator. Since we lost
-        // this information we are going to put space between words.
-        let synonyms = self
-            .synonyms(txn)?
-            .iter()
-            .map(|(key, values)| {
-                (
-                    key.join(" "),
-                    values.iter().map(|value| value.join(" ")).collect(),
-                )
-            })
-            .collect();
-
-        Ok(Settings {
-            displayed_attributes: match displayed_attributes {
-                Some(attrs) => Setting::Set(attrs),
-                None => Setting::Reset,
-            },
-            searchable_attributes: match searchable_attributes {
-                Some(attrs) => Setting::Set(attrs),
-                None => Setting::Reset,
-            },
-            filterable_attributes: Setting::Set(filterable_attributes),
-            sortable_attributes: Setting::Set(sortable_attributes),
-            ranking_rules: Setting::Set(criteria),
-            stop_words: Setting::Set(stop_words),
-            distinct_attribute: match distinct_field {
-                Some(field) => Setting::Set(field),
-                None => Setting::Reset,
-            },
-            synonyms: Setting::Set(synonyms),
-            _kind: PhantomData,
-        })
-    }
-
-    pub fn retrieve_documents<S: AsRef<str>>(
-        &self,
-        offset: usize,
-        limit: usize,
-        attributes_to_retrieve: Option<Vec<S>>,
-    ) -> Result<Vec<Map<String, Value>>> {
-        let txn = self.read_txn()?;
-
-        let fields_ids_map = self.fields_ids_map(&txn)?;
-        let fields_to_display =
-            self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?;
-
-        let iter = self.documents.range(&txn, &(..))?.skip(offset).take(limit);
-
-        let mut documents = Vec::new();
-
-        for entry in iter {
-            let (_id, obkv) = entry?;
-            let object = obkv_to_json(&fields_to_display, &fields_ids_map, obkv)?;
-            documents.push(object);
-        }
-
-        Ok(documents)
-    }
-
-    pub fn retrieve_document<S: AsRef<str>>(
-        &self,
-        doc_id: String,
-        attributes_to_retrieve: Option<Vec<S>>,
-    ) -> Result<Map<String, Value>> {
-        let txn = self.read_txn()?;
-
-        let fields_ids_map = self.fields_ids_map(&txn)?;
-
-        let fields_to_display =
-            self.fields_to_display(&txn, &attributes_to_retrieve, &fields_ids_map)?;
-
-        let internal_id = self
-            .external_documents_ids(&txn)?
-            .get(doc_id.as_bytes())
-            .ok_or_else(|| IndexError::DocumentNotFound(doc_id.clone()))?;
-
-        let document = self
-            .documents(&txn, std::iter::once(internal_id))?
-            .into_iter()
-            .next()
-            .map(|(_, d)| d)
-            .ok_or(IndexError::DocumentNotFound(doc_id))?;
-
-        let document = obkv_to_json(&fields_to_display, &fields_ids_map, document)?;
-
-        Ok(document)
-    }
-
-    pub fn size(&self) -> u64 {
-        self.env.size()
-    }
-
-    fn fields_to_display<S: AsRef<str>>(
-        &self,
-        txn: &heed::RoTxn,
-        attributes_to_retrieve: &Option<Vec<S>>,
-        fields_ids_map: &milli::FieldsIdsMap,
-    ) -> Result<Vec<FieldId>> {
-        let mut displayed_fields_ids = match self.displayed_fields_ids(txn)? {
-            Some(ids) => ids.into_iter().collect::<Vec<_>>(),
-            None => fields_ids_map.iter().map(|(id, _)| id).collect(),
-        };
-
-        let attributes_to_retrieve_ids = match attributes_to_retrieve {
-            Some(attrs) => attrs
-                .iter()
-                .filter_map(|f| fields_ids_map.id(f.as_ref()))
-                .collect::<HashSet<_>>(),
-            None => fields_ids_map.iter().map(|(id, _)| id).collect(),
-        };
-
-        displayed_fields_ids.retain(|fid| attributes_to_retrieve_ids.contains(fid));
-        Ok(displayed_fields_ids)
-    }
-}

meilisearch-http/src/index/update_handler.rs

@@ -1,89 +0,0 @@
-use std::fs::File;
-
-use crate::index::Index;
-use milli::update::UpdateBuilder;
-use milli::CompressionType;
-use rayon::ThreadPool;
-
-use crate::index_controller::UpdateMeta;
-use crate::index_controller::{Failed, Processed, Processing};
-use crate::option::IndexerOpts;
-
-pub struct UpdateHandler {
-    max_nb_chunks: Option<usize>,
-    chunk_compression_level: Option<u32>,
-    thread_pool: ThreadPool,
-    log_frequency: usize,
-    max_memory: Option<usize>,
-    chunk_compression_type: CompressionType,
-}
-
-impl UpdateHandler {
-    pub fn new(opt: &IndexerOpts) -> anyhow::Result<Self> {
-        let thread_pool = rayon::ThreadPoolBuilder::new()
-            .num_threads(opt.indexing_jobs.unwrap_or(num_cpus::get() / 2))
-            .build()?;
-
-        Ok(Self {
-            max_nb_chunks: opt.max_nb_chunks,
-            chunk_compression_level: opt.chunk_compression_level,
-            thread_pool,
-            log_frequency: opt.log_every_n,
-            max_memory: opt.max_memory.map(|m| m.get_bytes() as usize),
-            chunk_compression_type: opt.chunk_compression_type,
-        })
-    }
-
-    pub fn update_builder(&self, update_id: u64) -> UpdateBuilder {
-        // We prepare the update by using the update builder.
-        let mut update_builder = UpdateBuilder::new(update_id);
-        if let Some(max_nb_chunks) = self.max_nb_chunks {
-            update_builder.max_nb_chunks(max_nb_chunks);
-        }
-        if let Some(chunk_compression_level) = self.chunk_compression_level {
-            update_builder.chunk_compression_level(chunk_compression_level);
-        }
-        update_builder.thread_pool(&self.thread_pool);
-        update_builder.log_every_n(self.log_frequency);
-        if let Some(max_memory) = self.max_memory {
-            update_builder.max_memory(max_memory);
-        }
-        update_builder.chunk_compression_type(self.chunk_compression_type);
-        update_builder
-    }
-
-    pub fn handle_update(
-        &self,
-        meta: Processing,
-        content: Option<File>,
-        index: Index,
-    ) -> Result<Processed, Failed> {
-        use UpdateMeta::*;
-
-        let update_id = meta.id();
-
-        let update_builder = self.update_builder(update_id);
-
-        let result = match meta.meta() {
-            DocumentsAddition {
-                method,
-                format,
-                primary_key,
-            } => index.update_documents(
-                *format,
-                *method,
-                content,
-                update_builder,
-                primary_key.as_deref(),
-            ),
-            ClearDocuments => index.clear_documents(update_builder),
-            DeleteDocuments { ids } => index.delete_documents(ids, update_builder),
-            Settings(settings) => index.update_settings(&settings.clone().check(), update_builder),
-        };
-
-        match result {
-            Ok(result) => Ok(meta.process(result)),
-            Err(e) => Err(meta.fail(e.into())),
-        }
-    }
-}

meilisearch-http/src/index/updates.rs

@@ -1,383 +0,0 @@
-use std::collections::{BTreeMap, BTreeSet, HashSet};
-use std::io;
-use std::marker::PhantomData;
-use std::num::NonZeroUsize;
-
-use flate2::read::GzDecoder;
-use log::{debug, info, trace};
-use milli::update::{IndexDocumentsMethod, Setting, UpdateBuilder, UpdateFormat};
-use serde::{Deserialize, Serialize, Serializer};
-
-use crate::index_controller::UpdateResult;
-
-use super::error::Result;
-use super::Index;
-
-fn serialize_with_wildcard<S>(
-    field: &Setting<Vec<String>>,
-    s: S,
-) -> std::result::Result<S::Ok, S::Error>
-where
-    S: Serializer,
-{
-    let wildcard = vec!["*".to_string()];
-    match field {
-        Setting::Set(value) => Some(value),
-        Setting::Reset => Some(&wildcard),
-        Setting::NotSet => None,
-    }
-    .serialize(s)
-}
-
-#[derive(Clone, Default, Debug, Serialize)]
-pub struct Checked;
-
-#[derive(Clone, Default, Debug, Serialize, Deserialize)]
-pub struct Unchecked;
-
-#[derive(Debug, Clone, Default, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-#[serde(rename_all = "camelCase")]
-#[serde(bound(serialize = "T: Serialize", deserialize = "T: Deserialize<'static>"))]
-pub struct Settings<T> {
-    #[serde(
-        default,
-        serialize_with = "serialize_with_wildcard",
-        skip_serializing_if = "Setting::is_not_set"
-    )]
-    pub displayed_attributes: Setting<Vec<String>>,
-
-    #[serde(
-        default,
-        serialize_with = "serialize_with_wildcard",
-        skip_serializing_if = "Setting::is_not_set"
-    )]
-    pub searchable_attributes: Setting<Vec<String>>,
-
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    pub filterable_attributes: Setting<BTreeSet<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    pub sortable_attributes: Setting<BTreeSet<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    pub ranking_rules: Setting<Vec<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    pub stop_words: Setting<BTreeSet<String>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    pub synonyms: Setting<BTreeMap<String, Vec<String>>>,
-    #[serde(default, skip_serializing_if = "Setting::is_not_set")]
-    pub distinct_attribute: Setting<String>,
-
-    #[serde(skip)]
-    pub _kind: PhantomData<T>,
-}
-
-impl Settings<Checked> {
-    pub fn cleared() -> Settings<Checked> {
-        Settings {
-            displayed_attributes: Setting::Reset,
-            searchable_attributes: Setting::Reset,
-            filterable_attributes: Setting::Reset,
-            sortable_attributes: Setting::Reset,
-            ranking_rules: Setting::Reset,
-            stop_words: Setting::Reset,
-            synonyms: Setting::Reset,
-            distinct_attribute: Setting::Reset,
-            _kind: PhantomData,
-        }
-    }
-
-    pub fn into_unchecked(self) -> Settings<Unchecked> {
-        let Self {
-            displayed_attributes,
-            searchable_attributes,
-            filterable_attributes,
-            sortable_attributes,
-            ranking_rules,
-            stop_words,
-            synonyms,
-            distinct_attribute,
-            ..
-        } = self;
-
-        Settings {
-            displayed_attributes,
-            searchable_attributes,
-            filterable_attributes,
-            sortable_attributes,
-            ranking_rules,
-            stop_words,
-            synonyms,
-            distinct_attribute,
-            _kind: PhantomData,
-        }
-    }
-}
-
-impl Settings<Unchecked> {
-    pub fn check(self) -> Settings<Checked> {
-        let displayed_attributes = match self.displayed_attributes {
-            Setting::Set(fields) => {
-                if fields.iter().any(|f| f == "*") {
-                    Setting::Reset
-                } else {
-                    Setting::Set(fields)
-                }
-            }
-            otherwise => otherwise,
-        };
-
-        let searchable_attributes = match self.searchable_attributes {
-            Setting::Set(fields) => {
-                if fields.iter().any(|f| f == "*") {
-                    Setting::Reset
-                } else {
-                    Setting::Set(fields)
-                }
-            }
-            otherwise => otherwise,
-        };
-
-        Settings {
-            displayed_attributes,
-            searchable_attributes,
-            filterable_attributes: self.filterable_attributes,
-            sortable_attributes: self.sortable_attributes,
-            ranking_rules: self.ranking_rules,
-            stop_words: self.stop_words,
-            synonyms: self.synonyms,
-            distinct_attribute: self.distinct_attribute,
-            _kind: PhantomData,
-        }
-    }
-}
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(deny_unknown_fields)]
-#[serde(rename_all = "camelCase")]
-pub struct Facets {
-    pub level_group_size: Option<NonZeroUsize>,
-    pub min_level_size: Option<NonZeroUsize>,
-}
-
-impl Index {
-    pub fn update_documents(
-        &self,
-        format: UpdateFormat,
-        method: IndexDocumentsMethod,
-        content: Option<impl io::Read>,
-        update_builder: UpdateBuilder,
-        primary_key: Option<&str>,
-    ) -> Result<UpdateResult> {
-        let mut txn = self.write_txn()?;
-        let result = self.update_documents_txn(
-            &mut txn,
-            format,
-            method,
-            content,
-            update_builder,
-            primary_key,
-        )?;
-        txn.commit()?;
-        Ok(result)
-    }
-
-    pub fn update_documents_txn<'a, 'b>(
-        &'a self,
-        txn: &mut heed::RwTxn<'a, 'b>,
-        format: UpdateFormat,
-        method: IndexDocumentsMethod,
-        content: Option<impl io::Read>,
-        update_builder: UpdateBuilder,
-        primary_key: Option<&str>,
-    ) -> Result<UpdateResult> {
-        trace!("performing document addition");
-
-        // Set the primary key if not set already, ignore if already set.
-        if let (None, Some(primary_key)) = (self.primary_key(txn)?, primary_key) {
-            let mut builder = UpdateBuilder::new(0).settings(txn, self);
-            builder.set_primary_key(primary_key.to_string());
-            builder.execute(|_, _| ())?;
-        }
-
-        let mut builder = update_builder.index_documents(txn, self);
-        builder.update_format(format);
-        builder.index_documents_method(method);
-
-        let indexing_callback =
-            |indexing_step, update_id| debug!("update {}: {:?}", update_id, indexing_step);
-
-        let gzipped = false;
-        let addition = match content {
-            Some(content) if gzipped => {
-                builder.execute(GzDecoder::new(content), indexing_callback)?
-            }
-            Some(content) => builder.execute(content, indexing_callback)?,
-            None => builder.execute(std::io::empty(), indexing_callback)?,
-        };
-
-        info!("document addition done: {:?}", addition);
-
-        Ok(UpdateResult::DocumentsAddition(addition))
-    }
-
-    pub fn clear_documents(&self, update_builder: UpdateBuilder) -> Result<UpdateResult> {
-        // We must use the write transaction of the update here.
-        let mut wtxn = self.write_txn()?;
-        let builder = update_builder.clear_documents(&mut wtxn, self);
-
-        let _count = builder.execute()?;
-
-        wtxn.commit()
-            .and(Ok(UpdateResult::Other))
-            .map_err(Into::into)
-    }
-
-    pub fn update_settings_txn<'a, 'b>(
-        &'a self,
-        txn: &mut heed::RwTxn<'a, 'b>,
-        settings: &Settings<Checked>,
-        update_builder: UpdateBuilder,
-    ) -> Result<UpdateResult> {
-        // We must use the write transaction of the update here.
-        let mut builder = update_builder.settings(txn, self);
-
-        match settings.searchable_attributes {
-            Setting::Set(ref names) => builder.set_searchable_fields(names.clone()),
-            Setting::Reset => builder.reset_searchable_fields(),
-            Setting::NotSet => (),
-        }
-
-        match settings.displayed_attributes {
-            Setting::Set(ref names) => builder.set_displayed_fields(names.clone()),
-            Setting::Reset => builder.reset_displayed_fields(),
-            Setting::NotSet => (),
-        }
-
-        match settings.filterable_attributes {
-            Setting::Set(ref facets) => {
-                builder.set_filterable_fields(facets.clone().into_iter().collect())
-            }
-            Setting::Reset => builder.reset_filterable_fields(),
-            Setting::NotSet => (),
-        }
-
-        match settings.sortable_attributes {
-            Setting::Set(ref fields) => {
-                builder.set_sortable_fields(fields.iter().cloned().collect())
-            }
-            Setting::Reset => {
-                // TODO we must use the reset_sortable_fields in a futur PR.
-                builder.set_sortable_fields(HashSet::new())
-            }
-            Setting::NotSet => (),
-        }
-
-        match settings.ranking_rules {
-            Setting::Set(ref criteria) => builder.set_criteria(criteria.clone()),
-            Setting::Reset => builder.reset_criteria(),
-            Setting::NotSet => (),
-        }
-
-        match settings.stop_words {
-            Setting::Set(ref stop_words) => builder.set_stop_words(stop_words.clone()),
-            Setting::Reset => builder.reset_stop_words(),
-            Setting::NotSet => (),
-        }
-
-        match settings.synonyms {
-            Setting::Set(ref synonyms) => {
-                builder.set_synonyms(synonyms.clone().into_iter().collect())
-            }
-            Setting::Reset => builder.reset_synonyms(),
-            Setting::NotSet => (),
-        }
-
-        match settings.distinct_attribute {
-            Setting::Set(ref attr) => builder.set_distinct_field(attr.clone()),
-            Setting::Reset => builder.reset_distinct_field(),
-            Setting::NotSet => (),
-        }
-
-        builder.execute(|indexing_step, update_id| {
-            debug!("update {}: {:?}", update_id, indexing_step)
-        })?;
-
-        Ok(UpdateResult::Other)
-    }
-
-    pub fn update_settings(
-        &self,
-        settings: &Settings<Checked>,
-        update_builder: UpdateBuilder,
-    ) -> Result<UpdateResult> {
-        let mut txn = self.write_txn()?;
-        let result = self.update_settings_txn(&mut txn, settings, update_builder)?;
-        txn.commit()?;
-        Ok(result)
-    }
-
-    pub fn delete_documents(
-        &self,
-        document_ids: &[String],
-        update_builder: UpdateBuilder,
-    ) -> Result<UpdateResult> {
-        let mut txn = self.write_txn()?;
-        let mut builder = update_builder.delete_documents(&mut txn, self)?;
-
-        // We ignore unexisting document ids
-        document_ids.iter().for_each(|id| {
-            builder.delete_external_id(id);
-        });
-
-        let deleted = builder.execute()?;
-        txn.commit()
-            .and(Ok(UpdateResult::DocumentDeletion { deleted }))
-            .map_err(Into::into)
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn test_setting_check() {
-        // test no changes
-        let settings = Settings {
-            displayed_attributes: Setting::Set(vec![String::from("hello")]),
-            searchable_attributes: Setting::Set(vec![String::from("hello")]),
-            filterable_attributes: Setting::NotSet,
-            sortable_attributes: Setting::NotSet,
-            ranking_rules: Setting::NotSet,
-            stop_words: Setting::NotSet,
-            synonyms: Setting::NotSet,
-            distinct_attribute: Setting::NotSet,
-            _kind: PhantomData::<Unchecked>,
-        };
-
-        let checked = settings.clone().check();
-        assert_eq!(settings.displayed_attributes, checked.displayed_attributes);
-        assert_eq!(
-            settings.searchable_attributes,
-            checked.searchable_attributes
-        );
-
-        // test wildcard
-        // test no changes
-        let settings = Settings {
-            displayed_attributes: Setting::Set(vec![String::from("*")]),
-            searchable_attributes: Setting::Set(vec![String::from("hello"), String::from("*")]),
-            filterable_attributes: Setting::NotSet,
-            sortable_attributes: Setting::NotSet,
-            ranking_rules: Setting::NotSet,
-            stop_words: Setting::NotSet,
-            synonyms: Setting::NotSet,
-            distinct_attribute: Setting::NotSet,
-            _kind: PhantomData::<Unchecked>,
-        };
-
-        let checked = settings.check();
-        assert_eq!(checked.displayed_attributes, Setting::Reset);
-        assert_eq!(checked.searchable_attributes, Setting::Reset);
-    }
-}

meilisearch-http/src/index_controller/dump_actor/actor.rs

@@ -1,157 +0,0 @@
-use std::collections::HashMap;
-use std::path::{Path, PathBuf};
-use std::sync::Arc;
-
-use async_stream::stream;
-use chrono::Utc;
-use futures::{lock::Mutex, stream::StreamExt};
-use log::{error, trace};
-use tokio::sync::{mpsc, oneshot, RwLock};
-use update_actor::UpdateActorHandle;
-use uuid_resolver::UuidResolverHandle;
-
-use super::error::{DumpActorError, Result};
-use super::{DumpInfo, DumpMsg, DumpStatus, DumpTask};
-use crate::index_controller::{update_actor, uuid_resolver};
-
-pub const CONCURRENT_DUMP_MSG: usize = 10;
-
-pub struct DumpActor<UuidResolver, Update> {
-    inbox: Option<mpsc::Receiver<DumpMsg>>,
-    uuid_resolver: UuidResolver,
-    update: Update,
-    dump_path: PathBuf,
-    lock: Arc<Mutex<()>>,
-    dump_infos: Arc<RwLock<HashMap<String, DumpInfo>>>,
-    update_db_size: usize,
-    index_db_size: usize,
-}
-
-/// Generate uid from creation date
-fn generate_uid() -> String {
-    Utc::now().format("%Y%m%d-%H%M%S%3f").to_string()
-}
-
-impl<UuidResolver, Update> DumpActor<UuidResolver, Update>
-where
-    UuidResolver: UuidResolverHandle + Send + Sync + Clone + 'static,
-    Update: UpdateActorHandle + Send + Sync + Clone + 'static,
-{
-    pub fn new(
-        inbox: mpsc::Receiver<DumpMsg>,
-        uuid_resolver: UuidResolver,
-        update: Update,
-        dump_path: impl AsRef<Path>,
-        index_db_size: usize,
-        update_db_size: usize,
-    ) -> Self {
-        let dump_infos = Arc::new(RwLock::new(HashMap::new()));
-        let lock = Arc::new(Mutex::new(()));
-        Self {
-            inbox: Some(inbox),
-            uuid_resolver,
-            update,
-            dump_path: dump_path.as_ref().into(),
-            dump_infos,
-            lock,
-            index_db_size,
-            update_db_size,
-        }
-    }
-
-    pub async fn run(mut self) {
-        trace!("Started dump actor.");
-
-        let mut inbox = self
-            .inbox
-            .take()
-            .expect("Dump Actor must have a inbox at this point.");
-
-        let stream = stream! {
-            loop {
-                match inbox.recv().await {
-                    Some(msg) => yield msg,
-                    None => break,
-                }
-            }
-        };
-
-        stream
-            .for_each_concurrent(Some(CONCURRENT_DUMP_MSG), |msg| self.handle_message(msg))
-            .await;
-
-        error!("Dump actor stopped.");
-    }
-
-    async fn handle_message(&self, msg: DumpMsg) {
-        use DumpMsg::*;
-
-        match msg {
-            CreateDump { ret } => {
-                let _ = self.handle_create_dump(ret).await;
-            }
-            DumpInfo { ret, uid } => {
-                let _ = ret.send(self.handle_dump_info(uid).await);
-            }
-        }
-    }
-
-    async fn handle_create_dump(&self, ret: oneshot::Sender<Result<DumpInfo>>) {
-        let uid = generate_uid();
-        let info = DumpInfo::new(uid.clone(), DumpStatus::InProgress);
-
-        let _lock = match self.lock.try_lock() {
-            Some(lock) => lock,
-            None => {
-                ret.send(Err(DumpActorError::DumpAlreadyRunning))
-                    .expect("Dump actor is dead");
-                return;
-            }
-        };
-
-        self.dump_infos
-            .write()
-            .await
-            .insert(uid.clone(), info.clone());
-
-        ret.send(Ok(info)).expect("Dump actor is dead");
-
-        let task = DumpTask {
-            path: self.dump_path.clone(),
-            uuid_resolver: self.uuid_resolver.clone(),
-            update_handle: self.update.clone(),
-            uid: uid.clone(),
-            update_db_size: self.update_db_size,
-            index_db_size: self.index_db_size,
-        };
-
-        let task_result = tokio::task::spawn(task.run()).await;
-
-        let mut dump_infos = self.dump_infos.write().await;
-        let dump_infos = dump_infos
-            .get_mut(&uid)
-            .expect("dump entry deleted while lock was acquired");
-
-        match task_result {
-            Ok(Ok(())) => {
-                dump_infos.done();
-                trace!("Dump succeed");
-            }
-            Ok(Err(e)) => {
-                dump_infos.with_error(e.to_string());
-                error!("Dump failed: {}", e);
-            }
-            Err(_) => {
-                dump_infos.with_error("Unexpected error while performing dump.".to_string());
-                error!("Dump panicked. Dump status set to failed");
-            }
-        };
-    }
-
-    async fn handle_dump_info(&self, uid: String) -> Result<DumpInfo> {
-        match self.dump_infos.read().await.get(&uid) {
-            Some(info) => Ok(info.clone()),
-            _ => Err(DumpActorError::DumpDoesNotExist(uid)),
-        }
-    }
-}

meilisearch-http/src/index_controller/dump_actor/error.rs

@@ -1,52 +0,0 @@
-use meilisearch_error::{Code, ErrorCode};
-
-use crate::index_controller::update_actor::error::UpdateActorError;
-use crate::index_controller::uuid_resolver::error::UuidResolverError;
-
-pub type Result<T> = std::result::Result<T, DumpActorError>;
-
-#[derive(thiserror::Error, Debug)]
-pub enum DumpActorError {
-    #[error("Another dump is already in progress")]
-    DumpAlreadyRunning,
-    #[error("Dump `{0}` not found")]
-    DumpDoesNotExist(String),
-    #[error("Internal error: {0}")]
-    Internal(Box<dyn std::error::Error + Send + Sync + 'static>),
-    #[error("{0}")]
-    UuidResolver(#[from] UuidResolverError),
-    #[error("{0}")]
-    UpdateActor(#[from] UpdateActorError),
-}
-
-macro_rules! internal_error {
-    ($($other:path), *) => {
-        $(
-            impl From<$other> for DumpActorError {
-                fn from(other: $other) -> Self {
-                    Self::Internal(Box::new(other))
-                }
-            }
-        )*
-    }
-}
-
-internal_error!(
-    heed::Error,
-    std::io::Error,
-    tokio::task::JoinError,
-    serde_json::error::Error,
-    tempfile::PersistError
-);
-
-impl ErrorCode for DumpActorError {
-    fn error_code(&self) -> Code {
-        match self {
-            DumpActorError::DumpAlreadyRunning => Code::DumpAlreadyInProgress,
-            DumpActorError::DumpDoesNotExist(_) => Code::NotFound,
-            DumpActorError::Internal(_) => Code::Internal,
-            DumpActorError::UuidResolver(e) => e.error_code(),
-            DumpActorError::UpdateActor(e) => e.error_code(),
-        }
-    }
-}

meilisearch-http/src/index_controller/dump_actor/handle_impl.rs

@@ -1,53 +0,0 @@
-use std::path::Path;
-
-use actix_web::web::Bytes;
-use tokio::sync::{mpsc, oneshot};
-
-use super::error::Result;
-use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg};
-
-#[derive(Clone)]
-pub struct DumpActorHandleImpl {
-    sender: mpsc::Sender<DumpMsg>,
-}
-
-#[async_trait::async_trait]
-impl DumpActorHandle for DumpActorHandleImpl {
-    async fn create_dump(&self) -> Result<DumpInfo> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = DumpMsg::CreateDump { ret };
-        let _ = self.sender.send(msg).await;
-        receiver.await.expect("IndexActor has been killed")
-    }
-
-    async fn dump_info(&self, uid: String) -> Result<DumpInfo> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = DumpMsg::DumpInfo { ret, uid };
-        let _ = self.sender.send(msg).await;
-        receiver.await.expect("IndexActor has been killed")
-    }
-}
-
-impl DumpActorHandleImpl {
-    pub fn new(
-        path: impl AsRef<Path>,
-        uuid_resolver: crate::index_controller::uuid_resolver::UuidResolverHandleImpl,
-        update: crate::index_controller::update_actor::UpdateActorHandleImpl<Bytes>,
-        index_db_size: usize,
-        update_db_size: usize,
-    ) -> anyhow::Result<Self> {
-        let (sender, receiver) = mpsc::channel(10);
-        let actor = DumpActor::new(
-            receiver,
-            uuid_resolver,
-            update,
-            path,
-            index_db_size,
-            update_db_size,
-        );
-
-        tokio::task::spawn(actor.run());
-
-        Ok(Self { sender })
-    }
-}

meilisearch-http/src/index_controller/dump_actor/loaders/mod.rs

@@ -1,2 +0,0 @@
-pub mod v1;
-pub mod v2;

meilisearch-http/src/index_controller/dump_actor/loaders/v1.rs

@@ -1,228 +0,0 @@
-use std::collections::{BTreeMap, BTreeSet};
-use std::fs::{create_dir_all, File};
-use std::io::BufRead;
-use std::marker::PhantomData;
-use std::path::Path;
-use std::sync::Arc;
-
-use heed::EnvOpenOptions;
-use log::{error, info, warn};
-use milli::update::{IndexDocumentsMethod, Setting, UpdateFormat};
-use serde::{Deserialize, Deserializer, Serialize};
-use uuid::Uuid;
-
-use crate::index_controller::{self, uuid_resolver::HeedUuidStore, IndexMetadata};
-use crate::index_controller::{asc_ranking_rule, desc_ranking_rule};
-use crate::{
-    index::{update_handler::UpdateHandler, Index, Unchecked},
-    option::IndexerOpts,
-};
-
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct MetadataV1 {
-    db_version: String,
-    indexes: Vec<IndexMetadata>,
-}
-
-impl MetadataV1 {
-    pub fn load_dump(
-        self,
-        src: impl AsRef<Path>,
-        dst: impl AsRef<Path>,
-        size: usize,
-        indexer_options: &IndexerOpts,
-    ) -> anyhow::Result<()> {
-        info!(
-            "Loading dump, dump database version: {}, dump version: V1",
-            self.db_version
-        );
-
-        let uuid_store = HeedUuidStore::new(&dst)?;
-        for index in self.indexes {
-            let uuid = Uuid::new_v4();
-            uuid_store.insert(index.uid.clone(), uuid)?;
-            let src = src.as_ref().join(index.uid);
-            load_index(
-                &src,
-                &dst,
-                uuid,
-                index.meta.primary_key.as_deref(),
-                size,
-                indexer_options,
-            )?;
-        }
-
-        Ok(())
-    }
-}
-
-pub fn deserialize_some<'de, T, D>(deserializer: D) -> std::result::Result<Option<T>, D::Error>
-where
-    T: Deserialize<'de>,
-    D: Deserializer<'de>,
-{
-    Deserialize::deserialize(deserializer).map(Some)
-}
-
-// These are the settings used in legacy meilisearch (<v0.21.0).
-#[derive(Default, Clone, Serialize, Deserialize, Debug)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-struct Settings {
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub ranking_rules: Option<Option<Vec<String>>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub distinct_attribute: Option<Option<String>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub searchable_attributes: Option<Option<Vec<String>>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub displayed_attributes: Option<Option<BTreeSet<String>>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub stop_words: Option<Option<BTreeSet<String>>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub synonyms: Option<Option<BTreeMap<String, Vec<String>>>>,
-    #[serde(default, deserialize_with = "deserialize_some")]
-    pub attributes_for_faceting: Option<Option<Vec<String>>>,
-}
-
-fn load_index(
-    src: impl AsRef<Path>,
-    dst: impl AsRef<Path>,
-    uuid: Uuid,
-    primary_key: Option<&str>,
-    size: usize,
-    indexer_options: &IndexerOpts,
-) -> anyhow::Result<()> {
-    let index_path = dst.as_ref().join(&format!("indexes/index-{}", uuid));
-
-    create_dir_all(&index_path)?;
-    let mut options = EnvOpenOptions::new();
-    options.map_size(size);
-    let index = milli::Index::new(options, index_path)?;
-    let index = Index(Arc::new(index));
-
-    // extract `settings.json` file and import content
-    let settings = import_settings(&src)?;
-    let settings: index_controller::Settings<Unchecked> = settings.into();
-
-    let mut txn = index.write_txn()?;
-
-    let handler = UpdateHandler::new(indexer_options)?;
-
-    index.update_settings_txn(&mut txn, &settings.check(), handler.update_builder(0))?;
-
-    let file = File::open(&src.as_ref().join("documents.jsonl"))?;
-    let mut reader = std::io::BufReader::new(file);
-    reader.fill_buf()?;
-    if !reader.buffer().is_empty() {
-        index.update_documents_txn(
-            &mut txn,
-            UpdateFormat::JsonStream,
-            IndexDocumentsMethod::ReplaceDocuments,
-            Some(reader),
-            handler.update_builder(0),
-            primary_key,
-        )?;
-    }
-
-    txn.commit()?;
-
-    // Finaly, we extract the original milli::Index and close it
-    Arc::try_unwrap(index.0)
-        .map_err(|_e| "Couldn't close the index properly")
-        .unwrap()
-        .prepare_for_closing()
-        .wait();
-
-    // Updates are ignored in dumps V1.
-
-    Ok(())
-}
-
-/// we need to **always** be able to convert the old settings to the settings currently being used
-impl From<Settings> for index_controller::Settings<Unchecked> {
-    fn from(settings: Settings) -> Self {
-        Self {
-            distinct_attribute: match settings.distinct_attribute {
-                Some(Some(attr)) => Setting::Set(attr),
-                Some(None) => Setting::Reset,
-                None => Setting::NotSet
-            },
-            // we need to convert the old `Vec<String>` into a `BTreeSet<String>`
-            displayed_attributes: match settings.displayed_attributes {
-                Some(Some(attrs)) => Setting::Set(attrs.into_iter().collect()),
-                Some(None) => Setting::Reset,
-                None => Setting::NotSet
-            },
-            searchable_attributes: match settings.searchable_attributes {
-                Some(Some(attrs)) => Setting::Set(attrs),
-                Some(None) => Setting::Reset,
-                None => Setting::NotSet
-            },
-            filterable_attributes: match settings.attributes_for_faceting {
-                Some(Some(attrs)) => Setting::Set(attrs.into_iter().collect()),
-                Some(None) => Setting::Reset,
-                None => Setting::NotSet
-            },
-            sortable_attributes: Setting::NotSet,
-            ranking_rules: match settings.ranking_rules {
-                Some(Some(ranking_rules)) => Setting::Set(ranking_rules.into_iter().filter_map(|criterion| {
-                    match criterion.as_str() {
-                        "words" | "typo" | "proximity" | "attribute" | "exactness" => Some(criterion),
-                        s if s.starts_with("asc") => asc_ranking_rule(s).map(|f| format!("{}:asc", f)),
-                        s if s.starts_with("desc") => desc_ranking_rule(s).map(|f| format!("{}:desc", f)),
-                        "wordsPosition" => {
-                            warn!("The criteria `attribute` and `wordsPosition` have been merged \
-                                into a single criterion `attribute` so `wordsPositon` will be \
-                                ignored");
-                            None
-                        }
-                        s => {
-                            error!("Unknown criterion found in the dump: `{}`, it will be ignored", s);
-                            None
-                        }
-                    }
-                }).collect()),
-                Some(None) => Setting::Reset,
-                None => Setting::NotSet
-            },
-            // we need to convert the old `Vec<String>` into a `BTreeSet<String>`
-            stop_words: match settings.stop_words {
-                Some(Some(stop_words)) => Setting::Set(stop_words.into_iter().collect()),
-                Some(None) => Setting::Reset,
-                None => Setting::NotSet
-            },
-            // we need to convert the old `Vec<String>` into a `BTreeMap<String>`
-            synonyms: match settings.synonyms {
-                Some(Some(synonyms)) => Setting::Set(synonyms.into_iter().collect()),
-                Some(None) => Setting::Reset,
-                None => Setting::NotSet
-            },
-            _kind: PhantomData,
-        }
-    }
-}
-
-/// Extract Settings from `settings.json` file present at provided `dir_path`
-fn import_settings(dir_path: impl AsRef<Path>) -> anyhow::Result<Settings> {
-    let path = dir_path.as_ref().join("settings.json");
-    let file = File::open(path)?;
-    let reader = std::io::BufReader::new(file);
-    let metadata = serde_json::from_reader(reader)?;
-
-    Ok(metadata)
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-
-    #[test]
-    fn settings_format_regression() {
-        let settings = Settings::default();
-        assert_eq!(
-            r##"{"rankingRules":null,"distinctAttribute":null,"searchableAttributes":null,"displayedAttributes":null,"stopWords":null,"synonyms":null,"attributesForFaceting":null}"##,
-            serde_json::to_string(&settings).unwrap()
-        );
-    }
-}

meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs

@@ -1,59 +0,0 @@
-use std::path::Path;
-
-use chrono::{DateTime, Utc};
-use log::info;
-use serde::{Deserialize, Serialize};
-
-use crate::index::Index;
-use crate::index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore};
-use crate::option::IndexerOpts;
-
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct MetadataV2 {
-    db_version: String,
-    index_db_size: usize,
-    update_db_size: usize,
-    dump_date: DateTime<Utc>,
-}
-
-impl MetadataV2 {
-    pub fn new(index_db_size: usize, update_db_size: usize) -> Self {
-        Self {
-            db_version: env!("CARGO_PKG_VERSION").to_string(),
-            index_db_size,
-            update_db_size,
-            dump_date: Utc::now(),
-        }
-    }
-
-    pub fn load_dump(
-        self,
-        src: impl AsRef<Path>,
-        dst: impl AsRef<Path>,
-        index_db_size: usize,
-        update_db_size: usize,
-        indexing_options: &IndexerOpts,
-    ) -> anyhow::Result<()> {
-        info!(
-            "Loading dump from {}, dump database version: {}, dump version: V2",
-            self.dump_date, self.db_version
-        );
-
-        info!("Loading index database.");
-        HeedUuidStore::load_dump(src.as_ref(), &dst)?;
-
-        info!("Loading updates.");
-        UpdateStore::load_dump(&src, &dst, update_db_size)?;
-
-        info!("Loading indexes.");
-        let indexes_path = src.as_ref().join("indexes");
-        let indexes = indexes_path.read_dir()?;
-        for index in indexes {
-            let index = index?;
-            Index::load_dump(&index.path(), &dst, index_db_size, indexing_options)?;
-        }
-
-        Ok(())
-    }
-}

meilisearch-http/src/index_controller/dump_actor/message.rs

@@ -1,14 +0,0 @@
-use tokio::sync::oneshot;
-
-use super::error::Result;
-use super::DumpInfo;
-
-pub enum DumpMsg {
-    CreateDump {
-        ret: oneshot::Sender<Result<DumpInfo>>,
-    },
-    DumpInfo {
-        uid: String,
-        ret: oneshot::Sender<Result<DumpInfo>>,
-    },
-}

meilisearch-http/src/index_controller/dump_actor/mod.rs

@@ -1,203 +0,0 @@
-use std::fs::File;
-use std::path::{Path, PathBuf};
-
-use anyhow::Context;
-use chrono::{DateTime, Utc};
-use log::{info, trace, warn};
-#[cfg(test)]
-use mockall::automock;
-use serde::{Deserialize, Serialize};
-use tokio::fs::create_dir_all;
-
-use loaders::v1::MetadataV1;
-use loaders::v2::MetadataV2;
-
-pub use actor::DumpActor;
-pub use handle_impl::*;
-pub use message::DumpMsg;
-
-use super::{update_actor::UpdateActorHandle, uuid_resolver::UuidResolverHandle};
-use crate::index_controller::dump_actor::error::DumpActorError;
-use crate::{helpers::compression, option::IndexerOpts};
-use error::Result;
-
-mod actor;
-pub mod error;
-mod handle_impl;
-mod loaders;
-mod message;
-
-const META_FILE_NAME: &str = "metadata.json";
-
-#[async_trait::async_trait]
-#[cfg_attr(test, automock)]
-pub trait DumpActorHandle {
-    /// Start the creation of a dump
-    /// Implementation: [handle_impl::DumpActorHandleImpl::create_dump]
-    async fn create_dump(&self) -> Result<DumpInfo>;
-
-    /// Return the status of an already created dump
-    /// Implementation: [handle_impl::DumpActorHandleImpl::dump_info]
-    async fn dump_info(&self, uid: String) -> Result<DumpInfo>;
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(tag = "dumpVersion")]
-pub enum Metadata {
-    V1(MetadataV1),
-    V2(MetadataV2),
-}
-
-impl Metadata {
-    pub fn new_v2(index_db_size: usize, update_db_size: usize) -> Self {
-        let meta = MetadataV2::new(index_db_size, update_db_size);
-        Self::V2(meta)
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
-#[serde(rename_all = "snake_case")]
-pub enum DumpStatus {
-    Done,
-    InProgress,
-    Failed,
-}
-
-#[derive(Debug, Serialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct DumpInfo {
-    pub uid: String,
-    pub status: DumpStatus,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub error: Option<String>,
-    started_at: DateTime<Utc>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    finished_at: Option<DateTime<Utc>>,
-}
-
-impl DumpInfo {
-    pub fn new(uid: String, status: DumpStatus) -> Self {
-        Self {
-            uid,
-            status,
-            error: None,
-            started_at: Utc::now(),
-            finished_at: None,
-        }
-    }
-
-    pub fn with_error(&mut self, error: String) {
-        self.status = DumpStatus::Failed;
-        self.finished_at = Some(Utc::now());
-        self.error = Some(error);
-    }
-
-    pub fn done(&mut self) {
-        self.finished_at = Some(Utc::now());
-        self.status = DumpStatus::Done;
-    }
-
-    pub fn dump_already_in_progress(&self) -> bool {
-        self.status == DumpStatus::InProgress
-    }
-}
-
-pub fn load_dump(
-    dst_path: impl AsRef<Path>,
-    src_path: impl AsRef<Path>,
-    index_db_size: usize,
-    update_db_size: usize,
-    indexer_opts: &IndexerOpts,
-) -> anyhow::Result<()> {
-    let tmp_src = tempfile::tempdir_in(".")?;
-    let tmp_src_path = tmp_src.path();
-
-    compression::from_tar_gz(&src_path, tmp_src_path)?;
-
-    let meta_path = tmp_src_path.join(META_FILE_NAME);
-    let mut meta_file = File::open(&meta_path)?;
-    let meta: Metadata = serde_json::from_reader(&mut meta_file)?;
-
-    let dst_dir = dst_path
-        .as_ref()
-        .parent()
-        .with_context(|| format!("Invalid db path: {}", dst_path.as_ref().display()))?;
-
-    let tmp_dst = tempfile::tempdir_in(dst_dir)?;
-
-    match meta {
-        Metadata::V1(meta) => {
-            meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size, indexer_opts)?
-        }
-        Metadata::V2(meta) => meta.load_dump(
-            &tmp_src_path,
-            tmp_dst.path(),
-            index_db_size,
-            update_db_size,
-            indexer_opts,
-        )?,
-    }
-    // Persist and atomically rename the db
-    let persisted_dump = tmp_dst.into_path();
-    if dst_path.as_ref().exists() {
-        warn!("Overwriting database at {}", dst_path.as_ref().display());
-        std::fs::remove_dir_all(&dst_path)?;
-    }
-
-    std::fs::rename(&persisted_dump, &dst_path)?;
-
-    Ok(())
-}
-
-struct DumpTask<U, P> {
-    path: PathBuf,
-    uuid_resolver: U,
-    update_handle: P,
-    uid: String,
-    update_db_size: usize,
-    index_db_size: usize,
-}
-
-impl<U, P> DumpTask<U, P>
-where
-    U: UuidResolverHandle + Send + Sync + Clone + 'static,
-    P: UpdateActorHandle + Send + Sync + Clone + 'static,
-{
-    async fn run(self) -> Result<()> {
-        trace!("Performing dump.");
-
-        create_dir_all(&self.path).await?;
-
-        let path_clone = self.path.clone();
-        let temp_dump_dir =
-            tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(path_clone)).await??;
-        let temp_dump_path = temp_dump_dir.path().to_owned();
-
-        let meta = Metadata::new_v2(self.index_db_size, self.update_db_size);
-        let meta_path = temp_dump_path.join(META_FILE_NAME);
-        let mut meta_file = File::create(&meta_path)?;
-        serde_json::to_writer(&mut meta_file, &meta)?;
-
-        let uuids = self.uuid_resolver.dump(temp_dump_path.clone()).await?;
-
-        self.update_handle
-            .dump(uuids, temp_dump_path.clone())
-            .await?;
-
-        let dump_path = tokio::task::spawn_blocking(move || -> Result<PathBuf> {
-            let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?;
-            compression::to_tar_gz(temp_dump_path, temp_dump_file.path())
-                .map_err(|e| DumpActorError::Internal(e.into()))?;
-
-            let dump_path = self.path.join(self.uid).with_extension("dump");
-            temp_dump_file.persist(&dump_path)?;
-
-            Ok(dump_path)
-        })
-        .await??;
-
-        info!("Created dump in {:?}.", dump_path);
-
-        Ok(())
-    }
-}

meilisearch-http/src/index_controller/error.rs

@@ -1,40 +0,0 @@
-use meilisearch_error::Code;
-use meilisearch_error::ErrorCode;
-
-use crate::index::error::IndexError;
-
-use super::dump_actor::error::DumpActorError;
-use super::index_actor::error::IndexActorError;
-use super::update_actor::error::UpdateActorError;
-use super::uuid_resolver::error::UuidResolverError;
-
-pub type Result<T> = std::result::Result<T, IndexControllerError>;
-
-#[derive(Debug, thiserror::Error)]
-pub enum IndexControllerError {
-    #[error("Index creation must have an uid")]
-    MissingUid,
-    #[error("{0}")]
-    Uuid(#[from] UuidResolverError),
-    #[error("{0}")]
-    IndexActor(#[from] IndexActorError),
-    #[error("{0}")]
-    UpdateActor(#[from] UpdateActorError),
-    #[error("{0}")]
-    DumpActor(#[from] DumpActorError),
-    #[error("{0}")]
-    IndexError(#[from] IndexError),
-}
-
-impl ErrorCode for IndexControllerError {
-    fn error_code(&self) -> Code {
-        match self {
-            IndexControllerError::MissingUid => Code::BadRequest,
-            IndexControllerError::Uuid(e) => e.error_code(),
-            IndexControllerError::IndexActor(e) => e.error_code(),
-            IndexControllerError::UpdateActor(e) => e.error_code(),
-            IndexControllerError::DumpActor(e) => e.error_code(),
-            IndexControllerError::IndexError(e) => e.error_code(),
-        }
-    }
-}

meilisearch-http/src/index_controller/index_actor/actor.rs

@@ -1,351 +0,0 @@
-use std::fs::File;
-use std::path::PathBuf;
-use std::sync::Arc;
-
-use async_stream::stream;
-use futures::stream::StreamExt;
-use heed::CompactionOption;
-use log::debug;
-use milli::update::UpdateBuilder;
-use tokio::task::spawn_blocking;
-use tokio::{fs, sync::mpsc};
-use uuid::Uuid;
-
-use crate::index::{
-    update_handler::UpdateHandler, Checked, Document, SearchQuery, SearchResult, Settings,
-};
-use crate::index_controller::{
-    get_arc_ownership_blocking, Failed, IndexStats, Processed, Processing,
-};
-use crate::option::IndexerOpts;
-
-use super::error::{IndexActorError, Result};
-use super::{IndexMeta, IndexMsg, IndexSettings, IndexStore};
-
-pub const CONCURRENT_INDEX_MSG: usize = 10;
-
-pub struct IndexActor<S> {
-    receiver: Option<mpsc::Receiver<IndexMsg>>,
-    update_handler: Arc<UpdateHandler>,
-    store: S,
-}
-
-impl<S: IndexStore + Sync + Send> IndexActor<S> {
-    pub fn new(
-        receiver: mpsc::Receiver<IndexMsg>,
-        store: S,
-        options: &IndexerOpts,
-    ) -> anyhow::Result<Self> {
-        let update_handler = UpdateHandler::new(options)?;
-        let update_handler = Arc::new(update_handler);
-        let receiver = Some(receiver);
-        Ok(Self {
-            receiver,
-            update_handler,
-            store,
-        })
-    }
-
-    /// `run` poll the write_receiver and read_receiver concurrently, but while messages send
-    /// through the read channel are processed concurrently, the messages sent through the write
-    /// channel are processed one at a time.
-    pub async fn run(mut self) {
-        let mut receiver = self
-            .receiver
-            .take()
-            .expect("Index Actor must have a inbox at this point.");
-
-        let stream = stream! {
-            loop {
-                match receiver.recv().await {
-                    Some(msg) => yield msg,
-                    None => break,
-                }
-            }
-        };
-
-        stream
-            .for_each_concurrent(Some(CONCURRENT_INDEX_MSG), |msg| self.handle_message(msg))
-            .await;
-    }
-
-    async fn handle_message(&self, msg: IndexMsg) {
-        use IndexMsg::*;
-        match msg {
-            CreateIndex {
-                uuid,
-                primary_key,
-                ret,
-            } => {
-                let _ = ret.send(self.handle_create_index(uuid, primary_key).await);
-            }
-            Update {
-                ret,
-                meta,
-                data,
-                uuid,
-            } => {
-                let _ = ret.send(self.handle_update(uuid, meta, data).await);
-            }
-            Search { ret, query, uuid } => {
-                let _ = ret.send(self.handle_search(uuid, query).await);
-            }
-            Settings { ret, uuid } => {
-                let _ = ret.send(self.handle_settings(uuid).await);
-            }
-            Documents {
-                ret,
-                uuid,
-                attributes_to_retrieve,
-                offset,
-                limit,
-            } => {
-                let _ = ret.send(
-                    self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve)
-                        .await,
-                );
-            }
-            Document {
-                uuid,
-                attributes_to_retrieve,
-                doc_id,
-                ret,
-            } => {
-                let _ = ret.send(
-                    self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve)
-                        .await,
-                );
-            }
-            Delete { uuid, ret } => {
-                let _ = ret.send(self.handle_delete(uuid).await);
-            }
-            GetMeta { uuid, ret } => {
-                let _ = ret.send(self.handle_get_meta(uuid).await);
-            }
-            UpdateIndex {
-                uuid,
-                index_settings,
-                ret,
-            } => {
-                let _ = ret.send(self.handle_update_index(uuid, index_settings).await);
-            }
-            Snapshot { uuid, path, ret } => {
-                let _ = ret.send(self.handle_snapshot(uuid, path).await);
-            }
-            Dump { uuid, path, ret } => {
-                let _ = ret.send(self.handle_dump(uuid, path).await);
-            }
-            GetStats { uuid, ret } => {
-                let _ = ret.send(self.handle_get_stats(uuid).await);
-            }
-        }
-    }
-
-    async fn handle_search(&self, uuid: Uuid, query: SearchQuery) -> Result<SearchResult> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-        let result = spawn_blocking(move || index.perform_search(query)).await??;
-        Ok(result)
-    }
-
-    async fn handle_create_index(
-        &self,
-        uuid: Uuid,
-        primary_key: Option<String>,
-    ) -> Result<IndexMeta> {
-        let index = self.store.create(uuid, primary_key).await?;
-        let meta = spawn_blocking(move || IndexMeta::new(&index)).await??;
-        Ok(meta)
-    }
-
-    async fn handle_update(
-        &self,
-        uuid: Uuid,
-        meta: Processing,
-        data: Option<File>,
-    ) -> Result<std::result::Result<Processed, Failed>> {
-        debug!("Processing update {}", meta.id());
-        let update_handler = self.update_handler.clone();
-        let index = match self.store.get(uuid).await? {
-            Some(index) => index,
-            None => self.store.create(uuid, None).await?,
-        };
-
-        Ok(spawn_blocking(move || update_handler.handle_update(meta, data, index)).await?)
-    }
-
-    async fn handle_settings(&self, uuid: Uuid) -> Result<Settings<Checked>> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-        let result = spawn_blocking(move || index.settings()).await??;
-        Ok(result)
-    }
-
-    async fn handle_fetch_documents(
-        &self,
-        uuid: Uuid,
-        offset: usize,
-        limit: usize,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Vec<Document>> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-        let result =
-            spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve))
-                .await??;
-
-        Ok(result)
-    }
-
-    async fn handle_fetch_document(
-        &self,
-        uuid: Uuid,
-        doc_id: String,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Document> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-
-        let result =
-            spawn_blocking(move || index.retrieve_document(doc_id, attributes_to_retrieve))
-                .await??;
-
-        Ok(result)
-    }
-
-    async fn handle_delete(&self, uuid: Uuid) -> Result<()> {
-        let index = self.store.delete(uuid).await?;
-
-        if let Some(index) = index {
-            tokio::task::spawn(async move {
-                let index = index.0;
-                let store = get_arc_ownership_blocking(index).await;
-                spawn_blocking(move || {
-                    store.prepare_for_closing().wait();
-                    debug!("Index closed");
-                });
-            });
-        }
-
-        Ok(())
-    }
-
-    async fn handle_get_meta(&self, uuid: Uuid) -> Result<IndexMeta> {
-        match self.store.get(uuid).await? {
-            Some(index) => {
-                let meta = spawn_blocking(move || IndexMeta::new(&index)).await??;
-                Ok(meta)
-            }
-            None => Err(IndexActorError::UnexistingIndex),
-        }
-    }
-
-    async fn handle_update_index(
-        &self,
-        uuid: Uuid,
-        index_settings: IndexSettings,
-    ) -> Result<IndexMeta> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-
-        let result = spawn_blocking(move || match index_settings.primary_key {
-            Some(primary_key) => {
-                let mut txn = index.write_txn()?;
-                if index.primary_key(&txn)?.is_some() {
-                    return Err(IndexActorError::ExistingPrimaryKey);
-                }
-                let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index);
-                builder.set_primary_key(primary_key);
-                builder.execute(|_, _| ())?;
-                let meta = IndexMeta::new_txn(&index, &txn)?;
-                txn.commit()?;
-                Ok(meta)
-            }
-            None => {
-                let meta = IndexMeta::new(&index)?;
-                Ok(meta)
-            }
-        })
-        .await??;
-
-        Ok(result)
-    }
-
-    async fn handle_snapshot(&self, uuid: Uuid, mut path: PathBuf) -> Result<()> {
-        use tokio::fs::create_dir_all;
-
-        path.push("indexes");
-        create_dir_all(&path).await?;
-
-        if let Some(index) = self.store.get(uuid).await? {
-            let mut index_path = path.join(format!("index-{}", uuid));
-
-            create_dir_all(&index_path).await?;
-
-            index_path.push("data.mdb");
-            spawn_blocking(move || -> Result<()> {
-                // Get write txn to wait for ongoing write transaction before snapshot.
-                let _txn = index.write_txn()?;
-                index
-                    .env
-                    .copy_to_path(index_path, CompactionOption::Enabled)?;
-                Ok(())
-            })
-            .await??;
-        }
-
-        Ok(())
-    }
-
-    /// Create a `documents.jsonl` and a `settings.json` in `path/uid/` with a dump of all the
-    /// documents and all the settings.
-    async fn handle_dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-
-        let path = path.join(format!("indexes/index-{}/", uuid));
-        fs::create_dir_all(&path).await?;
-
-        tokio::task::spawn_blocking(move || index.dump(path)).await??;
-
-        Ok(())
-    }
-
-    async fn handle_get_stats(&self, uuid: Uuid) -> Result<IndexStats> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-
-        spawn_blocking(move || {
-            let rtxn = index.read_txn()?;
-
-            Ok(IndexStats {
-                size: index.size(),
-                number_of_documents: index.number_of_documents(&rtxn)?,
-                is_indexing: None,
-                field_distribution: index.field_distribution(&rtxn)?,
-            })
-        })
-        .await?
-    }
-}

meilisearch-http/src/index_controller/index_actor/error.rs

@@ -1,48 +0,0 @@
-use meilisearch_error::{Code, ErrorCode};
-
-use crate::{error::MilliError, index::error::IndexError};
-
-pub type Result<T> = std::result::Result<T, IndexActorError>;
-
-#[derive(thiserror::Error, Debug)]
-pub enum IndexActorError {
-    #[error("{0}")]
-    IndexError(#[from] IndexError),
-    #[error("Index already exists")]
-    IndexAlreadyExists,
-    #[error("Index not found")]
-    UnexistingIndex,
-    #[error("A primary key is already present. It's impossible to update it")]
-    ExistingPrimaryKey,
-    #[error("Internal Error: {0}")]
-    Internal(Box<dyn std::error::Error + Send + Sync + 'static>),
-    #[error("{0}")]
-    Milli(#[from] milli::Error),
-}
-
-macro_rules! internal_error {
-    ($($other:path), *) => {
-        $(
-            impl From<$other> for IndexActorError {
-                fn from(other: $other) -> Self {
-                    Self::Internal(Box::new(other))
-                }
-            }
-        )*
-    }
-}
-
-internal_error!(heed::Error, tokio::task::JoinError, std::io::Error);
-
-impl ErrorCode for IndexActorError {
-    fn error_code(&self) -> Code {
-        match self {
-            IndexActorError::IndexError(e) => e.error_code(),
-            IndexActorError::IndexAlreadyExists => Code::IndexAlreadyExists,
-            IndexActorError::UnexistingIndex => Code::IndexNotFound,
-            IndexActorError::ExistingPrimaryKey => Code::PrimaryKeyAlreadyPresent,
-            IndexActorError::Internal(_) => Code::Internal,
-            IndexActorError::Milli(e) => MilliError(e).error_code(),
-        }
-    }
-}

meilisearch-http/src/index_controller/index_actor/handle_impl.rs

@@ -1,164 +0,0 @@
-use crate::option::IndexerOpts;
-use std::path::{Path, PathBuf};
-
-use tokio::sync::{mpsc, oneshot};
-use uuid::Uuid;
-
-use crate::{
-    index::Checked,
-    index_controller::{IndexSettings, IndexStats, Processing},
-};
-use crate::{
-    index::{Document, SearchQuery, SearchResult, Settings},
-    index_controller::{Failed, Processed},
-};
-
-use super::error::Result;
-use super::{IndexActor, IndexActorHandle, IndexMeta, IndexMsg, MapIndexStore};
-
-#[derive(Clone)]
-pub struct IndexActorHandleImpl {
-    sender: mpsc::Sender<IndexMsg>,
-}
-
-#[async_trait::async_trait]
-impl IndexActorHandle for IndexActorHandleImpl {
-    async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::CreateIndex {
-            ret,
-            uuid,
-            primary_key,
-        };
-        let _ = self.sender.send(msg).await;
-        receiver.await.expect("IndexActor has been killed")
-    }
-
-    async fn update(
-        &self,
-        uuid: Uuid,
-        meta: Processing,
-        data: Option<std::fs::File>,
-    ) -> Result<std::result::Result<Processed, Failed>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Update {
-            ret,
-            meta,
-            data,
-            uuid,
-        };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result<SearchResult> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Search { uuid, query, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn settings(&self, uuid: Uuid) -> Result<Settings<Checked>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Settings { uuid, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn documents(
-        &self,
-        uuid: Uuid,
-        offset: usize,
-        limit: usize,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Vec<Document>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Documents {
-            uuid,
-            ret,
-            offset,
-            attributes_to_retrieve,
-            limit,
-        };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn document(
-        &self,
-        uuid: Uuid,
-        doc_id: String,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Document> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Document {
-            uuid,
-            ret,
-            doc_id,
-            attributes_to_retrieve,
-        };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn delete(&self, uuid: Uuid) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Delete { uuid, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn get_index_meta(&self, uuid: Uuid) -> Result<IndexMeta> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::GetMeta { uuid, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result<IndexMeta> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::UpdateIndex {
-            uuid,
-            index_settings,
-            ret,
-        };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Snapshot { uuid, path, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Dump { uuid, path, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn get_index_stats(&self, uuid: Uuid) -> Result<IndexStats> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::GetStats { uuid, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-}
-
-impl IndexActorHandleImpl {
-    pub fn new(
-        path: impl AsRef<Path>,
-        index_size: usize,
-        options: &IndexerOpts,
-    ) -> anyhow::Result<Self> {
-        let (sender, receiver) = mpsc::channel(100);
-
-        let store = MapIndexStore::new(path, index_size);
-        let actor = IndexActor::new(receiver, store, options)?;
-        tokio::task::spawn(actor.run());
-        Ok(Self { sender })
-    }
-}

meilisearch-http/src/index_controller/index_actor/message.rs

@@ -1,74 +0,0 @@
-use std::path::PathBuf;
-
-use tokio::sync::oneshot;
-use uuid::Uuid;
-
-use super::error::Result as IndexResult;
-use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings};
-use crate::index_controller::{Failed, IndexStats, Processed, Processing};
-
-use super::{IndexMeta, IndexSettings};
-
-#[allow(clippy::large_enum_variant)]
-pub enum IndexMsg {
-    CreateIndex {
-        uuid: Uuid,
-        primary_key: Option<String>,
-        ret: oneshot::Sender<IndexResult<IndexMeta>>,
-    },
-    Update {
-        uuid: Uuid,
-        meta: Processing,
-        data: Option<std::fs::File>,
-        ret: oneshot::Sender<IndexResult<Result<Processed, Failed>>>,
-    },
-    Search {
-        uuid: Uuid,
-        query: SearchQuery,
-        ret: oneshot::Sender<IndexResult<SearchResult>>,
-    },
-    Settings {
-        uuid: Uuid,
-        ret: oneshot::Sender<IndexResult<Settings<Checked>>>,
-    },
-    Documents {
-        uuid: Uuid,
-        attributes_to_retrieve: Option<Vec<String>>,
-        offset: usize,
-        limit: usize,
-        ret: oneshot::Sender<IndexResult<Vec<Document>>>,
-    },
-    Document {
-        uuid: Uuid,
-        attributes_to_retrieve: Option<Vec<String>>,
-        doc_id: String,
-        ret: oneshot::Sender<IndexResult<Document>>,
-    },
-    Delete {
-        uuid: Uuid,
-        ret: oneshot::Sender<IndexResult<()>>,
-    },
-    GetMeta {
-        uuid: Uuid,
-        ret: oneshot::Sender<IndexResult<IndexMeta>>,
-    },
-    UpdateIndex {
-        uuid: Uuid,
-        index_settings: IndexSettings,
-        ret: oneshot::Sender<IndexResult<IndexMeta>>,
-    },
-    Snapshot {
-        uuid: Uuid,
-        path: PathBuf,
-        ret: oneshot::Sender<IndexResult<()>>,
-    },
-    Dump {
-        uuid: Uuid,
-        path: PathBuf,
-        ret: oneshot::Sender<IndexResult<()>>,
-    },
-    GetStats {
-        uuid: Uuid,
-        ret: oneshot::Sender<IndexResult<IndexStats>>,
-    },
-}

meilisearch-http/src/index_controller/index_actor/mod.rs

@@ -1,169 +0,0 @@
-use std::fs::File;
-use std::path::PathBuf;
-
-use chrono::{DateTime, Utc};
-#[cfg(test)]
-use mockall::automock;
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-
-use actor::IndexActor;
-pub use actor::CONCURRENT_INDEX_MSG;
-pub use handle_impl::IndexActorHandleImpl;
-use message::IndexMsg;
-use store::{IndexStore, MapIndexStore};
-
-use crate::index::{Checked, Document, Index, SearchQuery, SearchResult, Settings};
-use crate::index_controller::{Failed, IndexStats, Processed, Processing};
-use error::Result;
-
-use super::IndexSettings;
-
-mod actor;
-pub mod error;
-mod handle_impl;
-mod message;
-mod store;
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct IndexMeta {
-    created_at: DateTime<Utc>,
-    pub updated_at: DateTime<Utc>,
-    pub primary_key: Option<String>,
-}
-
-impl IndexMeta {
-    fn new(index: &Index) -> Result<Self> {
-        let txn = index.read_txn()?;
-        Self::new_txn(index, &txn)
-    }
-
-    fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result<Self> {
-        let created_at = index.created_at(txn)?;
-        let updated_at = index.updated_at(txn)?;
-        let primary_key = index.primary_key(txn)?.map(String::from);
-        Ok(Self {
-            created_at,
-            updated_at,
-            primary_key,
-        })
-    }
-}
-
-#[async_trait::async_trait]
-#[cfg_attr(test, automock)]
-pub trait IndexActorHandle {
-    async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta>;
-    async fn update(
-        &self,
-        uuid: Uuid,
-        meta: Processing,
-        data: Option<File>,
-    ) -> Result<std::result::Result<Processed, Failed>>;
-    async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result<SearchResult>;
-    async fn settings(&self, uuid: Uuid) -> Result<Settings<Checked>>;
-
-    async fn documents(
-        &self,
-        uuid: Uuid,
-        offset: usize,
-        limit: usize,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Vec<Document>>;
-    async fn document(
-        &self,
-        uuid: Uuid,
-        doc_id: String,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Document>;
-    async fn delete(&self, uuid: Uuid) -> Result<()>;
-    async fn get_index_meta(&self, uuid: Uuid) -> Result<IndexMeta>;
-    async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result<IndexMeta>;
-    async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>;
-    async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()>;
-    async fn get_index_stats(&self, uuid: Uuid) -> Result<IndexStats>;
-}
-
-#[cfg(test)]
-mod test {
-    use std::sync::Arc;
-
-    use super::*;
-
-    #[async_trait::async_trait]
-    /// Useful for passing around an `Arc<MockIndexActorHandle>` in tests.
-    impl IndexActorHandle for Arc<MockIndexActorHandle> {
-        async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta> {
-            self.as_ref().create_index(uuid, primary_key).await
-        }
-
-        async fn update(
-            &self,
-            uuid: Uuid,
-            meta: Processing,
-            data: Option<std::fs::File>,
-        ) -> Result<std::result::Result<Processed, Failed>> {
-            self.as_ref().update(uuid, meta, data).await
-        }
-
-        async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result<SearchResult> {
-            self.as_ref().search(uuid, query).await
-        }
-
-        async fn settings(&self, uuid: Uuid) -> Result<Settings<Checked>> {
-            self.as_ref().settings(uuid).await
-        }
-
-        async fn documents(
-            &self,
-            uuid: Uuid,
-            offset: usize,
-            limit: usize,
-            attributes_to_retrieve: Option<Vec<String>>,
-        ) -> Result<Vec<Document>> {
-            self.as_ref()
-                .documents(uuid, offset, limit, attributes_to_retrieve)
-                .await
-        }
-
-        async fn document(
-            &self,
-            uuid: Uuid,
-            doc_id: String,
-            attributes_to_retrieve: Option<Vec<String>>,
-        ) -> Result<Document> {
-            self.as_ref()
-                .document(uuid, doc_id, attributes_to_retrieve)
-                .await
-        }
-
-        async fn delete(&self, uuid: Uuid) -> Result<()> {
-            self.as_ref().delete(uuid).await
-        }
-
-        async fn get_index_meta(&self, uuid: Uuid) -> Result<IndexMeta> {
-            self.as_ref().get_index_meta(uuid).await
-        }
-
-        async fn update_index(
-            &self,
-            uuid: Uuid,
-            index_settings: IndexSettings,
-        ) -> Result<IndexMeta> {
-            self.as_ref().update_index(uuid, index_settings).await
-        }
-
-        async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
-            self.as_ref().snapshot(uuid, path).await
-        }
-
-        async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
-            self.as_ref().dump(uuid, path).await
-        }
-
-        async fn get_index_stats(&self, uuid: Uuid) -> Result<IndexStats> {
-            self.as_ref().get_index_stats(uuid).await
-        }
-    }
-}

meilisearch-http/src/index_controller/mod.rs

@@ -1,456 +0,0 @@
-use std::collections::BTreeMap;
-use std::path::Path;
-use std::sync::Arc;
-use std::time::Duration;
-
-use actix_web::web::Bytes;
-use chrono::{DateTime, Utc};
-use futures::stream::StreamExt;
-use log::error;
-use log::info;
-use milli::FieldDistribution;
-use serde::{Deserialize, Serialize};
-use tokio::sync::mpsc;
-use tokio::time::sleep;
-use uuid::Uuid;
-
-use dump_actor::DumpActorHandle;
-pub use dump_actor::{DumpInfo, DumpStatus};
-use index_actor::IndexActorHandle;
-use snapshot::{load_snapshot, SnapshotService};
-use update_actor::UpdateActorHandle;
-pub use updates::*;
-use uuid_resolver::{error::UuidResolverError, UuidResolverHandle};
-
-use crate::extractors::payload::Payload;
-use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings};
-use crate::option::Opt;
-use error::Result;
-
-use self::dump_actor::load_dump;
-use self::error::IndexControllerError;
-
-mod dump_actor;
-pub mod error;
-pub mod index_actor;
-mod snapshot;
-mod update_actor;
-mod updates;
-mod uuid_resolver;
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct IndexMetadata {
-    #[serde(skip)]
-    pub uuid: Uuid,
-    pub uid: String,
-    name: String,
-    #[serde(flatten)]
-    pub meta: index_actor::IndexMeta,
-}
-
-#[derive(Clone, Debug)]
-pub struct IndexSettings {
-    pub uid: Option<String>,
-    pub primary_key: Option<String>,
-}
-
-#[derive(Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct IndexStats {
-    #[serde(skip)]
-    pub size: u64,
-    pub number_of_documents: u64,
-    /// Whether the current index is performing an update. It is initially `None` when the
-    /// index returns it, since it is the `UpdateStore` that knows what index is currently indexing. It is
-    /// later set to either true or false, we we retrieve the information from the `UpdateStore`
-    pub is_indexing: Option<bool>,
-    pub field_distribution: FieldDistribution,
-}
-
-#[derive(Clone)]
-pub struct IndexController {
-    uuid_resolver: uuid_resolver::UuidResolverHandleImpl,
-    index_handle: index_actor::IndexActorHandleImpl,
-    update_handle: update_actor::UpdateActorHandleImpl<Bytes>,
-    dump_handle: dump_actor::DumpActorHandleImpl,
-}
-
-#[derive(Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct Stats {
-    pub database_size: u64,
-    pub last_update: Option<DateTime<Utc>>,
-    pub indexes: BTreeMap<String, IndexStats>,
-}
-
-impl IndexController {
-    pub fn new(path: impl AsRef<Path>, options: &Opt) -> anyhow::Result<Self> {
-        let index_size = options.max_index_size.get_bytes() as usize;
-        let update_store_size = options.max_index_size.get_bytes() as usize;
-
-        if let Some(ref path) = options.import_snapshot {
-            info!("Loading from snapshot {:?}", path);
-            load_snapshot(
-                &options.db_path,
-                path,
-                options.ignore_snapshot_if_db_exists,
-                options.ignore_missing_snapshot,
-            )?;
-        } else if let Some(ref src_path) = options.import_dump {
-            load_dump(
-                &options.db_path,
-                src_path,
-                options.max_index_size.get_bytes() as usize,
-                options.max_udb_size.get_bytes() as usize,
-                &options.indexer_options,
-            )?;
-        }
-
-        std::fs::create_dir_all(&path)?;
-
-        let uuid_resolver = uuid_resolver::UuidResolverHandleImpl::new(&path)?;
-        let index_handle =
-            index_actor::IndexActorHandleImpl::new(&path, index_size, &options.indexer_options)?;
-        let update_handle = update_actor::UpdateActorHandleImpl::new(
-            index_handle.clone(),
-            &path,
-            update_store_size,
-        )?;
-        let dump_handle = dump_actor::DumpActorHandleImpl::new(
-            &options.dumps_dir,
-            uuid_resolver.clone(),
-            update_handle.clone(),
-            options.max_index_size.get_bytes() as usize,
-            options.max_udb_size.get_bytes() as usize,
-        )?;
-
-        if options.schedule_snapshot {
-            let snapshot_service = SnapshotService::new(
-                uuid_resolver.clone(),
-                update_handle.clone(),
-                Duration::from_secs(options.snapshot_interval_sec),
-                options.snapshot_dir.clone(),
-                options
-                    .db_path
-                    .file_name()
-                    .map(|n| n.to_owned().into_string().expect("invalid path"))
-                    .unwrap_or_else(|| String::from("data.ms")),
-            );
-
-            tokio::task::spawn(snapshot_service.run());
-        }
-
-        Ok(Self {
-            uuid_resolver,
-            index_handle,
-            update_handle,
-            dump_handle,
-        })
-    }
-
-    pub async fn add_documents(
-        &self,
-        uid: String,
-        method: milli::update::IndexDocumentsMethod,
-        format: milli::update::UpdateFormat,
-        payload: Payload,
-        primary_key: Option<String>,
-    ) -> Result<UpdateStatus> {
-        let perform_update = |uuid| async move {
-            let meta = UpdateMeta::DocumentsAddition {
-                method,
-                format,
-                primary_key,
-            };
-            let (sender, receiver) = mpsc::channel(10);
-
-            // It is necessary to spawn a local task to send the payload to the update handle to
-            // prevent dead_locking between the update_handle::update that waits for the update to be
-            // registered and the update_actor that waits for the the payload to be sent to it.
-            tokio::task::spawn_local(async move {
-                payload
-                    .for_each(|r| async {
-                        let _ = sender.send(r).await;
-                    })
-                    .await
-            });
-
-            // This must be done *AFTER* spawning the task.
-            self.update_handle.update(meta, receiver, uuid).await
-        };
-
-        match self.uuid_resolver.get(uid).await {
-            Ok(uuid) => Ok(perform_update(uuid).await?),
-            Err(UuidResolverError::UnexistingIndex(name)) => {
-                let uuid = Uuid::new_v4();
-                let status = perform_update(uuid).await?;
-                // ignore if index creation fails now, since it may already have been created
-                let _ = self.index_handle.create_index(uuid, None).await;
-                self.uuid_resolver.insert(name, uuid).await?;
-                Ok(status)
-            }
-            Err(e) => Err(e.into()),
-        }
-    }
-
-    pub async fn clear_documents(&self, uid: String) -> Result<UpdateStatus> {
-        let uuid = self.uuid_resolver.get(uid).await?;
-        let meta = UpdateMeta::ClearDocuments;
-        let (_, receiver) = mpsc::channel(1);
-        let status = self.update_handle.update(meta, receiver, uuid).await?;
-        Ok(status)
-    }
-
-    pub async fn delete_documents(
-        &self,
-        uid: String,
-        documents: Vec<String>,
-    ) -> Result<UpdateStatus> {
-        let uuid = self.uuid_resolver.get(uid).await?;
-        let meta = UpdateMeta::DeleteDocuments { ids: documents };
-        let (_, receiver) = mpsc::channel(1);
-        let status = self.update_handle.update(meta, receiver, uuid).await?;
-        Ok(status)
-    }
-
-    pub async fn update_settings(
-        &self,
-        uid: String,
-        settings: Settings<Checked>,
-        create: bool,
-    ) -> Result<UpdateStatus> {
-        let perform_udpate = |uuid| async move {
-            let meta = UpdateMeta::Settings(settings.into_unchecked());
-            // Nothing so send, drop the sender right away, as not to block the update actor.
-            let (_, receiver) = mpsc::channel(1);
-            self.update_handle.update(meta, receiver, uuid).await
-        };
-
-        match self.uuid_resolver.get(uid).await {
-            Ok(uuid) => Ok(perform_udpate(uuid).await?),
-            Err(UuidResolverError::UnexistingIndex(name)) if create => {
-                let uuid = Uuid::new_v4();
-                let status = perform_udpate(uuid).await?;
-                // ignore if index creation fails now, since it may already have been created
-                let _ = self.index_handle.create_index(uuid, None).await;
-                self.uuid_resolver.insert(name, uuid).await?;
-                Ok(status)
-            }
-            Err(e) => Err(e.into()),
-        }
-    }
-
-    pub async fn create_index(&self, index_settings: IndexSettings) -> Result<IndexMetadata> {
-        let IndexSettings { uid, primary_key } = index_settings;
-        let uid = uid.ok_or(IndexControllerError::MissingUid)?;
-        let uuid = Uuid::new_v4();
-        let meta = self.index_handle.create_index(uuid, primary_key).await?;
-        self.uuid_resolver.insert(uid.clone(), uuid).await?;
-        let meta = IndexMetadata {
-            uuid,
-            name: uid.clone(),
-            uid,
-            meta,
-        };
-
-        Ok(meta)
-    }
-
-    pub async fn delete_index(&self, uid: String) -> Result<()> {
-        let uuid = self.uuid_resolver.delete(uid).await?;
-
-        // We remove the index from the resolver synchronously, and effectively perform the index
-        // deletion as a background task.
-        let update_handle = self.update_handle.clone();
-        let index_handle = self.index_handle.clone();
-        tokio::spawn(async move {
-            if let Err(e) = update_handle.delete(uuid).await {
-                error!("Error while deleting index: {}", e);
-            }
-            if let Err(e) = index_handle.delete(uuid).await {
-                error!("Error while deleting index: {}", e);
-            }
-        });
-
-        Ok(())
-    }
-
-    pub async fn update_status(&self, uid: String, id: u64) -> Result<UpdateStatus> {
-        let uuid = self.uuid_resolver.get(uid).await?;
-        let result = self.update_handle.update_status(uuid, id).await?;
-        Ok(result)
-    }
-
-    pub async fn all_update_status(&self, uid: String) -> Result<Vec<UpdateStatus>> {
-        let uuid = self.uuid_resolver.get(uid).await?;
-        let result = self.update_handle.get_all_updates_status(uuid).await?;
-        Ok(result)
-    }
-
-    pub async fn list_indexes(&self) -> Result<Vec<IndexMetadata>> {
-        let uuids = self.uuid_resolver.list().await?;
-
-        let mut ret = Vec::new();
-
-        for (uid, uuid) in uuids {
-            let meta = self.index_handle.get_index_meta(uuid).await?;
-            let meta = IndexMetadata {
-                uuid,
-                name: uid.clone(),
-                uid,
-                meta,
-            };
-            ret.push(meta);
-        }
-
-        Ok(ret)
-    }
-
-    pub async fn settings(&self, uid: String) -> Result<Settings<Checked>> {
-        let uuid = self.uuid_resolver.get(uid.clone()).await?;
-        let settings = self.index_handle.settings(uuid).await?;
-        Ok(settings)
-    }
-
-    pub async fn documents(
-        &self,
-        uid: String,
-        offset: usize,
-        limit: usize,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Vec<Document>> {
-        let uuid = self.uuid_resolver.get(uid.clone()).await?;
-        let documents = self
-            .index_handle
-            .documents(uuid, offset, limit, attributes_to_retrieve)
-            .await?;
-        Ok(documents)
-    }
-
-    pub async fn document(
-        &self,
-        uid: String,
-        doc_id: String,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Document> {
-        let uuid = self.uuid_resolver.get(uid.clone()).await?;
-        let document = self
-            .index_handle
-            .document(uuid, doc_id, attributes_to_retrieve)
-            .await?;
-        Ok(document)
-    }
-
-    pub async fn update_index(
-        &self,
-        uid: String,
-        mut index_settings: IndexSettings,
-    ) -> Result<IndexMetadata> {
-        if index_settings.uid.is_some() {
-            index_settings.uid.take();
-        }
-
-        let uuid = self.uuid_resolver.get(uid.clone()).await?;
-        let meta = self.index_handle.update_index(uuid, index_settings).await?;
-        let meta = IndexMetadata {
-            uuid,
-            name: uid.clone(),
-            uid,
-            meta,
-        };
-        Ok(meta)
-    }
-
-    pub async fn search(&self, uid: String, query: SearchQuery) -> Result<SearchResult> {
-        let uuid = self.uuid_resolver.get(uid).await?;
-        let result = self.index_handle.search(uuid, query).await?;
-        Ok(result)
-    }
-
-    pub async fn get_index(&self, uid: String) -> Result<IndexMetadata> {
-        let uuid = self.uuid_resolver.get(uid.clone()).await?;
-        let meta = self.index_handle.get_index_meta(uuid).await?;
-        let meta = IndexMetadata {
-            uuid,
-            name: uid.clone(),
-            uid,
-            meta,
-        };
-        Ok(meta)
-    }
-
-    pub async fn get_uuids_size(&self) -> Result<u64> {
-        Ok(self.uuid_resolver.get_size().await?)
-    }
-
-    pub async fn get_index_stats(&self, uid: String) -> Result<IndexStats> {
-        let uuid = self.uuid_resolver.get(uid).await?;
-        let update_infos = self.update_handle.get_info().await?;
-        let mut stats = self.index_handle.get_index_stats(uuid).await?;
-        // Check if the currently indexing update is from out index.
-        stats.is_indexing = Some(Some(uuid) == update_infos.processing);
-        Ok(stats)
-    }
-
-    pub async fn get_all_stats(&self) -> Result<Stats> {
-        let update_infos = self.update_handle.get_info().await?;
-        let mut database_size = self.get_uuids_size().await? + update_infos.size;
-        let mut last_update: Option<DateTime<_>> = None;
-        let mut indexes = BTreeMap::new();
-
-        for index in self.list_indexes().await? {
-            let mut index_stats = self.index_handle.get_index_stats(index.uuid).await?;
-            database_size += index_stats.size;
-
-            last_update = last_update.map_or(Some(index.meta.updated_at), |last| {
-                Some(last.max(index.meta.updated_at))
-            });
-
-            index_stats.is_indexing = Some(Some(index.uuid) == update_infos.processing);
-
-            indexes.insert(index.uid, index_stats);
-        }
-
-        Ok(Stats {
-            database_size,
-            last_update,
-            indexes,
-        })
-    }
-
-    pub async fn create_dump(&self) -> Result<DumpInfo> {
-        Ok(self.dump_handle.create_dump().await?)
-    }
-
-    pub async fn dump_info(&self, uid: String) -> Result<DumpInfo> {
-        Ok(self.dump_handle.dump_info(uid).await?)
-    }
-}
-
-pub async fn get_arc_ownership_blocking<T>(mut item: Arc<T>) -> T {
-    loop {
-        match Arc::try_unwrap(item) {
-            Ok(item) => return item,
-            Err(item_arc) => {
-                item = item_arc;
-                sleep(Duration::from_millis(100)).await;
-                continue;
-            }
-        }
-    }
-}
-
-/// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name.
-pub fn asc_ranking_rule(text: &str) -> Option<&str> {
-    text.split_once("asc(")
-        .and_then(|(_, tail)| tail.rsplit_once(")"))
-        .map(|(field, _)| field)
-}
-
-/// Parses the v1 version of the Desc ranking rules `asc(price)`and returns the field name.
-pub fn desc_ranking_rule(text: &str) -> Option<&str> {
-    text.split_once("desc(")
-        .and_then(|(_, tail)| tail.rsplit_once(")"))
-        .map(|(field, _)| field)
-}

meilisearch-http/src/index_controller/snapshot.rs

@@ -1,268 +0,0 @@
-use std::path::{Path, PathBuf};
-use std::time::Duration;
-
-use anyhow::bail;
-use log::{error, info, trace};
-use tokio::fs;
-use tokio::task::spawn_blocking;
-use tokio::time::sleep;
-
-use super::update_actor::UpdateActorHandle;
-use super::uuid_resolver::UuidResolverHandle;
-use crate::helpers::compression;
-
-pub struct SnapshotService<U, R> {
-    uuid_resolver_handle: R,
-    update_handle: U,
-    snapshot_period: Duration,
-    snapshot_path: PathBuf,
-    db_name: String,
-}
-
-impl<U, R> SnapshotService<U, R>
-where
-    U: UpdateActorHandle,
-    R: UuidResolverHandle,
-{
-    pub fn new(
-        uuid_resolver_handle: R,
-        update_handle: U,
-        snapshot_period: Duration,
-        snapshot_path: PathBuf,
-        db_name: String,
-    ) -> Self {
-        Self {
-            uuid_resolver_handle,
-            update_handle,
-            snapshot_period,
-            snapshot_path,
-            db_name,
-        }
-    }
-
-    pub async fn run(self) {
-        info!(
-            "Snapshot scheduled every {}s.",
-            self.snapshot_period.as_secs()
-        );
-        loop {
-            if let Err(e) = self.perform_snapshot().await {
-                error!("Error while performing snapshot: {}", e);
-            }
-            sleep(self.snapshot_period).await;
-        }
-    }
-
-    async fn perform_snapshot(&self) -> anyhow::Result<()> {
-        trace!("Performing snapshot.");
-
-        let snapshot_dir = self.snapshot_path.clone();
-        fs::create_dir_all(&snapshot_dir).await?;
-        let temp_snapshot_dir =
-            spawn_blocking(move || tempfile::tempdir_in(snapshot_dir)).await??;
-        let temp_snapshot_path = temp_snapshot_dir.path().to_owned();
-
-        let uuids = self
-            .uuid_resolver_handle
-            .snapshot(temp_snapshot_path.clone())
-            .await?;
-
-        if uuids.is_empty() {
-            return Ok(());
-        }
-
-        self.update_handle
-            .snapshot(uuids, temp_snapshot_path.clone())
-            .await?;
-        let snapshot_dir = self.snapshot_path.clone();
-        let snapshot_path = self
-            .snapshot_path
-            .join(format!("{}.snapshot", self.db_name));
-        let snapshot_path = spawn_blocking(move || -> anyhow::Result<PathBuf> {
-            let temp_snapshot_file = tempfile::NamedTempFile::new_in(snapshot_dir)?;
-            let temp_snapshot_file_path = temp_snapshot_file.path().to_owned();
-            compression::to_tar_gz(temp_snapshot_path, temp_snapshot_file_path)?;
-            temp_snapshot_file.persist(&snapshot_path)?;
-            Ok(snapshot_path)
-        })
-        .await??;
-
-        trace!("Created snapshot in {:?}.", snapshot_path);
-
-        Ok(())
-    }
-}
-
-pub fn load_snapshot(
-    db_path: impl AsRef<Path>,
-    snapshot_path: impl AsRef<Path>,
-    ignore_snapshot_if_db_exists: bool,
-    ignore_missing_snapshot: bool,
-) -> anyhow::Result<()> {
-    if !db_path.as_ref().exists() && snapshot_path.as_ref().exists() {
-        match compression::from_tar_gz(snapshot_path, &db_path) {
-            Ok(()) => Ok(()),
-            Err(e) => {
-                // clean created db folder
-                std::fs::remove_dir_all(&db_path)?;
-                Err(e)
-            }
-        }
-    } else if db_path.as_ref().exists() && !ignore_snapshot_if_db_exists {
-        bail!(
-            "database already exists at {:?}, try to delete it or rename it",
-            db_path
-                .as_ref()
-                .canonicalize()
-                .unwrap_or_else(|_| db_path.as_ref().to_owned())
-        )
-    } else if !snapshot_path.as_ref().exists() && !ignore_missing_snapshot {
-        bail!(
-            "snapshot doesn't exist at {:?}",
-            snapshot_path
-                .as_ref()
-                .canonicalize()
-                .unwrap_or_else(|_| snapshot_path.as_ref().to_owned())
-        )
-    } else {
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use std::iter::FromIterator;
-    use std::{collections::HashSet, sync::Arc};
-
-    use futures::future::{err, ok};
-    use rand::Rng;
-    use tokio::time::timeout;
-    use uuid::Uuid;
-
-    use super::*;
-    use crate::index_controller::index_actor::MockIndexActorHandle;
-    use crate::index_controller::update_actor::{
-        error::UpdateActorError, MockUpdateActorHandle, UpdateActorHandleImpl,
-    };
-    use crate::index_controller::uuid_resolver::{
-        error::UuidResolverError, MockUuidResolverHandle,
-    };
-
-    #[actix_rt::test]
-    async fn test_normal() {
-        let mut rng = rand::thread_rng();
-        let uuids_num: usize = rng.gen_range(5, 10);
-        let uuids = (0..uuids_num)
-            .map(|_| Uuid::new_v4())
-            .collect::<HashSet<_>>();
-
-        let mut uuid_resolver = MockUuidResolverHandle::new();
-        let uuids_clone = uuids.clone();
-        uuid_resolver
-            .expect_snapshot()
-            .times(1)
-            .returning(move |_| Box::pin(ok(uuids_clone.clone())));
-
-        let uuids_clone = uuids.clone();
-        let mut index_handle = MockIndexActorHandle::new();
-        index_handle
-            .expect_snapshot()
-            .withf(move |uuid, _path| uuids_clone.contains(uuid))
-            .times(uuids_num)
-            .returning(move |_, _| Box::pin(ok(())));
-
-        let dir = tempfile::tempdir_in(".").unwrap();
-        let handle = Arc::new(index_handle);
-        let update_handle =
-            UpdateActorHandleImpl::<Vec<u8>>::new(handle.clone(), dir.path(), 4096 * 100).unwrap();
-
-        let snapshot_path = tempfile::tempdir_in(".").unwrap();
-        let snapshot_service = SnapshotService::new(
-            uuid_resolver,
-            update_handle,
-            Duration::from_millis(100),
-            snapshot_path.path().to_owned(),
-            "data.ms".to_string(),
-        );
-
-        snapshot_service.perform_snapshot().await.unwrap();
-    }
-
-    #[actix_rt::test]
-    async fn error_performing_uuid_snapshot() {
-        let mut uuid_resolver = MockUuidResolverHandle::new();
-        uuid_resolver
-            .expect_snapshot()
-            .times(1)
-            // abitrary error
-            .returning(|_| Box::pin(err(UuidResolverError::NameAlreadyExist)));
-
-        let update_handle = MockUpdateActorHandle::new();
-
-        let snapshot_path = tempfile::tempdir_in(".").unwrap();
-        let snapshot_service = SnapshotService::new(
-            uuid_resolver,
-            update_handle,
-            Duration::from_millis(100),
-            snapshot_path.path().to_owned(),
-            "data.ms".to_string(),
-        );
-
-        assert!(snapshot_service.perform_snapshot().await.is_err());
-        // Nothing was written to the file
-        assert!(!snapshot_path.path().join("data.ms.snapshot").exists());
-    }
-
-    #[actix_rt::test]
-    async fn error_performing_index_snapshot() {
-        let uuid = Uuid::new_v4();
-        let mut uuid_resolver = MockUuidResolverHandle::new();
-        uuid_resolver
-            .expect_snapshot()
-            .times(1)
-            .returning(move |_| Box::pin(ok(HashSet::from_iter(Some(uuid)))));
-
-        let mut update_handle = MockUpdateActorHandle::new();
-        update_handle
-            .expect_snapshot()
-            // abitrary error
-            .returning(|_, _| Box::pin(err(UpdateActorError::UnexistingUpdate(0))));
-
-        let snapshot_path = tempfile::tempdir_in(".").unwrap();
-        let snapshot_service = SnapshotService::new(
-            uuid_resolver,
-            update_handle,
-            Duration::from_millis(100),
-            snapshot_path.path().to_owned(),
-            "data.ms".to_string(),
-        );
-
-        assert!(snapshot_service.perform_snapshot().await.is_err());
-        // Nothing was written to the file
-        assert!(!snapshot_path.path().join("data.ms.snapshot").exists());
-    }
-
-    #[actix_rt::test]
-    async fn test_loop() {
-        let mut uuid_resolver = MockUuidResolverHandle::new();
-        uuid_resolver
-            .expect_snapshot()
-            // we expect the funtion to be called between 2 and 3 time in the given interval.
-            .times(2..4)
-            // abitrary error, to short-circuit the function
-            .returning(move |_| Box::pin(err(UuidResolverError::NameAlreadyExist)));
-
-        let update_handle = MockUpdateActorHandle::new();
-
-        let snapshot_path = tempfile::tempdir_in(".").unwrap();
-        let snapshot_service = SnapshotService::new(
-            uuid_resolver,
-            update_handle,
-            Duration::from_millis(100),
-            snapshot_path.path().to_owned(),
-            "data.ms".to_string(),
-        );
-
-        let _ = timeout(Duration::from_millis(300), snapshot_service.run()).await;
-    }
-}

meilisearch-http/src/index_controller/update_actor/actor.rs

@@ -1,270 +0,0 @@
-use std::collections::HashSet;
-use std::io::SeekFrom;
-use std::path::{Path, PathBuf};
-use std::sync::atomic::AtomicBool;
-use std::sync::Arc;
-
-use async_stream::stream;
-use futures::StreamExt;
-use log::trace;
-use serdeval::*;
-use tokio::fs;
-use tokio::io::AsyncWriteExt;
-use tokio::sync::mpsc;
-use uuid::Uuid;
-
-use super::error::{Result, UpdateActorError};
-use super::{PayloadData, UpdateMsg, UpdateStore, UpdateStoreInfo};
-use crate::index_controller::index_actor::IndexActorHandle;
-use crate::index_controller::{UpdateMeta, UpdateStatus};
-
-pub struct UpdateActor<D, I> {
-    path: PathBuf,
-    store: Arc<UpdateStore>,
-    inbox: Option<mpsc::Receiver<UpdateMsg<D>>>,
-    index_handle: I,
-    must_exit: Arc<AtomicBool>,
-}
-
-impl<D, I> UpdateActor<D, I>
-where
-    D: AsRef<[u8]> + Sized + 'static,
-    I: IndexActorHandle + Clone + Send + Sync + 'static,
-{
-    pub fn new(
-        update_db_size: usize,
-        inbox: mpsc::Receiver<UpdateMsg<D>>,
-        path: impl AsRef<Path>,
-        index_handle: I,
-    ) -> anyhow::Result<Self> {
-        let path = path.as_ref().join("updates");
-
-        std::fs::create_dir_all(&path)?;
-
-        let mut options = heed::EnvOpenOptions::new();
-        options.map_size(update_db_size);
-
-        let must_exit = Arc::new(AtomicBool::new(false));
-
-        let store = UpdateStore::open(options, &path, index_handle.clone(), must_exit.clone())?;
-        std::fs::create_dir_all(path.join("update_files"))?;
-        let inbox = Some(inbox);
-        Ok(Self {
-            path,
-            store,
-            inbox,
-            index_handle,
-            must_exit,
-        })
-    }
-
-    pub async fn run(mut self) {
-        use UpdateMsg::*;
-
-        trace!("Started update actor.");
-
-        let mut inbox = self
-            .inbox
-            .take()
-            .expect("A receiver should be present by now.");
-
-        let must_exit = self.must_exit.clone();
-        let stream = stream! {
-            loop {
-                let msg = inbox.recv().await;
-
-                if must_exit.load(std::sync::atomic::Ordering::Relaxed) {
-                    break;
-                }
-
-                match msg {
-                    Some(msg) => yield msg,
-                    None => break,
-                }
-            }
-        };
-
-        stream
-            .for_each_concurrent(Some(10), |msg| async {
-                match msg {
-                    Update {
-                        uuid,
-                        meta,
-                        data,
-                        ret,
-                    } => {
-                        let _ = ret.send(self.handle_update(uuid, meta, data).await);
-                    }
-                    ListUpdates { uuid, ret } => {
-                        let _ = ret.send(self.handle_list_updates(uuid).await);
-                    }
-                    GetUpdate { uuid, ret, id } => {
-                        let _ = ret.send(self.handle_get_update(uuid, id).await);
-                    }
-                    Delete { uuid, ret } => {
-                        let _ = ret.send(self.handle_delete(uuid).await);
-                    }
-                    Snapshot { uuids, path, ret } => {
-                        let _ = ret.send(self.handle_snapshot(uuids, path).await);
-                    }
-                    GetInfo { ret } => {
-                        let _ = ret.send(self.handle_get_info().await);
-                    }
-                    Dump { uuids, path, ret } => {
-                        let _ = ret.send(self.handle_dump(uuids, path).await);
-                    }
-                }
-            })
-            .await;
-    }
-
-    async fn handle_update(
-        &self,
-        uuid: Uuid,
-        meta: UpdateMeta,
-        payload: mpsc::Receiver<PayloadData<D>>,
-    ) -> Result<UpdateStatus> {
-        let file_path = match meta {
-            UpdateMeta::DocumentsAddition { .. } => {
-                let update_file_id = uuid::Uuid::new_v4();
-                let path = self
-                    .path
-                    .join(format!("update_files/update_{}", update_file_id));
-                let mut file = fs::OpenOptions::new()
-                    .read(true)
-                    .write(true)
-                    .create(true)
-                    .open(&path)
-                    .await?;
-
-                async fn write_to_file<D>(
-                    file: &mut fs::File,
-                    mut payload: mpsc::Receiver<PayloadData<D>>,
-                ) -> Result<usize>
-                where
-                    D: AsRef<[u8]> + Sized + 'static,
-                {
-                    let mut file_len = 0;
-
-                    while let Some(bytes) = payload.recv().await {
-                        let bytes = bytes?;
-                        file_len += bytes.as_ref().len();
-                        file.write_all(bytes.as_ref()).await?;
-                    }
-
-                    file.flush().await?;
-
-                    Ok(file_len)
-                }
-
-                let file_len = write_to_file(&mut file, payload).await;
-
-                match file_len {
-                    Ok(len) if len > 0 => {
-                        let file = file.into_std().await;
-                        Some((file, update_file_id))
-                    }
-                    Err(e) => {
-                        fs::remove_file(&path).await?;
-                        return Err(e);
-                    }
-                    _ => {
-                        fs::remove_file(&path).await?;
-                        None
-                    }
-                }
-            }
-            _ => None,
-        };
-
-        let update_store = self.store.clone();
-
-        tokio::task::spawn_blocking(move || {
-            use std::io::{BufReader, Seek};
-
-            // If the payload is empty, ignore the check.
-            let update_uuid = if let Some((mut file, uuid)) = file_path {
-                // set the file back to the beginning
-                file.seek(SeekFrom::Start(0))?;
-                // Check that the json payload is valid:
-                let reader = BufReader::new(&mut file);
-                // Validate that the payload is in the correct format.
-                let _: Seq<Map<Str, Any>> = serde_json::from_reader(reader)
-                    .map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?;
-
-                Some(uuid)
-            } else {
-                None
-            };
-
-            // The payload is valid, we can register it to the update store.
-            let status = update_store
-                .register_update(meta, update_uuid, uuid)
-                .map(UpdateStatus::Enqueued)?;
-            Ok(status)
-        })
-        .await?
-    }
-
-    async fn handle_list_updates(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>> {
-        let update_store = self.store.clone();
-        tokio::task::spawn_blocking(move || {
-            let result = update_store.list(uuid)?;
-            Ok(result)
-        })
-        .await?
-    }
-
-    async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus> {
-        let store = self.store.clone();
-        tokio::task::spawn_blocking(move || {
-            let result = store
-                .meta(uuid, id)?
-                .ok_or(UpdateActorError::UnexistingUpdate(id))?;
-            Ok(result)
-        })
-        .await?
-    }
-
-    async fn handle_delete(&self, uuid: Uuid) -> Result<()> {
-        let store = self.store.clone();
-
-        tokio::task::spawn_blocking(move || store.delete_all(uuid)).await??;
-
-        Ok(())
-    }
-
-    async fn handle_snapshot(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
-        let index_handle = self.index_handle.clone();
-        let update_store = self.store.clone();
-
-        tokio::task::spawn_blocking(move || update_store.snapshot(&uuids, &path, index_handle))
-            .await??;
-
-        Ok(())
-    }
-
-    async fn handle_dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
-        let index_handle = self.index_handle.clone();
-        let update_store = self.store.clone();
-
-        tokio::task::spawn_blocking(move || -> Result<()> {
-            update_store.dump(&uuids, path.to_path_buf(), index_handle)?;
-            Ok(())
-        })
-        .await??;
-
-        Ok(())
-    }
-
-    async fn handle_get_info(&self) -> Result<UpdateStoreInfo> {
-        let update_store = self.store.clone();
-        let info = tokio::task::spawn_blocking(move || -> Result<UpdateStoreInfo> {
-            let info = update_store.get_info()?;
-            Ok(info)
-        })
-        .await??;
-
-        Ok(info)
-    }
-}

meilisearch-http/src/index_controller/update_actor/error.rs

@@ -1,61 +0,0 @@
-use std::error::Error;
-
-use meilisearch_error::{Code, ErrorCode};
-
-use crate::index_controller::index_actor::error::IndexActorError;
-
-pub type Result<T> = std::result::Result<T, UpdateActorError>;
-
-#[derive(Debug, thiserror::Error)]
-#[allow(clippy::large_enum_variant)]
-pub enum UpdateActorError {
-    #[error("Update {0} not found.")]
-    UnexistingUpdate(u64),
-    #[error("Internal error: {0}")]
-    Internal(Box<dyn Error + Send + Sync + 'static>),
-    #[error("{0}")]
-    IndexActor(#[from] IndexActorError),
-    #[error(
-        "update store was shut down due to a fatal error, please check your logs for more info."
-    )]
-    FatalUpdateStoreError,
-    #[error("{0}")]
-    InvalidPayload(Box<dyn Error + Send + Sync + 'static>),
-    #[error("{0}")]
-    PayloadError(#[from] actix_web::error::PayloadError),
-}
-
-impl<T> From<tokio::sync::mpsc::error::SendError<T>> for UpdateActorError {
-    fn from(_: tokio::sync::mpsc::error::SendError<T>) -> Self {
-        Self::FatalUpdateStoreError
-    }
-}
-
-impl From<tokio::sync::oneshot::error::RecvError> for UpdateActorError {
-    fn from(_: tokio::sync::oneshot::error::RecvError) -> Self {
-        Self::FatalUpdateStoreError
-    }
-}
-
-internal_error!(
-    UpdateActorError: heed::Error,
-    std::io::Error,
-    serde_json::Error,
-    tokio::task::JoinError
-);
-
-impl ErrorCode for UpdateActorError {
-    fn error_code(&self) -> Code {
-        match self {
-            UpdateActorError::UnexistingUpdate(_) => Code::NotFound,
-            UpdateActorError::Internal(_) => Code::Internal,
-            UpdateActorError::IndexActor(e) => e.error_code(),
-            UpdateActorError::FatalUpdateStoreError => Code::Internal,
-            UpdateActorError::InvalidPayload(_) => Code::BadRequest,
-            UpdateActorError::PayloadError(error) => match error {
-                actix_http::error::PayloadError::Overflow => Code::PayloadTooLarge,
-                _ => Code::Internal,
-            },
-        }
-    }
-}

meilisearch-http/src/index_controller/update_actor/handle_impl.rs

@@ -1,103 +0,0 @@
-use std::collections::HashSet;
-use std::path::{Path, PathBuf};
-
-use tokio::sync::{mpsc, oneshot};
-use uuid::Uuid;
-
-use crate::index_controller::{IndexActorHandle, UpdateStatus};
-
-use super::error::Result;
-use super::{PayloadData, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStoreInfo};
-
-#[derive(Clone)]
-pub struct UpdateActorHandleImpl<D> {
-    sender: mpsc::Sender<UpdateMsg<D>>,
-}
-
-impl<D> UpdateActorHandleImpl<D>
-where
-    D: AsRef<[u8]> + Sized + 'static + Sync + Send,
-{
-    pub fn new<I>(
-        index_handle: I,
-        path: impl AsRef<Path>,
-        update_store_size: usize,
-    ) -> anyhow::Result<Self>
-    where
-        I: IndexActorHandle + Clone + Send + Sync + 'static,
-    {
-        let path = path.as_ref().to_owned();
-        let (sender, receiver) = mpsc::channel(100);
-        let actor = UpdateActor::new(update_store_size, receiver, path, index_handle)?;
-
-        tokio::task::spawn(actor.run());
-
-        Ok(Self { sender })
-    }
-}
-
-#[async_trait::async_trait]
-impl<D> UpdateActorHandle for UpdateActorHandleImpl<D>
-where
-    D: AsRef<[u8]> + Sized + 'static + Sync + Send,
-{
-    type Data = D;
-
-    async fn get_all_updates_status(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::ListUpdates { uuid, ret };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-    async fn update_status(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::GetUpdate { uuid, id, ret };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-
-    async fn delete(&self, uuid: Uuid) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::Delete { uuid, ret };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-
-    async fn snapshot(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::Snapshot { uuids, path, ret };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-
-    async fn dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::Dump { uuids, path, ret };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-
-    async fn get_info(&self) -> Result<UpdateStoreInfo> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::GetInfo { ret };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-
-    async fn update(
-        &self,
-        meta: UpdateMeta,
-        data: mpsc::Receiver<PayloadData<Self::Data>>,
-        uuid: Uuid,
-    ) -> Result<UpdateStatus> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::Update {
-            uuid,
-            data,
-            meta,
-            ret,
-        };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-}

meilisearch-http/src/index_controller/update_actor/message.rs

@@ -1,43 +0,0 @@
-use std::collections::HashSet;
-use std::path::PathBuf;
-
-use tokio::sync::{mpsc, oneshot};
-use uuid::Uuid;
-
-use super::error::Result;
-use super::{PayloadData, UpdateMeta, UpdateStatus, UpdateStoreInfo};
-
-pub enum UpdateMsg<D> {
-    Update {
-        uuid: Uuid,
-        meta: UpdateMeta,
-        data: mpsc::Receiver<PayloadData<D>>,
-        ret: oneshot::Sender<Result<UpdateStatus>>,
-    },
-    ListUpdates {
-        uuid: Uuid,
-        ret: oneshot::Sender<Result<Vec<UpdateStatus>>>,
-    },
-    GetUpdate {
-        uuid: Uuid,
-        ret: oneshot::Sender<Result<UpdateStatus>>,
-        id: u64,
-    },
-    Delete {
-        uuid: Uuid,
-        ret: oneshot::Sender<Result<()>>,
-    },
-    Snapshot {
-        uuids: HashSet<Uuid>,
-        path: PathBuf,
-        ret: oneshot::Sender<Result<()>>,
-    },
-    Dump {
-        uuids: HashSet<Uuid>,
-        path: PathBuf,
-        ret: oneshot::Sender<Result<()>>,
-    },
-    GetInfo {
-        ret: oneshot::Sender<Result<UpdateStoreInfo>>,
-    },
-}

meilisearch-http/src/index_controller/update_actor/mod.rs

@@ -1,44 +0,0 @@
-use std::{collections::HashSet, path::PathBuf};
-
-use actix_http::error::PayloadError;
-use tokio::sync::mpsc;
-use uuid::Uuid;
-
-use crate::index_controller::{UpdateMeta, UpdateStatus};
-
-use actor::UpdateActor;
-use error::Result;
-use message::UpdateMsg;
-
-pub use handle_impl::UpdateActorHandleImpl;
-pub use store::{UpdateStore, UpdateStoreInfo};
-
-mod actor;
-pub mod error;
-mod handle_impl;
-mod message;
-pub mod store;
-
-type PayloadData<D> = std::result::Result<D, PayloadError>;
-
-#[cfg(test)]
-use mockall::automock;
-
-#[async_trait::async_trait]
-#[cfg_attr(test, automock(type Data=Vec<u8>;))]
-pub trait UpdateActorHandle {
-    type Data: AsRef<[u8]> + Sized + 'static + Sync + Send;
-
-    async fn get_all_updates_status(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>>;
-    async fn update_status(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus>;
-    async fn delete(&self, uuid: Uuid) -> Result<()>;
-    async fn snapshot(&self, uuid: HashSet<Uuid>, path: PathBuf) -> Result<()>;
-    async fn dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()>;
-    async fn get_info(&self) -> Result<UpdateStoreInfo>;
-    async fn update(
-        &self,
-        meta: UpdateMeta,
-        data: mpsc::Receiver<PayloadData<Self::Data>>,
-        uuid: Uuid,
-    ) -> Result<UpdateStatus>;
-}

meilisearch-http/src/index_controller/update_actor/store/codec.rs

@@ -1,86 +0,0 @@
-use std::{borrow::Cow, convert::TryInto, mem::size_of};
-
-use heed::{BytesDecode, BytesEncode};
-use uuid::Uuid;
-
-pub struct NextIdCodec;
-
-pub enum NextIdKey {
-    Global,
-    Index(Uuid),
-}
-
-impl<'a> BytesEncode<'a> for NextIdCodec {
-    type EItem = NextIdKey;
-
-    fn bytes_encode(item: &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
-        match item {
-            NextIdKey::Global => Some(Cow::Borrowed(b"__global__")),
-            NextIdKey::Index(ref uuid) => Some(Cow::Borrowed(uuid.as_bytes())),
-        }
-    }
-}
-
-pub struct PendingKeyCodec;
-
-impl<'a> BytesEncode<'a> for PendingKeyCodec {
-    type EItem = (u64, Uuid, u64);
-
-    fn bytes_encode((global_id, uuid, update_id): &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
-        let mut bytes = Vec::with_capacity(size_of::<Self::EItem>());
-        bytes.extend_from_slice(&global_id.to_be_bytes());
-        bytes.extend_from_slice(uuid.as_bytes());
-        bytes.extend_from_slice(&update_id.to_be_bytes());
-        Some(Cow::Owned(bytes))
-    }
-}
-
-impl<'a> BytesDecode<'a> for PendingKeyCodec {
-    type DItem = (u64, Uuid, u64);
-
-    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
-        let global_id_bytes = bytes.get(0..size_of::<u64>())?.try_into().ok()?;
-        let global_id = u64::from_be_bytes(global_id_bytes);
-
-        let uuid_bytes = bytes
-            .get(size_of::<u64>()..(size_of::<u64>() + size_of::<Uuid>()))?
-            .try_into()
-            .ok()?;
-        let uuid = Uuid::from_bytes(uuid_bytes);
-
-        let update_id_bytes = bytes
-            .get((size_of::<u64>() + size_of::<Uuid>())..)?
-            .try_into()
-            .ok()?;
-        let update_id = u64::from_be_bytes(update_id_bytes);
-
-        Some((global_id, uuid, update_id))
-    }
-}
-
-pub struct UpdateKeyCodec;
-
-impl<'a> BytesEncode<'a> for UpdateKeyCodec {
-    type EItem = (Uuid, u64);
-
-    fn bytes_encode((uuid, update_id): &'a Self::EItem) -> Option<Cow<'a, [u8]>> {
-        let mut bytes = Vec::with_capacity(size_of::<Self::EItem>());
-        bytes.extend_from_slice(uuid.as_bytes());
-        bytes.extend_from_slice(&update_id.to_be_bytes());
-        Some(Cow::Owned(bytes))
-    }
-}
-
-impl<'a> BytesDecode<'a> for UpdateKeyCodec {
-    type DItem = (Uuid, u64);
-
-    fn bytes_decode(bytes: &'a [u8]) -> Option<Self::DItem> {
-        let uuid_bytes = bytes.get(0..size_of::<Uuid>())?.try_into().ok()?;
-        let uuid = Uuid::from_bytes(uuid_bytes);
-
-        let update_id_bytes = bytes.get(size_of::<Uuid>()..)?.try_into().ok()?;
-        let update_id = u64::from_be_bytes(update_id_bytes);
-
-        Some((uuid, update_id))
-    }
-}

meilisearch-http/src/index_controller/update_actor/store/dump.rs

@@ -1,184 +0,0 @@
-use std::{
-    collections::HashSet,
-    fs::{create_dir_all, File},
-    io::{BufRead, BufReader, Write},
-    path::{Path, PathBuf},
-};
-
-use heed::{EnvOpenOptions, RoTxn};
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-
-use super::{Result, State, UpdateStore};
-use crate::index_controller::{
-    index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path, Enqueued,
-    UpdateStatus,
-};
-
-#[derive(Serialize, Deserialize)]
-struct UpdateEntry {
-    uuid: Uuid,
-    update: UpdateStatus,
-}
-
-impl UpdateStore {
-    pub fn dump(
-        &self,
-        uuids: &HashSet<Uuid>,
-        path: PathBuf,
-        handle: impl IndexActorHandle,
-    ) -> Result<()> {
-        let state_lock = self.state.write();
-        state_lock.swap(State::Dumping);
-
-        // txn must *always* be acquired after state lock, or it will dead lock.
-        let txn = self.env.write_txn()?;
-
-        let dump_path = path.join("updates");
-        create_dir_all(&dump_path)?;
-
-        self.dump_updates(&txn, uuids, &dump_path)?;
-
-        let fut = dump_indexes(uuids, handle, &path);
-        tokio::runtime::Handle::current().block_on(fut)?;
-
-        state_lock.swap(State::Idle);
-
-        Ok(())
-    }
-
-    fn dump_updates(
-        &self,
-        txn: &RoTxn,
-        uuids: &HashSet<Uuid>,
-        path: impl AsRef<Path>,
-    ) -> Result<()> {
-        let dump_data_path = path.as_ref().join("data.jsonl");
-        let mut dump_data_file = File::create(dump_data_path)?;
-
-        let update_files_path = path.as_ref().join(super::UPDATE_DIR);
-        create_dir_all(&update_files_path)?;
-
-        self.dump_pending(txn, uuids, &mut dump_data_file, &path)?;
-        self.dump_completed(txn, uuids, &mut dump_data_file)?;
-
-        Ok(())
-    }
-
-    fn dump_pending(
-        &self,
-        txn: &RoTxn,
-        uuids: &HashSet<Uuid>,
-        mut file: &mut File,
-        dst_path: impl AsRef<Path>,
-    ) -> Result<()> {
-        let pendings = self.pending_queue.iter(txn)?.lazily_decode_data();
-
-        for pending in pendings {
-            let ((_, uuid, _), data) = pending?;
-            if uuids.contains(&uuid) {
-                let update = data.decode()?;
-
-                if let Some(ref update_uuid) = update.content {
-                    let src = super::update_uuid_to_file_path(&self.path, *update_uuid);
-                    let dst = super::update_uuid_to_file_path(&dst_path, *update_uuid);
-                    std::fs::copy(src, dst)?;
-                }
-
-                let update_json = UpdateEntry {
-                    uuid,
-                    update: update.into(),
-                };
-
-                serde_json::to_writer(&mut file, &update_json)?;
-                file.write_all(b"\n")?;
-            }
-        }
-
-        Ok(())
-    }
-
-    fn dump_completed(
-        &self,
-        txn: &RoTxn,
-        uuids: &HashSet<Uuid>,
-        mut file: &mut File,
-    ) -> Result<()> {
-        let updates = self.updates.iter(txn)?.lazily_decode_data();
-
-        for update in updates {
-            let ((uuid, _), data) = update?;
-            if uuids.contains(&uuid) {
-                let update = data.decode()?;
-
-                let update_json = UpdateEntry { uuid, update };
-
-                serde_json::to_writer(&mut file, &update_json)?;
-                file.write_all(b"\n")?;
-            }
-        }
-
-        Ok(())
-    }
-
-    pub fn load_dump(
-        src: impl AsRef<Path>,
-        dst: impl AsRef<Path>,
-        db_size: usize,
-    ) -> anyhow::Result<()> {
-        let dst_update_path = dst.as_ref().join("updates/");
-        create_dir_all(&dst_update_path)?;
-
-        let mut options = EnvOpenOptions::new();
-        options.map_size(db_size as usize);
-        let (store, _) = UpdateStore::new(options, &dst_update_path)?;
-
-        let src_update_path = src.as_ref().join("updates");
-        let update_data = File::open(&src_update_path.join("data.jsonl"))?;
-        let mut update_data = BufReader::new(update_data);
-
-        std::fs::create_dir_all(dst_update_path.join("update_files/"))?;
-
-        let mut wtxn = store.env.write_txn()?;
-        let mut line = String::new();
-        loop {
-            match update_data.read_line(&mut line) {
-                Ok(0) => break,
-                Ok(_) => {
-                    let UpdateEntry { uuid, update } = serde_json::from_str(&line)?;
-                    store.register_raw_updates(&mut wtxn, &update, uuid)?;
-
-                    // Copy ascociated update path if it exists
-                    if let UpdateStatus::Enqueued(Enqueued {
-                        content: Some(uuid),
-                        ..
-                    }) = update
-                    {
-                        let src = update_uuid_to_file_path(&src_update_path, uuid);
-                        let dst = update_uuid_to_file_path(&dst_update_path, uuid);
-                        std::fs::copy(src, dst)?;
-                    }
-                }
-                _ => break,
-            }
-
-            line.clear();
-        }
-
-        wtxn.commit()?;
-
-        Ok(())
-    }
-}
-
-async fn dump_indexes(
-    uuids: &HashSet<Uuid>,
-    handle: impl IndexActorHandle,
-    path: impl AsRef<Path>,
-) -> Result<()> {
-    for uuid in uuids {
-        handle.dump(*uuid, path.as_ref().to_owned()).await?;
-    }
-
-    Ok(())
-}

meilisearch-http/src/index_controller/update_actor/store/mod.rs

@@ -1,729 +0,0 @@
-mod codec;
-pub mod dump;
-
-use std::fs::{copy, create_dir_all, remove_file, File};
-use std::path::Path;
-use std::sync::atomic::{AtomicBool, Ordering};
-use std::sync::Arc;
-use std::{
-    collections::{BTreeMap, HashSet},
-    path::PathBuf,
-    time::Duration,
-};
-
-use arc_swap::ArcSwap;
-use futures::StreamExt;
-use heed::types::{ByteSlice, OwnedType, SerdeJson};
-use heed::zerocopy::U64;
-use heed::{CompactionOption, Database, Env, EnvOpenOptions};
-use log::error;
-use parking_lot::{Mutex, MutexGuard};
-use tokio::runtime::Handle;
-use tokio::sync::mpsc;
-use tokio::sync::mpsc::error::TrySendError;
-use tokio::time::timeout;
-use uuid::Uuid;
-
-use codec::*;
-
-use super::error::Result;
-use super::UpdateMeta;
-use crate::helpers::EnvSizer;
-use crate::index_controller::{index_actor::CONCURRENT_INDEX_MSG, updates::*, IndexActorHandle};
-
-#[allow(clippy::upper_case_acronyms)]
-type BEU64 = U64<heed::byteorder::BE>;
-
-const UPDATE_DIR: &str = "update_files";
-
-pub struct UpdateStoreInfo {
-    /// Size of the update store in bytes.
-    pub size: u64,
-    /// Uuid of the currently processing update if it exists
-    pub processing: Option<Uuid>,
-}
-
-/// A data structure that allows concurrent reads AND exactly one writer.
-pub struct StateLock {
-    lock: Mutex<()>,
-    data: ArcSwap<State>,
-}
-
-pub struct StateLockGuard<'a> {
-    _lock: MutexGuard<'a, ()>,
-    state: &'a StateLock,
-}
-
-impl StateLockGuard<'_> {
-    pub fn swap(&self, state: State) -> Arc<State> {
-        self.state.data.swap(Arc::new(state))
-    }
-}
-
-impl StateLock {
-    fn from_state(state: State) -> Self {
-        let lock = Mutex::new(());
-        let data = ArcSwap::from(Arc::new(state));
-        Self { lock, data }
-    }
-
-    pub fn read(&self) -> Arc<State> {
-        self.data.load().clone()
-    }
-
-    pub fn write(&self) -> StateLockGuard {
-        let _lock = self.lock.lock();
-        let state = &self;
-        StateLockGuard { _lock, state }
-    }
-}
-
-#[allow(clippy::large_enum_variant)]
-pub enum State {
-    Idle,
-    Processing(Uuid, Processing),
-    Snapshoting,
-    Dumping,
-}
-
-#[derive(Clone)]
-pub struct UpdateStore {
-    pub env: Env,
-    /// A queue containing the updates to process, ordered by arrival.
-    /// The key are built as follow:
-    /// | global_update_id | index_uuid | update_id |
-    /// |     8-bytes      |  16-bytes  |  8-bytes  |
-    pending_queue: Database<PendingKeyCodec, SerdeJson<Enqueued>>,
-    /// Map indexes to the next available update id. If NextIdKey::Global is queried, then the next
-    /// global update id is returned
-    next_update_id: Database<NextIdCodec, OwnedType<BEU64>>,
-    /// Contains all the performed updates meta, be they failed, aborted, or processed.
-    /// The keys are built as follow:
-    /// |    Uuid  |   id    |
-    /// | 16-bytes | 8-bytes |
-    updates: Database<UpdateKeyCodec, SerdeJson<UpdateStatus>>,
-    /// Indicates the current state of the update store,
-    state: Arc<StateLock>,
-    /// Wake up the loop when a new event occurs.
-    notification_sender: mpsc::Sender<()>,
-    path: PathBuf,
-}
-
-impl UpdateStore {
-    fn new(
-        mut options: EnvOpenOptions,
-        path: impl AsRef<Path>,
-    ) -> anyhow::Result<(Self, mpsc::Receiver<()>)> {
-        options.max_dbs(5);
-
-        let env = options.open(&path)?;
-        let pending_queue = env.create_database(Some("pending-queue"))?;
-        let next_update_id = env.create_database(Some("next-update-id"))?;
-        let updates = env.create_database(Some("updates"))?;
-
-        let state = Arc::new(StateLock::from_state(State::Idle));
-
-        let (notification_sender, notification_receiver) = mpsc::channel(1);
-
-        Ok((
-            Self {
-                env,
-                pending_queue,
-                next_update_id,
-                updates,
-                state,
-                notification_sender,
-                path: path.as_ref().to_owned(),
-            },
-            notification_receiver,
-        ))
-    }
-
-    pub fn open(
-        options: EnvOpenOptions,
-        path: impl AsRef<Path>,
-        index_handle: impl IndexActorHandle + Clone + Sync + Send + 'static,
-        must_exit: Arc<AtomicBool>,
-    ) -> anyhow::Result<Arc<Self>> {
-        let (update_store, mut notification_receiver) = Self::new(options, path)?;
-        let update_store = Arc::new(update_store);
-
-        // Send a first notification to trigger the process.
-        if let Err(TrySendError::Closed(())) = update_store.notification_sender.try_send(()) {
-            panic!("Failed to init update store");
-        }
-
-        // We need a weak reference so we can take ownership on the arc later when we
-        // want to close the index.
-        let duration = Duration::from_secs(10 * 60); // 10 minutes
-        let update_store_weak = Arc::downgrade(&update_store);
-        tokio::task::spawn(async move {
-            // Block and wait for something to process with a timeout. The timeout
-            // function returns a Result and we must just unlock the loop on Result.
-            'outer: while timeout(duration, notification_receiver.recv())
-                .await
-                .map_or(true, |o| o.is_some())
-            {
-                loop {
-                    match update_store_weak.upgrade() {
-                        Some(update_store) => {
-                            let handler = index_handle.clone();
-                            let res = tokio::task::spawn_blocking(move || {
-                                update_store.process_pending_update(handler)
-                            })
-                            .await
-                            .expect("Fatal error processing update.");
-                            match res {
-                                Ok(Some(_)) => (),
-                                Ok(None) => break,
-                                Err(e) => {
-                                    error!("Fatal error while processing an update that requires the update store to shutdown: {}", e);
-                                    must_exit.store(true, Ordering::SeqCst);
-                                    break 'outer;
-                                }
-                            }
-                        }
-                        // the ownership on the arc has been taken, we need to exit.
-                        None => break 'outer,
-                    }
-                }
-            }
-
-            error!("Update store loop exited.");
-        });
-
-        Ok(update_store)
-    }
-
-    /// Returns the next global update id and the next update id for a given `index_uuid`.
-    fn next_update_id(&self, txn: &mut heed::RwTxn, index_uuid: Uuid) -> heed::Result<(u64, u64)> {
-        let global_id = self
-            .next_update_id
-            .get(txn, &NextIdKey::Global)?
-            .map(U64::get)
-            .unwrap_or_default();
-
-        self.next_update_id
-            .put(txn, &NextIdKey::Global, &BEU64::new(global_id + 1))?;
-
-        let update_id = self.next_update_id_raw(txn, index_uuid)?;
-
-        Ok((global_id, update_id))
-    }
-
-    /// Returns the next next update id for a given `index_uuid` without
-    /// incrementing the global update id. This is useful for the dumps.
-    fn next_update_id_raw(&self, txn: &mut heed::RwTxn, index_uuid: Uuid) -> heed::Result<u64> {
-        let update_id = self
-            .next_update_id
-            .get(txn, &NextIdKey::Index(index_uuid))?
-            .map(U64::get)
-            .unwrap_or_default();
-
-        self.next_update_id.put(
-            txn,
-            &NextIdKey::Index(index_uuid),
-            &BEU64::new(update_id + 1),
-        )?;
-
-        Ok(update_id)
-    }
-
-    /// Registers the update content in the pending store and the meta
-    /// into the pending-meta store. Returns the new unique update id.
-    pub fn register_update(
-        &self,
-        meta: UpdateMeta,
-        content: Option<Uuid>,
-        index_uuid: Uuid,
-    ) -> heed::Result<Enqueued> {
-        let mut txn = self.env.write_txn()?;
-
-        let (global_id, update_id) = self.next_update_id(&mut txn, index_uuid)?;
-        let meta = Enqueued::new(meta, update_id, content);
-
-        self.pending_queue
-            .put(&mut txn, &(global_id, index_uuid, update_id), &meta)?;
-
-        txn.commit()?;
-
-        if let Err(TrySendError::Closed(())) = self.notification_sender.try_send(()) {
-            panic!("Update store loop exited");
-        }
-
-        Ok(meta)
-    }
-
-    /// Push already processed update in the UpdateStore without triggering the notification
-    /// process. This is useful for the dumps.
-    pub fn register_raw_updates(
-        &self,
-        wtxn: &mut heed::RwTxn,
-        update: &UpdateStatus,
-        index_uuid: Uuid,
-    ) -> heed::Result<()> {
-        match update {
-            UpdateStatus::Enqueued(enqueued) => {
-                let (global_id, _update_id) = self.next_update_id(wtxn, index_uuid)?;
-                self.pending_queue.remap_key_type::<PendingKeyCodec>().put(
-                    wtxn,
-                    &(global_id, index_uuid, enqueued.id()),
-                    enqueued,
-                )?;
-            }
-            _ => {
-                let _update_id = self.next_update_id_raw(wtxn, index_uuid)?;
-                self.updates.put(wtxn, &(index_uuid, update.id()), update)?;
-            }
-        }
-        Ok(())
-    }
-
-    /// Executes the user provided function on the next pending update (the one with the lowest id).
-    /// This is asynchronous as it let the user process the update with a read-only txn and
-    /// only writing the result meta to the processed-meta store *after* it has been processed.
-    fn process_pending_update(&self, index_handle: impl IndexActorHandle) -> Result<Option<()>> {
-        // Create a read transaction to be able to retrieve the pending update in order.
-        let rtxn = self.env.read_txn()?;
-        let first_meta = self.pending_queue.first(&rtxn)?;
-        drop(rtxn);
-
-        // If there is a pending update we process and only keep
-        // a reader while processing it, not a writer.
-        match first_meta {
-            Some(((global_id, index_uuid, _), mut pending)) => {
-                let content = pending.content.take();
-                let processing = pending.processing();
-                // Acquire the state lock and set the current state to processing.
-                // txn must *always* be acquired after state lock, or it will dead lock.
-                let state = self.state.write();
-                state.swap(State::Processing(index_uuid, processing.clone()));
-
-                let result =
-                    self.perform_update(content, processing, index_handle, index_uuid, global_id);
-
-                state.swap(State::Idle);
-
-                result
-            }
-            None => Ok(None),
-        }
-    }
-
-    fn perform_update(
-        &self,
-        content: Option<Uuid>,
-        processing: Processing,
-        index_handle: impl IndexActorHandle,
-        index_uuid: Uuid,
-        global_id: u64,
-    ) -> Result<Option<()>> {
-        let content_path = content.map(|uuid| update_uuid_to_file_path(&self.path, uuid));
-        let update_id = processing.id();
-
-        let file = match content_path {
-            Some(ref path) => {
-                let file = File::open(path)?;
-                Some(file)
-            }
-            None => None,
-        };
-
-        // Process the pending update using the provided user function.
-        let handle = Handle::current();
-        let result =
-            match handle.block_on(index_handle.update(index_uuid, processing.clone(), file)) {
-                Ok(result) => result,
-                Err(e) => Err(processing.fail(e.into())),
-            };
-
-        // Once the pending update have been successfully processed
-        // we must remove the content from the pending and processing stores and
-        // write the *new* meta to the processed-meta store and commit.
-        let mut wtxn = self.env.write_txn()?;
-        self.pending_queue
-            .delete(&mut wtxn, &(global_id, index_uuid, update_id))?;
-
-        let result = match result {
-            Ok(res) => res.into(),
-            Err(res) => res.into(),
-        };
-
-        self.updates
-            .put(&mut wtxn, &(index_uuid, update_id), &result)?;
-
-        wtxn.commit()?;
-
-        if let Some(ref path) = content_path {
-            remove_file(&path)?;
-        }
-
-        Ok(Some(()))
-    }
-
-    /// List the updates for `index_uuid`.
-    pub fn list(&self, index_uuid: Uuid) -> Result<Vec<UpdateStatus>> {
-        let mut update_list = BTreeMap::<u64, UpdateStatus>::new();
-
-        let txn = self.env.read_txn()?;
-
-        let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data();
-        for entry in pendings {
-            let ((_, uuid, id), pending) = entry?;
-            if uuid == index_uuid {
-                update_list.insert(id, pending.decode()?.into());
-            }
-        }
-
-        let updates = self
-            .updates
-            .remap_key_type::<ByteSlice>()
-            .prefix_iter(&txn, index_uuid.as_bytes())?;
-
-        for entry in updates {
-            let (_, update) = entry?;
-            update_list.insert(update.id(), update);
-        }
-
-        // If the currently processing update is from this index, replace the corresponding pending update with this one.
-        match *self.state.read() {
-            State::Processing(uuid, ref processing) if uuid == index_uuid => {
-                update_list.insert(processing.id(), processing.clone().into());
-            }
-            _ => (),
-        }
-
-        Ok(update_list.into_iter().map(|(_, v)| v).collect())
-    }
-
-    /// Returns the update associated meta or `None` if the update doesn't exist.
-    pub fn meta(&self, index_uuid: Uuid, update_id: u64) -> heed::Result<Option<UpdateStatus>> {
-        // Check if the update is the one currently processing
-        match *self.state.read() {
-            State::Processing(uuid, ref processing)
-                if uuid == index_uuid && processing.id() == update_id =>
-            {
-                return Ok(Some(processing.clone().into()));
-            }
-            _ => (),
-        }
-
-        let txn = self.env.read_txn()?;
-        // Else, check if it is in the updates database:
-        let update = self.updates.get(&txn, &(index_uuid, update_id))?;
-
-        if let Some(update) = update {
-            return Ok(Some(update));
-        }
-
-        // If nothing was found yet, we resolve to iterate over the pending queue.
-        let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data();
-
-        for entry in pendings {
-            let ((_, uuid, id), pending) = entry?;
-            if uuid == index_uuid && id == update_id {
-                return Ok(Some(pending.decode()?.into()));
-            }
-        }
-
-        // No update was found.
-        Ok(None)
-    }
-
-    /// Delete all updates for an index from the update store. If the currently processing update
-    /// is for `index_uuid`, the call will block until the update is terminated.
-    pub fn delete_all(&self, index_uuid: Uuid) -> Result<()> {
-        let mut txn = self.env.write_txn()?;
-        // Contains all the content file paths that we need to be removed if the deletion was successful.
-        let mut uuids_to_remove = Vec::new();
-
-        let mut pendings = self.pending_queue.iter_mut(&mut txn)?.lazily_decode_data();
-
-        while let Some(Ok(((_, uuid, _), pending))) = pendings.next() {
-            if uuid == index_uuid {
-                let mut pending = pending.decode()?;
-                if let Some(update_uuid) = pending.content.take() {
-                    uuids_to_remove.push(update_uuid);
-                }
-
-                // Invariant check: we can only delete the current entry when we don't hold
-                // references to it anymore. This must be done after we have retrieved its content.
-                unsafe {
-                    pendings.del_current()?;
-                }
-            }
-        }
-
-        drop(pendings);
-
-        let mut updates = self
-            .updates
-            .remap_key_type::<ByteSlice>()
-            .prefix_iter_mut(&mut txn, index_uuid.as_bytes())?
-            .lazily_decode_data();
-
-        while let Some(_) = updates.next() {
-            unsafe {
-                updates.del_current()?;
-            }
-        }
-
-        drop(updates);
-
-        txn.commit()?;
-
-        // If the currently processing update is from our index, we wait until it is
-        // finished before returning. This ensure that no write to the index occurs after we delete it.
-        if let State::Processing(uuid, _) = *self.state.read() {
-            if uuid == index_uuid {
-                // wait for a write lock, do nothing with it.
-                self.state.write();
-            }
-        }
-
-        // Finally, remove any outstanding update files. This must be done after waiting for the
-        // last update to ensure that the update files are not deleted before the update needs
-        // them.
-        uuids_to_remove
-            .iter()
-            .map(|uuid| update_uuid_to_file_path(&self.path, *uuid))
-            .for_each(|path| {
-                let _ = remove_file(path);
-            });
-
-        Ok(())
-    }
-
-    pub fn snapshot(
-        &self,
-        uuids: &HashSet<Uuid>,
-        path: impl AsRef<Path>,
-        handle: impl IndexActorHandle + Clone,
-    ) -> Result<()> {
-        let state_lock = self.state.write();
-        state_lock.swap(State::Snapshoting);
-
-        let txn = self.env.write_txn()?;
-
-        let update_path = path.as_ref().join("updates");
-        create_dir_all(&update_path)?;
-
-        // acquire write lock to prevent further writes during snapshot
-        create_dir_all(&update_path)?;
-        let db_path = update_path.join("data.mdb");
-
-        // create db snapshot
-        self.env.copy_to_path(&db_path, CompactionOption::Enabled)?;
-
-        let update_files_path = update_path.join(UPDATE_DIR);
-        create_dir_all(&update_files_path)?;
-
-        let pendings = self.pending_queue.iter(&txn)?.lazily_decode_data();
-
-        for entry in pendings {
-            let ((_, uuid, _), pending) = entry?;
-            if uuids.contains(&uuid) {
-                if let Enqueued {
-                    content: Some(uuid),
-                    ..
-                } = pending.decode()?
-                {
-                    let path = update_uuid_to_file_path(&self.path, uuid);
-                    copy(path, &update_files_path)?;
-                }
-            }
-        }
-
-        let path = &path.as_ref().to_path_buf();
-        let handle = &handle;
-        // Perform the snapshot of each index concurently. Only a third of the capabilities of
-        // the index actor at a time not to put too much pressure on the index actor
-        let mut stream = futures::stream::iter(uuids.iter())
-            .map(move |uuid| handle.snapshot(*uuid, path.clone()))
-            .buffer_unordered(CONCURRENT_INDEX_MSG / 3);
-
-        Handle::current().block_on(async {
-            while let Some(res) = stream.next().await {
-                res?;
-            }
-            Ok(()) as Result<()>
-        })?;
-
-        Ok(())
-    }
-
-    pub fn get_info(&self) -> Result<UpdateStoreInfo> {
-        let mut size = self.env.size();
-        let txn = self.env.read_txn()?;
-        for entry in self.pending_queue.iter(&txn)? {
-            let (_, pending) = entry?;
-            if let Enqueued {
-                content: Some(uuid),
-                ..
-            } = pending
-            {
-                let path = update_uuid_to_file_path(&self.path, uuid);
-                size += File::open(path)?.metadata()?.len();
-            }
-        }
-        let processing = match *self.state.read() {
-            State::Processing(uuid, _) => Some(uuid),
-            _ => None,
-        };
-
-        Ok(UpdateStoreInfo { size, processing })
-    }
-}
-
-fn update_uuid_to_file_path(root: impl AsRef<Path>, uuid: Uuid) -> PathBuf {
-    root.as_ref()
-        .join(UPDATE_DIR)
-        .join(format!("update_{}", uuid))
-}
-
-#[cfg(test)]
-mod test {
-    use super::*;
-    use crate::index_controller::{
-        index_actor::{error::IndexActorError, MockIndexActorHandle},
-        UpdateResult,
-    };
-
-    use futures::future::ok;
-
-    #[actix_rt::test]
-    async fn test_next_id() {
-        let dir = tempfile::tempdir_in(".").unwrap();
-        let mut options = EnvOpenOptions::new();
-        let handle = Arc::new(MockIndexActorHandle::new());
-        options.map_size(4096 * 100);
-        let update_store = UpdateStore::open(
-            options,
-            dir.path(),
-            handle,
-            Arc::new(AtomicBool::new(false)),
-        )
-        .unwrap();
-
-        let index1_uuid = Uuid::new_v4();
-        let index2_uuid = Uuid::new_v4();
-
-        let mut txn = update_store.env.write_txn().unwrap();
-        let ids = update_store.next_update_id(&mut txn, index1_uuid).unwrap();
-        txn.commit().unwrap();
-        assert_eq!((0, 0), ids);
-
-        let mut txn = update_store.env.write_txn().unwrap();
-        let ids = update_store.next_update_id(&mut txn, index2_uuid).unwrap();
-        txn.commit().unwrap();
-        assert_eq!((1, 0), ids);
-
-        let mut txn = update_store.env.write_txn().unwrap();
-        let ids = update_store.next_update_id(&mut txn, index1_uuid).unwrap();
-        txn.commit().unwrap();
-        assert_eq!((2, 1), ids);
-    }
-
-    #[actix_rt::test]
-    async fn test_register_update() {
-        let dir = tempfile::tempdir_in(".").unwrap();
-        let mut options = EnvOpenOptions::new();
-        let handle = Arc::new(MockIndexActorHandle::new());
-        options.map_size(4096 * 100);
-        let update_store = UpdateStore::open(
-            options,
-            dir.path(),
-            handle,
-            Arc::new(AtomicBool::new(false)),
-        )
-        .unwrap();
-        let meta = UpdateMeta::ClearDocuments;
-        let uuid = Uuid::new_v4();
-        let store_clone = update_store.clone();
-        tokio::task::spawn_blocking(move || {
-            store_clone.register_update(meta, None, uuid).unwrap();
-        })
-        .await
-        .unwrap();
-
-        let txn = update_store.env.read_txn().unwrap();
-        assert!(update_store
-            .pending_queue
-            .get(&txn, &(0, uuid, 0))
-            .unwrap()
-            .is_some());
-    }
-
-    #[actix_rt::test]
-    async fn test_process_update() {
-        let dir = tempfile::tempdir_in(".").unwrap();
-        let mut handle = MockIndexActorHandle::new();
-
-        handle
-            .expect_update()
-            .times(2)
-            .returning(|_index_uuid, processing, _file| {
-                if processing.id() == 0 {
-                    Box::pin(ok(Ok(processing.process(UpdateResult::Other))))
-                } else {
-                    Box::pin(ok(Err(
-                        processing.fail(IndexActorError::ExistingPrimaryKey.into())
-                    )))
-                }
-            });
-
-        let handle = Arc::new(handle);
-
-        let mut options = EnvOpenOptions::new();
-        options.map_size(4096 * 100);
-        let store = UpdateStore::open(
-            options,
-            dir.path(),
-            handle.clone(),
-            Arc::new(AtomicBool::new(false)),
-        )
-        .unwrap();
-
-        // wait a bit for the event loop exit.
-        tokio::time::sleep(std::time::Duration::from_millis(50)).await;
-
-        let mut txn = store.env.write_txn().unwrap();
-
-        let update = Enqueued::new(UpdateMeta::ClearDocuments, 0, None);
-        let uuid = Uuid::new_v4();
-
-        store
-            .pending_queue
-            .put(&mut txn, &(0, uuid, 0), &update)
-            .unwrap();
-
-        let update = Enqueued::new(UpdateMeta::ClearDocuments, 1, None);
-
-        store
-            .pending_queue
-            .put(&mut txn, &(1, uuid, 1), &update)
-            .unwrap();
-
-        txn.commit().unwrap();
-
-        // Process the pending, and check that it has been moved to the update databases, and
-        // removed from the pending database.
-        let store_clone = store.clone();
-        tokio::task::spawn_blocking(move || {
-            store_clone.process_pending_update(handle.clone()).unwrap();
-            store_clone.process_pending_update(handle).unwrap();
-        })
-        .await
-        .unwrap();
-
-        let txn = store.env.read_txn().unwrap();
-
-        assert!(store.pending_queue.first(&txn).unwrap().is_none());
-        let update = store.updates.get(&txn, &(uuid, 0)).unwrap().unwrap();
-
-        assert!(matches!(update, UpdateStatus::Processed(_)));
-        let update = store.updates.get(&txn, &(uuid, 1)).unwrap().unwrap();
-
-        assert!(matches!(update, UpdateStatus::Failed(_)));
-    }
-}

meilisearch-http/src/index_controller/updates.rs

@@ -1,233 +0,0 @@
-use chrono::{DateTime, Utc};
-use milli::update::{DocumentAdditionResult, IndexDocumentsMethod, UpdateFormat};
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-
-use crate::{
-    error::ResponseError,
-    index::{Settings, Unchecked},
-};
-
-#[derive(Debug, Clone, Serialize, Deserialize)]
-pub enum UpdateResult {
-    DocumentsAddition(DocumentAdditionResult),
-    DocumentDeletion { deleted: u64 },
-    Other,
-}
-
-#[allow(clippy::large_enum_variant)]
-#[derive(Debug, Clone, Serialize, Deserialize)]
-#[serde(tag = "type")]
-pub enum UpdateMeta {
-    DocumentsAddition {
-        method: IndexDocumentsMethod,
-        format: UpdateFormat,
-        primary_key: Option<String>,
-    },
-    ClearDocuments,
-    DeleteDocuments {
-        ids: Vec<String>,
-    },
-    Settings(Settings<Unchecked>),
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct Enqueued {
-    pub update_id: u64,
-    pub meta: UpdateMeta,
-    pub enqueued_at: DateTime<Utc>,
-    pub content: Option<Uuid>,
-}
-
-impl Enqueued {
-    pub fn new(meta: UpdateMeta, update_id: u64, content: Option<Uuid>) -> Self {
-        Self {
-            enqueued_at: Utc::now(),
-            meta,
-            update_id,
-            content,
-        }
-    }
-
-    pub fn processing(self) -> Processing {
-        Processing {
-            from: self,
-            started_processing_at: Utc::now(),
-        }
-    }
-
-    pub fn abort(self) -> Aborted {
-        Aborted {
-            from: self,
-            aborted_at: Utc::now(),
-        }
-    }
-
-    pub fn meta(&self) -> &UpdateMeta {
-        &self.meta
-    }
-
-    pub fn id(&self) -> u64 {
-        self.update_id
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct Processed {
-    pub success: UpdateResult,
-    pub processed_at: DateTime<Utc>,
-    #[serde(flatten)]
-    pub from: Processing,
-}
-
-impl Processed {
-    pub fn id(&self) -> u64 {
-        self.from.id()
-    }
-
-    pub fn meta(&self) -> &UpdateMeta {
-        self.from.meta()
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct Processing {
-    #[serde(flatten)]
-    pub from: Enqueued,
-    pub started_processing_at: DateTime<Utc>,
-}
-
-impl Processing {
-    pub fn id(&self) -> u64 {
-        self.from.id()
-    }
-
-    pub fn meta(&self) -> &UpdateMeta {
-        self.from.meta()
-    }
-
-    pub fn process(self, success: UpdateResult) -> Processed {
-        Processed {
-            success,
-            from: self,
-            processed_at: Utc::now(),
-        }
-    }
-
-    pub fn fail(self, error: ResponseError) -> Failed {
-        Failed {
-            from: self,
-            error,
-            failed_at: Utc::now(),
-        }
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct Aborted {
-    #[serde(flatten)]
-    from: Enqueued,
-    aborted_at: DateTime<Utc>,
-}
-
-impl Aborted {
-    pub fn id(&self) -> u64 {
-        self.from.id()
-    }
-
-    pub fn meta(&self) -> &UpdateMeta {
-        self.from.meta()
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-pub struct Failed {
-    #[serde(flatten)]
-    pub from: Processing,
-    pub error: ResponseError,
-    pub failed_at: DateTime<Utc>,
-}
-
-impl Failed {
-    pub fn id(&self) -> u64 {
-        self.from.id()
-    }
-
-    pub fn meta(&self) -> &UpdateMeta {
-        self.from.meta()
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(tag = "status", rename_all = "camelCase")]
-pub enum UpdateStatus {
-    Processing(Processing),
-    Enqueued(Enqueued),
-    Processed(Processed),
-    Aborted(Aborted),
-    Failed(Failed),
-}
-
-impl UpdateStatus {
-    pub fn id(&self) -> u64 {
-        match self {
-            UpdateStatus::Processing(u) => u.id(),
-            UpdateStatus::Enqueued(u) => u.id(),
-            UpdateStatus::Processed(u) => u.id(),
-            UpdateStatus::Aborted(u) => u.id(),
-            UpdateStatus::Failed(u) => u.id(),
-        }
-    }
-
-    pub fn meta(&self) -> &UpdateMeta {
-        match self {
-            UpdateStatus::Processing(u) => u.meta(),
-            UpdateStatus::Enqueued(u) => u.meta(),
-            UpdateStatus::Processed(u) => u.meta(),
-            UpdateStatus::Aborted(u) => u.meta(),
-            UpdateStatus::Failed(u) => u.meta(),
-        }
-    }
-
-    pub fn processed(&self) -> Option<&Processed> {
-        match self {
-            UpdateStatus::Processed(p) => Some(p),
-            _ => None,
-        }
-    }
-}
-
-impl From<Enqueued> for UpdateStatus {
-    fn from(other: Enqueued) -> Self {
-        Self::Enqueued(other)
-    }
-}
-
-impl From<Aborted> for UpdateStatus {
-    fn from(other: Aborted) -> Self {
-        Self::Aborted(other)
-    }
-}
-
-impl From<Processed> for UpdateStatus {
-    fn from(other: Processed) -> Self {
-        Self::Processed(other)
-    }
-}
-
-impl From<Processing> for UpdateStatus {
-    fn from(other: Processing) -> Self {
-        Self::Processing(other)
-    }
-}
-
-impl From<Failed> for UpdateStatus {
-    fn from(other: Failed) -> Self {
-        Self::Failed(other)
-    }
-}

meilisearch-http/src/index_controller/uuid_resolver/actor.rs

@@ -1,98 +0,0 @@
-use std::{collections::HashSet, path::PathBuf};
-
-use log::{trace, warn};
-use tokio::sync::mpsc;
-use uuid::Uuid;
-
-use super::{error::UuidResolverError, Result, UuidResolveMsg, UuidStore};
-
-pub struct UuidResolverActor<S> {
-    inbox: mpsc::Receiver<UuidResolveMsg>,
-    store: S,
-}
-
-impl<S: UuidStore> UuidResolverActor<S> {
-    pub fn new(inbox: mpsc::Receiver<UuidResolveMsg>, store: S) -> Self {
-        Self { inbox, store }
-    }
-
-    pub async fn run(mut self) {
-        use UuidResolveMsg::*;
-
-        trace!("uuid resolver started");
-
-        loop {
-            match self.inbox.recv().await {
-                Some(Get { uid: name, ret }) => {
-                    let _ = ret.send(self.handle_get(name).await);
-                }
-                Some(Delete { uid: name, ret }) => {
-                    let _ = ret.send(self.handle_delete(name).await);
-                }
-                Some(List { ret }) => {
-                    let _ = ret.send(self.handle_list().await);
-                }
-                Some(Insert { ret, uuid, name }) => {
-                    let _ = ret.send(self.handle_insert(name, uuid).await);
-                }
-                Some(SnapshotRequest { path, ret }) => {
-                    let _ = ret.send(self.handle_snapshot(path).await);
-                }
-                Some(GetSize { ret }) => {
-                    let _ = ret.send(self.handle_get_size().await);
-                }
-                Some(DumpRequest { path, ret }) => {
-                    let _ = ret.send(self.handle_dump(path).await);
-                }
-                // all senders have been dropped, need to quit.
-                None => break,
-            }
-        }
-
-        warn!("exiting uuid resolver loop");
-    }
-
-    async fn handle_get(&self, uid: String) -> Result<Uuid> {
-        self.store
-            .get_uuid(uid.clone())
-            .await?
-            .ok_or(UuidResolverError::UnexistingIndex(uid))
-    }
-
-    async fn handle_delete(&self, uid: String) -> Result<Uuid> {
-        self.store
-            .delete(uid.clone())
-            .await?
-            .ok_or(UuidResolverError::UnexistingIndex(uid))
-    }
-
-    async fn handle_list(&self) -> Result<Vec<(String, Uuid)>> {
-        let result = self.store.list().await?;
-        Ok(result)
-    }
-
-    async fn handle_snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
-        self.store.snapshot(path).await
-    }
-
-    async fn handle_dump(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
-        self.store.dump(path).await
-    }
-
-    async fn handle_insert(&self, uid: String, uuid: Uuid) -> Result<()> {
-        if !is_index_uid_valid(&uid) {
-            return Err(UuidResolverError::BadlyFormatted(uid));
-        }
-        self.store.insert(uid, uuid).await?;
-        Ok(())
-    }
-
-    async fn handle_get_size(&self) -> Result<u64> {
-        self.store.get_size().await
-    }
-}
-
-fn is_index_uid_valid(uid: &str) -> bool {
-    uid.chars()
-        .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
-}

meilisearch-http/src/index_controller/uuid_resolver/error.rs

@@ -1,34 +0,0 @@
-use meilisearch_error::{Code, ErrorCode};
-
-pub type Result<T> = std::result::Result<T, UuidResolverError>;
-
-#[derive(Debug, thiserror::Error)]
-pub enum UuidResolverError {
-    #[error("Index already exists.")]
-    NameAlreadyExist,
-    #[error("Index \"{0}\" not found.")]
-    UnexistingIndex(String),
-    #[error("Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).")]
-    BadlyFormatted(String),
-    #[error("Internal error: {0}")]
-    Internal(Box<dyn std::error::Error + Sync + Send + 'static>),
-}
-
-internal_error!(
-    UuidResolverError: heed::Error,
-    uuid::Error,
-    std::io::Error,
-    tokio::task::JoinError,
-    serde_json::Error
-);
-
-impl ErrorCode for UuidResolverError {
-    fn error_code(&self) -> Code {
-        match self {
-            UuidResolverError::NameAlreadyExist => Code::IndexAlreadyExists,
-            UuidResolverError::UnexistingIndex(_) => Code::IndexNotFound,
-            UuidResolverError::BadlyFormatted(_) => Code::InvalidIndexUid,
-            UuidResolverError::Internal(_) => Code::Internal,
-        }
-    }
-}

meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs

@@ -1,87 +0,0 @@
-use std::collections::HashSet;
-use std::path::{Path, PathBuf};
-
-use tokio::sync::{mpsc, oneshot};
-use uuid::Uuid;
-
-use super::{HeedUuidStore, Result, UuidResolveMsg, UuidResolverActor, UuidResolverHandle};
-
-#[derive(Clone)]
-pub struct UuidResolverHandleImpl {
-    sender: mpsc::Sender<UuidResolveMsg>,
-}
-
-impl UuidResolverHandleImpl {
-    pub fn new(path: impl AsRef<Path>) -> Result<Self> {
-        let (sender, reveiver) = mpsc::channel(100);
-        let store = HeedUuidStore::new(path)?;
-        let actor = UuidResolverActor::new(reveiver, store);
-        tokio::spawn(actor.run());
-        Ok(Self { sender })
-    }
-}
-
-#[async_trait::async_trait]
-impl UuidResolverHandle for UuidResolverHandleImpl {
-    async fn get(&self, name: String) -> Result<Uuid> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::Get { uid: name, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-
-    async fn delete(&self, name: String) -> Result<Uuid> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::Delete { uid: name, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-
-    async fn list(&self) -> Result<Vec<(String, Uuid)>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::List { ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-
-    async fn insert(&self, name: String, uuid: Uuid) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::Insert { ret, name, uuid };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-
-    async fn snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::SnapshotRequest { path, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-
-    async fn get_size(&self) -> Result<u64> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::GetSize { ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-    async fn dump(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::DumpRequest { ret, path };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-}

meilisearch-http/src/index_controller/uuid_resolver/message.rs

@@ -1,37 +0,0 @@
-use std::collections::HashSet;
-use std::path::PathBuf;
-
-use tokio::sync::oneshot;
-use uuid::Uuid;
-
-use super::Result;
-
-pub enum UuidResolveMsg {
-    Get {
-        uid: String,
-        ret: oneshot::Sender<Result<Uuid>>,
-    },
-    Delete {
-        uid: String,
-        ret: oneshot::Sender<Result<Uuid>>,
-    },
-    List {
-        ret: oneshot::Sender<Result<Vec<(String, Uuid)>>>,
-    },
-    Insert {
-        uuid: Uuid,
-        name: String,
-        ret: oneshot::Sender<Result<()>>,
-    },
-    SnapshotRequest {
-        path: PathBuf,
-        ret: oneshot::Sender<Result<HashSet<Uuid>>>,
-    },
-    GetSize {
-        ret: oneshot::Sender<Result<u64>>,
-    },
-    DumpRequest {
-        path: PathBuf,
-        ret: oneshot::Sender<Result<HashSet<Uuid>>>,
-    },
-}

meilisearch-http/src/index_controller/uuid_resolver/mod.rs

@@ -1,35 +0,0 @@
-mod actor;
-pub mod error;
-mod handle_impl;
-mod message;
-pub mod store;
-
-use std::collections::HashSet;
-use std::path::PathBuf;
-
-use uuid::Uuid;
-
-use actor::UuidResolverActor;
-use error::Result;
-use message::UuidResolveMsg;
-use store::UuidStore;
-
-#[cfg(test)]
-use mockall::automock;
-
-pub use handle_impl::UuidResolverHandleImpl;
-pub use store::HeedUuidStore;
-
-const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB
-
-#[async_trait::async_trait]
-#[cfg_attr(test, automock)]
-pub trait UuidResolverHandle {
-    async fn get(&self, name: String) -> Result<Uuid>;
-    async fn insert(&self, name: String, uuid: Uuid) -> Result<()>;
-    async fn delete(&self, name: String) -> Result<Uuid>;
-    async fn list(&self) -> Result<Vec<(String, Uuid)>>;
-    async fn snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>>;
-    async fn get_size(&self) -> Result<u64>;
-    async fn dump(&self, path: PathBuf) -> Result<HashSet<Uuid>>;
-}

meilisearch-http/src/index_controller/uuid_resolver/store.rs

@@ -1,224 +0,0 @@
-use std::collections::HashSet;
-use std::fs::{create_dir_all, File};
-use std::io::{BufRead, BufReader, Write};
-use std::path::{Path, PathBuf};
-
-use heed::types::{ByteSlice, Str};
-use heed::{CompactionOption, Database, Env, EnvOpenOptions};
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-
-use super::{error::UuidResolverError, Result, UUID_STORE_SIZE};
-use crate::helpers::EnvSizer;
-
-#[derive(Serialize, Deserialize)]
-struct DumpEntry {
-    uuid: Uuid,
-    uid: String,
-}
-
-const UUIDS_DB_PATH: &str = "index_uuids";
-
-#[async_trait::async_trait]
-pub trait UuidStore: Sized {
-    // Create a new entry for `name`. Return an error if `err` and the entry already exists, return
-    // the uuid otherwise.
-    async fn get_uuid(&self, uid: String) -> Result<Option<Uuid>>;
-    async fn delete(&self, uid: String) -> Result<Option<Uuid>>;
-    async fn list(&self) -> Result<Vec<(String, Uuid)>>;
-    async fn insert(&self, name: String, uuid: Uuid) -> Result<()>;
-    async fn snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>>;
-    async fn get_size(&self) -> Result<u64>;
-    async fn dump(&self, path: PathBuf) -> Result<HashSet<Uuid>>;
-}
-
-#[derive(Clone)]
-pub struct HeedUuidStore {
-    env: Env,
-    db: Database<Str, ByteSlice>,
-}
-
-impl HeedUuidStore {
-    pub fn new(path: impl AsRef<Path>) -> Result<Self> {
-        let path = path.as_ref().join(UUIDS_DB_PATH);
-        create_dir_all(&path)?;
-        let mut options = EnvOpenOptions::new();
-        options.map_size(UUID_STORE_SIZE); // 1GB
-        let env = options.open(path)?;
-        let db = env.create_database(None)?;
-        Ok(Self { env, db })
-    }
-
-    pub fn get_uuid(&self, name: String) -> Result<Option<Uuid>> {
-        let env = self.env.clone();
-        let db = self.db;
-        let txn = env.read_txn()?;
-        match db.get(&txn, &name)? {
-            Some(uuid) => {
-                let uuid = Uuid::from_slice(uuid)?;
-                Ok(Some(uuid))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn delete(&self, uid: String) -> Result<Option<Uuid>> {
-        let env = self.env.clone();
-        let db = self.db;
-        let mut txn = env.write_txn()?;
-        match db.get(&txn, &uid)? {
-            Some(uuid) => {
-                let uuid = Uuid::from_slice(uuid)?;
-                db.delete(&mut txn, &uid)?;
-                txn.commit()?;
-                Ok(Some(uuid))
-            }
-            None => Ok(None),
-        }
-    }
-
-    pub fn list(&self) -> Result<Vec<(String, Uuid)>> {
-        let env = self.env.clone();
-        let db = self.db;
-        let txn = env.read_txn()?;
-        let mut entries = Vec::new();
-        for entry in db.iter(&txn)? {
-            let (name, uuid) = entry?;
-            let uuid = Uuid::from_slice(uuid)?;
-            entries.push((name.to_owned(), uuid))
-        }
-        Ok(entries)
-    }
-
-    pub fn insert(&self, name: String, uuid: Uuid) -> Result<()> {
-        let env = self.env.clone();
-        let db = self.db;
-        let mut txn = env.write_txn()?;
-
-        if db.get(&txn, &name)?.is_some() {
-            return Err(UuidResolverError::NameAlreadyExist);
-        }
-
-        db.put(&mut txn, &name, uuid.as_bytes())?;
-        txn.commit()?;
-        Ok(())
-    }
-
-    pub fn snapshot(&self, mut path: PathBuf) -> Result<HashSet<Uuid>> {
-        let env = self.env.clone();
-        let db = self.db;
-        // Write transaction to acquire a lock on the database.
-        let txn = env.write_txn()?;
-        let mut entries = HashSet::new();
-        for entry in db.iter(&txn)? {
-            let (_, uuid) = entry?;
-            let uuid = Uuid::from_slice(uuid)?;
-            entries.insert(uuid);
-        }
-
-        // only perform snapshot if there are indexes
-        if !entries.is_empty() {
-            path.push(UUIDS_DB_PATH);
-            create_dir_all(&path).unwrap();
-            path.push("data.mdb");
-            env.copy_to_path(path, CompactionOption::Enabled)?;
-        }
-        Ok(entries)
-    }
-
-    pub fn get_size(&self) -> Result<u64> {
-        Ok(self.env.size())
-    }
-
-    pub fn dump(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
-        let dump_path = path.join(UUIDS_DB_PATH);
-        create_dir_all(&dump_path)?;
-        let dump_file_path = dump_path.join("data.jsonl");
-        let mut dump_file = File::create(&dump_file_path)?;
-        let mut uuids = HashSet::new();
-
-        let txn = self.env.read_txn()?;
-        for entry in self.db.iter(&txn)? {
-            let (uid, uuid) = entry?;
-            let uid = uid.to_string();
-            let uuid = Uuid::from_slice(uuid)?;
-
-            let entry = DumpEntry { uuid, uid };
-            serde_json::to_writer(&mut dump_file, &entry)?;
-            dump_file.write_all(b"\n").unwrap();
-
-            uuids.insert(uuid);
-        }
-
-        Ok(uuids)
-    }
-
-    pub fn load_dump(src: impl AsRef<Path>, dst: impl AsRef<Path>) -> Result<()> {
-        let uuid_resolver_path = dst.as_ref().join(UUIDS_DB_PATH);
-        std::fs::create_dir_all(&uuid_resolver_path)?;
-
-        let src_indexes = src.as_ref().join(UUIDS_DB_PATH).join("data.jsonl");
-        let indexes = File::open(&src_indexes)?;
-        let mut indexes = BufReader::new(indexes);
-        let mut line = String::new();
-
-        let db = Self::new(dst)?;
-        let mut txn = db.env.write_txn()?;
-
-        loop {
-            match indexes.read_line(&mut line) {
-                Ok(0) => break,
-                Ok(_) => {
-                    let DumpEntry { uuid, uid } = serde_json::from_str(&line)?;
-                    println!("importing {} {}", uid, uuid);
-                    db.db.put(&mut txn, &uid, uuid.as_bytes())?;
-                }
-                Err(e) => return Err(e.into()),
-            }
-
-            line.clear();
-        }
-        txn.commit()?;
-
-        db.env.prepare_for_closing().wait();
-
-        Ok(())
-    }
-}
-
-#[async_trait::async_trait]
-impl UuidStore for HeedUuidStore {
-    async fn get_uuid(&self, name: String) -> Result<Option<Uuid>> {
-        let this = self.clone();
-        tokio::task::spawn_blocking(move || this.get_uuid(name)).await?
-    }
-
-    async fn delete(&self, uid: String) -> Result<Option<Uuid>> {
-        let this = self.clone();
-        tokio::task::spawn_blocking(move || this.delete(uid)).await?
-    }
-
-    async fn list(&self) -> Result<Vec<(String, Uuid)>> {
-        let this = self.clone();
-        tokio::task::spawn_blocking(move || this.list()).await?
-    }
-
-    async fn insert(&self, name: String, uuid: Uuid) -> Result<()> {
-        let this = self.clone();
-        tokio::task::spawn_blocking(move || this.insert(name, uuid)).await?
-    }
-
-    async fn snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
-        let this = self.clone();
-        tokio::task::spawn_blocking(move || this.snapshot(path)).await?
-    }
-
-    async fn get_size(&self) -> Result<u64> {
-        self.get_size()
-    }
-
-    async fn dump(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
-        let this = self.clone();
-        tokio::task::spawn_blocking(move || this.dump(path)).await?
-    }
-}

meilisearch-http/src/lib.rs

@@ -1,109 +1,113 @@
-//! # MeiliSearch
-//! Hello there, future contributors. If you are here and see this code, it's probably because you want to add a super new fancy feature in MeiliSearch or fix a bug and first of all, thank you for that!
-//!
-//! To help you in this task, we'll try to do a little overview of the project.
-//! ## Milli
-//! [Milli](https://github.com/meilisearch/milli) is the core library of MeiliSearch. It's where we actually index documents and perform searches. Its purpose is to do these two tasks as fast as possible. You can give an update to milli, and it'll uses as many cores as provided to perform it as fast as possible. Nothing more. You can perform searches at the same time (search only uses one core).
-//! As you can see, we're missing quite a lot of features here; milli does not handle multiples indexes, it can't queue updates, it doesn't provide any web / API frontend, it doesn't implement dumps or snapshots, etc...
-//!
-//! ## `Index` module
-//! The [index] module is what encapsulates one milli index. It abstracts over its transaction and isolates a task that can be run into a thread. This is the unit of interaction with milli.
-//! If you add a feature to milli, you'll probably need to add it in this module too before exposing it to the rest of meilisearch.
-//!
-//! ## `IndexController` module
-//! To handle multiple indexes, we created an [index_controller]. It's in charge of creating new indexes, keeping references to all its indexes, forward asynchronous updates to its indexes, and provide an API to search in its indexes synchronously.
-//! To achieves this goal, we use an [actor model](https://en.wikipedia.org/wiki/Actor_model).
-//!
-//! ### The actor model
-//! Every actor is composed of at least three files:
-//! - `mod.rs` declare and import all the files used by the actor. We also describe the interface (= all the methods) used to interact with the actor. If you are not modifying anything inside of an actor, this is usually all you need to see.
-//! - `handle_impl.rs` implements the interface described in the `mod.rs`; in reality, there is no code logic in this file. Every method is only wrapping its parameters in a structure that is sent to the actor. This is useful for test and futureproofing.
-//! - `message.rs` contains an enum that describes all the interactions you can have with the actor.
-//! - `actor.rs` is used to create and execute the actor. It's where we'll write the loop looking for new messages and actually perform the tasks.
-//!
-//! MeiliSearch currently uses four actors:
-//! - [`uuid_resolver`](index_controller/uuid_resolver/index.html) hold the association between the user-provided indexes name and the internal [`uuid`](https://en.wikipedia.org/wiki/Universally_unique_identifier) representation we use.
-//! - [`index_actor`](index_controller::index_actor) is our representation of multiples indexes. Any request made to MeiliSearch that needs to talk to milli will pass through this actor.
-//! - [`update_actor`](index_controller/update_actor/index.html) is in charge of indexes updates. Since updates can take a long time to receive and  process, we need to:
-//!   1. Store them as fast as possible so we can continue to receive other updates even if nothing has been processed
-//!   2. Feed the `index_actor` with a new update every time it finished its current job.
-//! - [`dump_actor`](index_controller/dump_actor/index.html) this actor handle the  [dumps](https://docs.meilisearch.com/reference/api/dump.html). It needs to contact all the others actors and create a dump of everything that was currently happening.
-//!
-//! ## Data module
-//! The [data] module provide a unified interface to communicate with the index controller and other services (snapshot, dumps, ...), initialize the MeiliSearch instance
-//!
-//! ## HTTP server
-//! To handle the web and API part, we are using [actix-web](https://docs.rs/actix-web/); you can find all routes in the [routes] module.
-//! Currently, the configuration of actix-web is made in the [lib.rs](crate).
-//! Most of the routes use [extractors] to handle the authentication.
-
 #![allow(rustdoc::private_intra_doc_links)]
-pub mod data;
 #[macro_use]
 pub mod error;
+pub mod analytics;
+pub mod task;
 #[macro_use]
 pub mod extractors;
 pub mod helpers;
-mod index;
-mod index_controller;
 pub mod option;
 pub mod routes;
 
-#[cfg(all(not(debug_assertions), feature = "analytics"))]
-pub mod analytics;
+use std::sync::{atomic::AtomicBool, Arc};
+use std::time::Duration;
 
-use crate::extractors::authentication::AuthConfig;
-
-pub use self::data::Data;
+use crate::error::MeilisearchHttpError;
+use actix_web::error::JsonPayloadError;
+use analytics::Analytics;
+use error::PayloadError;
+use http::header::CONTENT_TYPE;
 pub use option::Opt;
 
-use actix_web::web;
+use actix_web::{web, HttpRequest};
 
-use extractors::authentication::policies::*;
 use extractors::payload::PayloadConfig;
+use meilisearch_auth::AuthController;
+use meilisearch_lib::MeiliSearch;
 
-pub fn configure_data(config: &mut web::ServiceConfig, data: Data) {
-    let http_payload_size_limit = data.http_payload_size_limit();
+pub static AUTOBATCHING_ENABLED: AtomicBool = AtomicBool::new(false);
+
+pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<MeiliSearch> {
+    let mut meilisearch = MeiliSearch::builder();
+
+    // enable autobatching?
+    AUTOBATCHING_ENABLED.store(
+        opt.scheduler_options.enable_auto_batching,
+        std::sync::atomic::Ordering::Relaxed,
+    );
+
+    meilisearch
+        .set_max_index_size(opt.max_index_size.get_bytes() as usize)
+        .set_max_task_store_size(opt.max_task_db_size.get_bytes() as usize)
+        // snapshot
+        .set_ignore_missing_snapshot(opt.ignore_missing_snapshot)
+        .set_ignore_snapshot_if_db_exists(opt.ignore_snapshot_if_db_exists)
+        .set_snapshot_interval(Duration::from_secs(opt.snapshot_interval_sec))
+        .set_snapshot_dir(opt.snapshot_dir.clone())
+        // dump
+        .set_ignore_missing_dump(opt.ignore_missing_dump)
+        .set_ignore_dump_if_db_exists(opt.ignore_dump_if_db_exists)
+        .set_dump_dst(opt.dumps_dir.clone());
+
+    if let Some(ref path) = opt.import_snapshot {
+        meilisearch.set_import_snapshot(path.clone());
+    }
+
+    if let Some(ref path) = opt.import_dump {
+        meilisearch.set_dump_src(path.clone());
+    }
+
+    if opt.schedule_snapshot {
+        meilisearch.set_schedule_snapshot();
+    }
+
+    meilisearch.build(
+        opt.db_path.clone(),
+        opt.indexer_options.clone(),
+        opt.scheduler_options.clone(),
+    )
+}
+
+pub fn configure_data(
+    config: &mut web::ServiceConfig,
+    data: MeiliSearch,
+    auth: AuthController,
+    opt: &Opt,
+    analytics: Arc<dyn Analytics>,
+) {
+    let http_payload_size_limit = opt.http_payload_size_limit.get_bytes() as usize;
     config
-        .data(data.clone())
         .app_data(data)
+        .app_data(auth)
+        .app_data(web::Data::from(analytics))
         .app_data(
             web::JsonConfig::default()
-                .limit(http_payload_size_limit)
-                .content_type(|_mime| true) // Accept all mime types
-                .error_handler(|err, _req| error::payload_error_handler(err).into()),
+                .content_type(|mime| mime == mime::APPLICATION_JSON)
+                .error_handler(|err, req: &HttpRequest| match err {
+                    JsonPayloadError::ContentType => match req.headers().get(CONTENT_TYPE) {
+                        Some(content_type) => MeilisearchHttpError::InvalidContentType(
+                            content_type.to_str().unwrap_or("unknown").to_string(),
+                            vec![mime::APPLICATION_JSON.to_string()],
+                        )
+                        .into(),
+                        None => MeilisearchHttpError::MissingContentType(vec![
+                            mime::APPLICATION_JSON.to_string(),
+                        ])
+                        .into(),
+                    },
+                    err => PayloadError::from(err).into(),
+                }),
         )
         .app_data(PayloadConfig::new(http_payload_size_limit))
         .app_data(
-            web::QueryConfig::default()
-                .error_handler(|err, _req| error::payload_error_handler(err).into()),
+            web::QueryConfig::default().error_handler(|err, _req| PayloadError::from(err).into()),
         );
 }
 
-pub fn configure_auth(config: &mut web::ServiceConfig, data: &Data) {
-    let keys = data.api_keys();
-    let auth_config = if let Some(ref master_key) = keys.master {
-        let private_key = keys.private.as_ref().unwrap();
-        let public_key = keys.public.as_ref().unwrap();
-        let mut policies = init_policies!(Public, Private, Admin);
-        create_users!(
-            policies,
-            master_key.as_bytes() => { Admin, Private, Public },
-            private_key.as_bytes() => { Private, Public },
-            public_key.as_bytes() => { Public }
-        );
-        AuthConfig::Auth(policies)
-    } else {
-        AuthConfig::NoAuth
-    };
-
-    config.app_data(auth_config);
-}
-
 #[cfg(feature = "mini-dashboard")]
 pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) {
     use actix_web::HttpResponse;
-    use actix_web_static_files::Resource;
+    use static_files::Resource;
 
     mod generated {
         include!(concat!(env!("OUT_DIR"), "/generated.rs"));
@@ -111,7 +115,6 @@ pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) {
 
     if enable_frontend {
         let generated = generated::generate();
-        let mut scope = web::scope("/");
         // Generate routes for mini-dashboard assets
         for (path, resource) in generated.into_iter() {
             let Resource {
@@ -119,16 +122,15 @@ pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) {
             } = resource;
             // Redirect index.html to /
             if path == "index.html" {
-                config.service(web::resource("/").route(
-                    web::get().to(move || HttpResponse::Ok().content_type(mime_type).body(data)),
-                ));
+                config.service(web::resource("/").route(web::get().to(move || async move {
+                    HttpResponse::Ok().content_type(mime_type).body(data)
+                })));
             } else {
-                scope = scope.service(web::resource(path).route(
-                    web::get().to(move || HttpResponse::Ok().content_type(mime_type).body(data)),
-                ));
+                config.service(web::resource(path).route(web::get().to(move || async move {
+                    HttpResponse::Ok().content_type(mime_type).body(data)
+                })));
             }
         }
-        config.service(scope);
     } else {
         config.service(web::resource("/").route(web::get().to(routes::running)));
     }
@@ -141,23 +143,24 @@ pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) {
 
 #[macro_export]
 macro_rules! create_app {
-    ($data:expr, $enable_frontend:expr) => {{
+    ($data:expr, $auth:expr, $enable_frontend:expr, $opt:expr, $analytics:expr) => {{
         use actix_cors::Cors;
         use actix_web::middleware::TrailingSlash;
         use actix_web::App;
         use actix_web::{middleware, web};
+        use meilisearch_http::error::MeilisearchHttpError;
         use meilisearch_http::routes;
-        use meilisearch_http::{configure_auth, configure_data, dashboard};
+        use meilisearch_http::{configure_data, dashboard};
+        use meilisearch_types::error::ResponseError;
 
         App::new()
-            .configure(|s| configure_data(s, $data.clone()))
-            .configure(|s| configure_auth(s, &$data))
+            .configure(|s| configure_data(s, $data.clone(), $auth.clone(), &$opt, $analytics))
             .configure(routes::configure)
             .configure(|s| dashboard(s, $enable_frontend))
             .wrap(
                 Cors::default()
                     .send_wildcard()
-                    .allowed_headers(vec!["content-type", "x-meili-api-key"])
+                    .allow_any_header()
                     .allow_any_origin()
                     .allow_any_method()
                     .max_age(86_400), // 24h

meilisearch-http/src/main.rs

@@ -1,21 +1,21 @@
 use std::env;
+use std::sync::Arc;
 
+use actix_web::http::KeepAlive;
 use actix_web::HttpServer;
-use main_error::MainError;
-use meilisearch_http::{create_app, Data, Opt};
-use structopt::StructOpt;
-
-#[cfg(all(not(debug_assertions), feature = "analytics"))]
+use clap::Parser;
+use meilisearch_auth::AuthController;
 use meilisearch_http::analytics;
+use meilisearch_http::analytics::Analytics;
+use meilisearch_http::{create_app, setup_meilisearch, Opt};
+use meilisearch_lib::MeiliSearch;
 
 #[cfg(target_os = "linux")]
 #[global_allocator]
-static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
-
-#[actix_web::main]
-async fn main() -> Result<(), MainError> {
-    let opt = Opt::from_args();
+static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 
+/// does all the setup before meilisearch is launched
+fn setup(opt: &Opt) -> anyhow::Result<()> {
     let mut log_builder = env_logger::Builder::new();
     log_builder.parse_filters(&opt.log_level);
     if opt.log_level == "info" {
@@ -25,40 +25,67 @@ async fn main() -> Result<(), MainError> {
 
     log_builder.init();
 
+    Ok(())
+}
+
+#[actix_web::main]
+async fn main() -> anyhow::Result<()> {
+    let opt = Opt::parse();
+
+    setup(&opt)?;
+
     match opt.env.as_ref() {
         "production" => {
             if opt.master_key.is_none() {
-                return Err(
+                anyhow::bail!(
                     "In production mode, the environment variable MEILI_MASTER_KEY is mandatory"
-                        .into(),
-                );
+                )
             }
         }
         "development" => (),
         _ => unreachable!(),
     }
 
-    let data = Data::new(opt.clone())?;
+    let meilisearch = setup_meilisearch(&opt)?;
+
+    let auth_controller = AuthController::new(&opt.db_path, &opt.master_key)?;
 
     #[cfg(all(not(debug_assertions), feature = "analytics"))]
-    if !opt.no_analytics {
-        let analytics_data = data.clone();
-        let analytics_opt = opt.clone();
-        tokio::task::spawn(analytics::analytics_sender(analytics_data, analytics_opt));
-    }
+    let (analytics, user) = if !opt.no_analytics {
+        analytics::SegmentAnalytics::new(&opt, &meilisearch).await
+    } else {
+        analytics::MockAnalytics::new(&opt)
+    };
+    #[cfg(any(debug_assertions, not(feature = "analytics")))]
+    let (analytics, user) = analytics::MockAnalytics::new(&opt);
 
-    print_launch_resume(&opt, &data);
+    print_launch_resume(&opt, &user);
 
-    run_http(data, opt).await?;
+    run_http(meilisearch, auth_controller, opt, analytics).await?;
 
     Ok(())
 }
 
-async fn run_http(data: Data, opt: Opt) -> Result<(), Box<dyn std::error::Error>> {
+async fn run_http(
+    data: MeiliSearch,
+    auth_controller: AuthController,
+    opt: Opt,
+    analytics: Arc<dyn Analytics>,
+) -> anyhow::Result<()> {
     let _enable_dashboard = &opt.env == "development";
-    let http_server = HttpServer::new(move || create_app!(data, _enable_dashboard))
-        // Disable signals allows the server to terminate immediately when a user enter CTRL-C
-        .disable_signals();
+    let opt_clone = opt.clone();
+    let http_server = HttpServer::new(move || {
+        create_app!(
+            data,
+            auth_controller,
+            _enable_dashboard,
+            opt_clone,
+            analytics.clone()
+        )
+    })
+    // Disable signals allows the server to terminate immediately when a user enter CTRL-C
+    .disable_signals()
+    .keep_alive(KeepAlive::Os);
 
     if let Some(config) = opt.get_ssl_config()? {
         http_server
@@ -66,24 +93,24 @@ async fn run_http(data: Data, opt: Opt) -> Result<(), Box<dyn std::error::Error>
             .run()
             .await?;
     } else {
-        http_server.bind(opt.http_addr)?.run().await?;
+        http_server.bind(&opt.http_addr)?.run().await?;
     }
     Ok(())
 }
 
-pub fn print_launch_resume(opt: &Opt, data: &Data) {
+pub fn print_launch_resume(opt: &Opt, user: &str) {
     let commit_sha = option_env!("VERGEN_GIT_SHA").unwrap_or("unknown");
     let commit_date = option_env!("VERGEN_GIT_COMMIT_TIMESTAMP").unwrap_or("unknown");
 
     let ascii_name = r#"
-888b     d888          d8b 888 d8b  .d8888b.                                    888
-8888b   d8888          Y8P 888 Y8P d88P  Y88b                                   888
-88888b.d88888              888     Y88b.                                        888
-888Y88888P888  .d88b.  888 888 888  "Y888b.    .d88b.   8888b.  888d888 .d8888b 88888b.
-888 Y888P 888 d8P  Y8b 888 888 888     "Y88b. d8P  Y8b     "88b 888P"  d88P"    888 "88b
-888  Y8P  888 88888888 888 888 888       "888 88888888 .d888888 888    888      888  888
-888   "   888 Y8b.     888 888 888 Y88b  d88P Y8b.     888  888 888    Y88b.    888  888
-888       888  "Y8888  888 888 888  "Y8888P"   "Y8888  "Y888888 888     "Y8888P 888  888
+888b     d888          d8b 888 d8b                                            888
+8888b   d8888          Y8P 888 Y8P                                            888
+88888b.d88888              888                                                888
+888Y88888P888  .d88b.  888 888 888 .d8888b   .d88b.   8888b.  888d888 .d8888b 88888b.
+888 Y888P 888 d8P  Y8b 888 888 888 88K      d8P  Y8b     "88b 888P"  d88P"    888 "88b
+888  Y8P  888 88888888 888 888 888 "Y8888b. 88888888 .d888888 888    888      888  888
+888   "   888 Y8b.     888 888 888      X88 Y8b.     888  888 888    Y88b.    888  888
+888       888  "Y8888  888 888 888  88888P'  "Y8888  "Y888888 888     "Y8888P 888  888
 "#;
 
     eprintln!("{}", ascii_name);
@@ -100,24 +127,28 @@ pub fn print_launch_resume(opt: &Opt, data: &Data) {
 
     #[cfg(all(not(debug_assertions), feature = "analytics"))]
     {
-        if opt.no_analytics {
-            eprintln!("Anonymous telemetry:\t\"Disabled\"");
-        } else {
+        if !opt.no_analytics {
             eprintln!(
                 "
-Thank you for using MeiliSearch!
+Thank you for using Meilisearch!
 
 We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html
 
-Anonymous telemetry:   \"Enabled\""
+Anonymous telemetry:\t\"Enabled\""
             );
+        } else {
+            eprintln!("Anonymous telemetry:\t\"Disabled\"");
         }
     }
 
+    if !user.is_empty() {
+        eprintln!("Instance UID:\t\t\"{}\"", user);
+    }
+
     eprintln!();
 
-    if data.api_keys().master.is_some() {
-        eprintln!("A Master Key has been set. Requests to MeiliSearch won't be authorized unless you provide an authentication key.");
+    if opt.master_key.is_some() {
+        eprintln!("A Master Key has been set. Requests to Meilisearch won't be authorized unless you provide an authentication key.");
     } else {
         eprintln!("No master key found; The server will accept unidentified requests. \
             If you need some protection in development mode, please export a key: export MEILI_MASTER_KEY=xxx");
@@ -126,6 +157,6 @@ Anonymous telemetry:   \"Enabled\""
     eprintln!();
     eprintln!("Documentation:\t\thttps://docs.meilisearch.com");
     eprintln!("Source code:\t\thttps://github.com/meilisearch/meilisearch");
-    eprintln!("Contact:\t\thttps://docs.meilisearch.com/resources/contact.html or bonjour@meilisearch.com");
+    eprintln!("Contact:\t\thttps://docs.meilisearch.com/resources/contact.html");
     eprintln!();
 }

meilisearch-http/src/option.rs

@@ -1,192 +1,172 @@
-use byte_unit::ByteError;
-use std::fmt;
+use std::fs;
 use std::io::{BufReader, Read};
-use std::ops::Deref;
 use std::path::PathBuf;
-use std::str::FromStr;
 use std::sync::Arc;
-use std::{error, fs};
 
 use byte_unit::Byte;
-use milli::CompressionType;
-use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
+use clap::Parser;
+use meilisearch_lib::options::{IndexerOpts, SchedulerConfig};
 use rustls::{
-    AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth,
+    server::{
+        AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient,
+        ServerSessionMemoryCache,
+    },
     RootCertStore,
 };
-use structopt::StructOpt;
-use sysinfo::{RefreshKind, System, SystemExt};
-
-#[derive(Debug, Clone, StructOpt)]
-pub struct IndexerOpts {
-    /// The amount of documents to skip before printing
-    /// a log regarding the indexing advancement.
-    #[structopt(long, default_value = "100000")] // 100k
-    pub log_every_n: usize,
-
-    /// Grenad max number of chunks in bytes.
-    #[structopt(long)]
-    pub max_nb_chunks: Option<usize>,
-
-    /// The maximum amount of memory the indexer will use. It defaults to 2/3
-    /// of the available memory. It is recommended to use something like 80%-90%
-    /// of the available memory, no more.
-    ///
-    /// In case the engine is unable to retrieve the available memory the engine will
-    /// try to use the memory it needs but without real limit, this can lead to
-    /// Out-Of-Memory issues and it is recommended to specify the amount of memory to use.
-    #[structopt(long, default_value)]
-    pub max_memory: MaxMemory,
-
-    /// The name of the compression algorithm to use when compressing intermediate
-    /// Grenad chunks while indexing documents.
-    ///
-    /// Choosing a fast algorithm will make the indexing faster but may consume more memory.
-    #[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])]
-    pub chunk_compression_type: CompressionType,
-
-    /// The level of compression of the chosen algorithm.
-    #[structopt(long, requires = "chunk-compression-type")]
-    pub chunk_compression_level: Option<u32>,
-
-    /// Number of parallel jobs for indexing, defaults to # of CPUs.
-    #[structopt(long)]
-    pub indexing_jobs: Option<usize>,
-}
-
-impl Default for IndexerOpts {
-    fn default() -> Self {
-        Self {
-            log_every_n: 100_000,
-            max_nb_chunks: None,
-            max_memory: MaxMemory::default(),
-            chunk_compression_type: CompressionType::None,
-            chunk_compression_level: None,
-            indexing_jobs: None,
-        }
-    }
-}
+use rustls_pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
+use serde::Serialize;
 
 const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
 
-#[derive(Debug, Clone, StructOpt)]
+#[derive(Debug, Clone, Parser, Serialize)]
+#[clap(version)]
 pub struct Opt {
     /// The destination where the database must be created.
-    #[structopt(long, env = "MEILI_DB_PATH", default_value = "./data.ms")]
+    #[clap(long, env = "MEILI_DB_PATH", default_value = "./data.ms")]
     pub db_path: PathBuf,
 
     /// The address on which the http server will listen.
-    #[structopt(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")]
+    #[clap(long, env = "MEILI_HTTP_ADDR", default_value = "127.0.0.1:7700")]
     pub http_addr: String,
 
     /// The master key allowing you to do everything on the server.
-    #[structopt(long, env = "MEILI_MASTER_KEY")]
+    #[serde(skip)]
+    #[clap(long, env = "MEILI_MASTER_KEY")]
     pub master_key: Option<String>,
 
     /// This environment variable must be set to `production` if you are running in production.
     /// If the server is running in development mode more logs will be displayed,
     /// and the master key can be avoided which implies that there is no security on the updates routes.
     /// This is useful to debug when integrating the engine with another service.
-    #[structopt(long, env = "MEILI_ENV", default_value = "development", possible_values = &POSSIBLE_ENV)]
+    #[clap(long, env = "MEILI_ENV", default_value = "development", possible_values = &POSSIBLE_ENV)]
     pub env: String,
 
     /// Do not send analytics to Meili.
     #[cfg(all(not(debug_assertions), feature = "analytics"))]
-    #[structopt(long, env = "MEILI_NO_ANALYTICS")]
+    #[serde(skip)] // we can't send true
+    #[clap(long, env = "MEILI_NO_ANALYTICS")]
     pub no_analytics: bool,
 
     /// The maximum size, in bytes, of the main lmdb database directory
-    #[structopt(long, env = "MEILI_MAX_INDEX_SIZE", default_value = "100 GiB")]
+    #[clap(long, env = "MEILI_MAX_INDEX_SIZE", default_value = "100 GiB")]
     pub max_index_size: Byte,
 
     /// The maximum size, in bytes, of the update lmdb database directory
-    #[structopt(long, env = "MEILI_MAX_UDB_SIZE", default_value = "100 GiB")]
-    pub max_udb_size: Byte,
+    #[clap(long, env = "MEILI_MAX_TASK_DB_SIZE", default_value = "100 GiB")]
+    pub max_task_db_size: Byte,
 
     /// The maximum size, in bytes, of accepted JSON payloads
-    #[structopt(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "100 MB")]
+    #[clap(long, env = "MEILI_HTTP_PAYLOAD_SIZE_LIMIT", default_value = "100 MB")]
     pub http_payload_size_limit: Byte,
 
     /// Read server certificates from CERTFILE.
     /// This should contain PEM-format certificates
     /// in the right order (the first certificate should
     /// certify KEYFILE, the last should be a root CA).
-    #[structopt(long, env = "MEILI_SSL_CERT_PATH", parse(from_os_str))]
+    #[serde(skip)]
+    #[clap(long, env = "MEILI_SSL_CERT_PATH", parse(from_os_str))]
     pub ssl_cert_path: Option<PathBuf>,
 
     /// Read private key from KEYFILE.  This should be a RSA
     /// private key or PKCS8-encoded private key, in PEM format.
-    #[structopt(long, env = "MEILI_SSL_KEY_PATH", parse(from_os_str))]
+    #[serde(skip)]
+    #[clap(long, env = "MEILI_SSL_KEY_PATH", parse(from_os_str))]
     pub ssl_key_path: Option<PathBuf>,
 
     /// Enable client authentication, and accept certificates
     /// signed by those roots provided in CERTFILE.
-    #[structopt(long, env = "MEILI_SSL_AUTH_PATH", parse(from_os_str))]
+    #[clap(long, env = "MEILI_SSL_AUTH_PATH", parse(from_os_str))]
+    #[serde(skip)]
     pub ssl_auth_path: Option<PathBuf>,
 
     /// Read DER-encoded OCSP response from OCSPFILE and staple to certificate.
     /// Optional
-    #[structopt(long, env = "MEILI_SSL_OCSP_PATH", parse(from_os_str))]
+    #[serde(skip)]
+    #[clap(long, env = "MEILI_SSL_OCSP_PATH", parse(from_os_str))]
     pub ssl_ocsp_path: Option<PathBuf>,
 
     /// Send a fatal alert if the client does not complete client authentication.
-    #[structopt(long, env = "MEILI_SSL_REQUIRE_AUTH")]
+    #[serde(skip)]
+    #[clap(long, env = "MEILI_SSL_REQUIRE_AUTH")]
     pub ssl_require_auth: bool,
 
     /// SSL support session resumption
-    #[structopt(long, env = "MEILI_SSL_RESUMPTION")]
+    #[serde(skip)]
+    #[clap(long, env = "MEILI_SSL_RESUMPTION")]
     pub ssl_resumption: bool,
 
     /// SSL support tickets.
-    #[structopt(long, env = "MEILI_SSL_TICKETS")]
+    #[serde(skip)]
+    #[clap(long, env = "MEILI_SSL_TICKETS")]
     pub ssl_tickets: bool,
 
     /// Defines the path of the snapshot file to import.
     /// This option will, by default, stop the process if a database already exist or if no snapshot exists at
     /// the given path. If this option is not specified no snapshot is imported.
-    #[structopt(long)]
+    #[clap(long)]
     pub import_snapshot: Option<PathBuf>,
 
     /// The engine will ignore a missing snapshot and not return an error in such case.
-    #[structopt(long, requires = "import-snapshot")]
+    #[clap(long, requires = "import-snapshot")]
     pub ignore_missing_snapshot: bool,
 
     /// The engine will skip snapshot importation and not return an error in such case.
-    #[structopt(long, requires = "import-snapshot")]
+    #[clap(long, requires = "import-snapshot")]
     pub ignore_snapshot_if_db_exists: bool,
 
     /// Defines the directory path where meilisearch will create snapshot each snapshot_time_gap.
-    #[structopt(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")]
+    #[clap(long, env = "MEILI_SNAPSHOT_DIR", default_value = "snapshots/")]
     pub snapshot_dir: PathBuf,
 
     /// Activate snapshot scheduling.
-    #[structopt(long, env = "MEILI_SCHEDULE_SNAPSHOT")]
+    #[clap(long, env = "MEILI_SCHEDULE_SNAPSHOT")]
     pub schedule_snapshot: bool,
 
     /// Defines time interval, in seconds, between each snapshot creation.
-    #[structopt(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC", default_value = "86400")] // 24h
+    #[clap(long, env = "MEILI_SNAPSHOT_INTERVAL_SEC", default_value = "86400")] // 24h
     pub snapshot_interval_sec: u64,
 
-    /// Folder where dumps are created when the dump route is called.
-    #[structopt(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")]
-    pub dumps_dir: PathBuf,
-
     /// Import a dump from the specified path, must be a `.dump` file.
-    #[structopt(long, conflicts_with = "import-snapshot")]
+    #[clap(long, conflicts_with = "import-snapshot")]
     pub import_dump: Option<PathBuf>,
 
+    /// If the dump doesn't exists, load or create the database specified by `db-path` instead.
+    #[clap(long, requires = "import-dump")]
+    pub ignore_missing_dump: bool,
+
+    /// Ignore the dump if a database already exists, and load that database instead.
+    #[clap(long, requires = "import-dump")]
+    pub ignore_dump_if_db_exists: bool,
+
+    /// Folder where dumps are created when the dump route is called.
+    #[clap(long, env = "MEILI_DUMPS_DIR", default_value = "dumps/")]
+    pub dumps_dir: PathBuf,
+
     /// Set the log level
-    #[structopt(long, env = "MEILI_LOG_LEVEL", default_value = "info")]
+    #[clap(long, env = "MEILI_LOG_LEVEL", default_value = "info")]
     pub log_level: String,
 
-    #[structopt(skip)]
+    #[serde(flatten)]
+    #[clap(flatten)]
     pub indexer_options: IndexerOpts,
+
+    #[serde(flatten)]
+    #[clap(flatten)]
+    pub scheduler_options: SchedulerConfig,
 }
 
 impl Opt {
-    pub fn get_ssl_config(&self) -> Result<Option<rustls::ServerConfig>, Box<dyn error::Error>> {
+    /// Wether analytics should be enabled or not.
+    #[cfg(all(not(debug_assertions), feature = "analytics"))]
+    pub fn analytics(&self) -> bool {
+        !self.no_analytics
+    }
+
+    pub fn get_ssl_config(&self) -> anyhow::Result<Option<rustls::ServerConfig>> {
         if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) {
-            let client_auth = match &self.ssl_auth_path {
+            let config = rustls::ServerConfig::builder().with_safe_defaults();
+
+            let config = match &self.ssl_auth_path {
                 Some(auth_path) => {
                     let roots = load_certs(auth_path.to_path_buf())?;
                     let mut client_auth_roots = RootCertStore::empty();
@@ -194,30 +174,32 @@ impl Opt {
                         client_auth_roots.add(&root).unwrap();
                     }
                     if self.ssl_require_auth {
-                        AllowAnyAuthenticatedClient::new(client_auth_roots)
+                        let verifier = AllowAnyAuthenticatedClient::new(client_auth_roots);
+                        config.with_client_cert_verifier(verifier)
                     } else {
-                        AllowAnyAnonymousOrAuthenticatedClient::new(client_auth_roots)
+                        let verifier =
+                            AllowAnyAnonymousOrAuthenticatedClient::new(client_auth_roots);
+                        config.with_client_cert_verifier(verifier)
                     }
                 }
-                None => NoClientAuth::new(),
+                None => config.with_no_client_auth(),
             };
 
-            let mut config = rustls::ServerConfig::new(client_auth);
-            config.key_log = Arc::new(rustls::KeyLogFile::new());
-
             let certs = load_certs(cert_path.to_path_buf())?;
             let privkey = load_private_key(key_path.to_path_buf())?;
             let ocsp = load_ocsp(&self.ssl_ocsp_path)?;
-            config
-                .set_single_cert_with_ocsp_and_sct(certs, privkey, ocsp, vec![])
-                .map_err(|_| "bad certificates/private key")?;
+            let mut config = config
+                .with_single_cert_with_ocsp_and_sct(certs, privkey, ocsp, vec![])
+                .map_err(|_| anyhow::anyhow!("bad certificates/private key"))?;
+
+            config.key_log = Arc::new(rustls::KeyLogFile::new());
 
             if self.ssl_resumption {
-                config.set_persistence(rustls::ServerSessionMemoryCache::new(256));
+                config.session_storage = ServerSessionMemoryCache::new(256);
             }
 
             if self.ssl_tickets {
-                config.ticketer = rustls::Ticketer::new();
+                config.ticketer = rustls::Ticketer::new().unwrap();
             }
 
             Ok(Some(config))
@@ -227,102 +209,63 @@ impl Opt {
     }
 }
 
-/// A type used to detect the max memory available and use 2/3 of it.
-#[derive(Debug, Clone, Copy)]
-pub struct MaxMemory(Option<Byte>);
-
-impl FromStr for MaxMemory {
-    type Err = ByteError;
-
-    fn from_str(s: &str) -> Result<MaxMemory, ByteError> {
-        Byte::from_str(s).map(Some).map(MaxMemory)
-    }
-}
-
-impl Default for MaxMemory {
-    fn default() -> MaxMemory {
-        MaxMemory(
-            total_memory_bytes()
-                .map(|bytes| bytes * 2 / 3)
-                .map(Byte::from_bytes),
-        )
-    }
-}
-
-impl fmt::Display for MaxMemory {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self.0 {
-            Some(memory) => write!(f, "{}", memory.get_appropriate_unit(true)),
-            None => f.write_str("unknown"),
-        }
-    }
-}
-
-impl Deref for MaxMemory {
-    type Target = Option<Byte>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl MaxMemory {
-    pub fn unlimited() -> Self {
-        Self(None)
-    }
-}
-
-/// Returns the total amount of bytes available or `None` if this system isn't supported.
-fn total_memory_bytes() -> Option<u64> {
-    if System::IS_SUPPORTED {
-        let memory_kind = RefreshKind::new().with_memory();
-        let mut system = System::new_with_specifics(memory_kind);
-        system.refresh_memory();
-        Some(system.total_memory() * 1024) // KiB into bytes
-    } else {
-        None
-    }
-}
-
-fn load_certs(filename: PathBuf) -> Result<Vec<rustls::Certificate>, Box<dyn error::Error>> {
-    let certfile = fs::File::open(filename).map_err(|_| "cannot open certificate file")?;
+fn load_certs(filename: PathBuf) -> anyhow::Result<Vec<rustls::Certificate>> {
+    let certfile =
+        fs::File::open(filename).map_err(|_| anyhow::anyhow!("cannot open certificate file"))?;
     let mut reader = BufReader::new(certfile);
-    Ok(certs(&mut reader).map_err(|_| "cannot read certificate file")?)
+    certs(&mut reader)
+        .map(|certs| certs.into_iter().map(rustls::Certificate).collect())
+        .map_err(|_| anyhow::anyhow!("cannot read certificate file"))
 }
 
-fn load_private_key(filename: PathBuf) -> Result<rustls::PrivateKey, Box<dyn error::Error>> {
+fn load_private_key(filename: PathBuf) -> anyhow::Result<rustls::PrivateKey> {
     let rsa_keys = {
-        let keyfile =
-            fs::File::open(filename.clone()).map_err(|_| "cannot open private key file")?;
+        let keyfile = fs::File::open(filename.clone())
+            .map_err(|_| anyhow::anyhow!("cannot open private key file"))?;
         let mut reader = BufReader::new(keyfile);
-        rsa_private_keys(&mut reader).map_err(|_| "file contains invalid rsa private key")?
+        rsa_private_keys(&mut reader)
+            .map_err(|_| anyhow::anyhow!("file contains invalid rsa private key"))?
     };
 
     let pkcs8_keys = {
-        let keyfile = fs::File::open(filename).map_err(|_| "cannot open private key file")?;
+        let keyfile = fs::File::open(filename)
+            .map_err(|_| anyhow::anyhow!("cannot open private key file"))?;
         let mut reader = BufReader::new(keyfile);
-        pkcs8_private_keys(&mut reader)
-            .map_err(|_| "file contains invalid pkcs8 private key (encrypted keys not supported)")?
+        pkcs8_private_keys(&mut reader).map_err(|_| {
+            anyhow::anyhow!(
+                "file contains invalid pkcs8 private key (encrypted keys not supported)"
+            )
+        })?
     };
 
     // prefer to load pkcs8 keys
     if !pkcs8_keys.is_empty() {
-        Ok(pkcs8_keys[0].clone())
+        Ok(rustls::PrivateKey(pkcs8_keys[0].clone()))
     } else {
         assert!(!rsa_keys.is_empty());
-        Ok(rsa_keys[0].clone())
+        Ok(rustls::PrivateKey(rsa_keys[0].clone()))
     }
 }
 
-fn load_ocsp(filename: &Option<PathBuf>) -> Result<Vec<u8>, Box<dyn error::Error>> {
+fn load_ocsp(filename: &Option<PathBuf>) -> anyhow::Result<Vec<u8>> {
     let mut ret = Vec::new();
 
     if let Some(ref name) = filename {
         fs::File::open(name)
-            .map_err(|_| "cannot open ocsp file")?
+            .map_err(|_| anyhow::anyhow!("cannot open ocsp file"))?
             .read_to_end(&mut ret)
-            .map_err(|_| "cannot read oscp file")?;
+            .map_err(|_| anyhow::anyhow!("cannot read oscp file"))?;
     }
 
     Ok(ret)
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_valid_opt() {
+        assert!(Opt::try_parse_from(Some("")).is_ok());
+    }
+}

meilisearch-http/src/routes/api_key.rs

@@ -0,0 +1,160 @@
+use std::str;
+
+use actix_web::{web, HttpRequest, HttpResponse};
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use time::OffsetDateTime;
+use uuid::Uuid;
+
+use meilisearch_auth::{error::AuthControllerError, Action, AuthController, Key};
+use meilisearch_types::error::{Code, ResponseError};
+
+use crate::extractors::{
+    authentication::{policies::*, GuardedData},
+    sequential_extractor::SeqHandler,
+};
+use crate::routes::Pagination;
+
+pub fn configure(cfg: &mut web::ServiceConfig) {
+    cfg.service(
+        web::resource("")
+            .route(web::post().to(SeqHandler(create_api_key)))
+            .route(web::get().to(SeqHandler(list_api_keys))),
+    )
+    .service(
+        web::resource("/{key}")
+            .route(web::get().to(SeqHandler(get_api_key)))
+            .route(web::patch().to(SeqHandler(patch_api_key)))
+            .route(web::delete().to(SeqHandler(delete_api_key))),
+    );
+}
+
+pub async fn create_api_key(
+    auth_controller: GuardedData<ActionPolicy<{ actions::KEYS_CREATE }>, AuthController>,
+    body: web::Json<Value>,
+    _req: HttpRequest,
+) -> Result<HttpResponse, ResponseError> {
+    let v = body.into_inner();
+    let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> {
+        let key = auth_controller.create_key(v)?;
+        Ok(KeyView::from_key(key, &auth_controller))
+    })
+    .await
+    .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??;
+
+    Ok(HttpResponse::Created().json(res))
+}
+
+pub async fn list_api_keys(
+    auth_controller: GuardedData<ActionPolicy<{ actions::KEYS_GET }>, AuthController>,
+    paginate: web::Query<Pagination>,
+) -> Result<HttpResponse, ResponseError> {
+    let page_view = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> {
+        let keys = auth_controller.list_keys()?;
+        let page_view = paginate.auto_paginate_sized(
+            keys.into_iter()
+                .map(|k| KeyView::from_key(k, &auth_controller)),
+        );
+
+        Ok(page_view)
+    })
+    .await
+    .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??;
+
+    Ok(HttpResponse::Ok().json(page_view))
+}
+
+pub async fn get_api_key(
+    auth_controller: GuardedData<ActionPolicy<{ actions::KEYS_GET }>, AuthController>,
+    path: web::Path<AuthParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let key = path.into_inner().key;
+
+    let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> {
+        let uid =
+            Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?;
+        let key = auth_controller.get_key(uid)?;
+
+        Ok(KeyView::from_key(key, &auth_controller))
+    })
+    .await
+    .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??;
+
+    Ok(HttpResponse::Ok().json(res))
+}
+
+pub async fn patch_api_key(
+    auth_controller: GuardedData<ActionPolicy<{ actions::KEYS_UPDATE }>, AuthController>,
+    body: web::Json<Value>,
+    path: web::Path<AuthParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let key = path.into_inner().key;
+    let body = body.into_inner();
+    let res = tokio::task::spawn_blocking(move || -> Result<_, AuthControllerError> {
+        let uid =
+            Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?;
+        let key = auth_controller.update_key(uid, body)?;
+
+        Ok(KeyView::from_key(key, &auth_controller))
+    })
+    .await
+    .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??;
+
+    Ok(HttpResponse::Ok().json(res))
+}
+
+pub async fn delete_api_key(
+    auth_controller: GuardedData<ActionPolicy<{ actions::KEYS_DELETE }>, AuthController>,
+    path: web::Path<AuthParam>,
+) -> Result<HttpResponse, ResponseError> {
+    let key = path.into_inner().key;
+    tokio::task::spawn_blocking(move || {
+        let uid =
+            Uuid::parse_str(&key).or_else(|_| auth_controller.get_uid_from_encoded_key(&key))?;
+        auth_controller.delete_key(uid)
+    })
+    .await
+    .map_err(|e| ResponseError::from_msg(e.to_string(), Code::Internal))??;
+
+    Ok(HttpResponse::NoContent().finish())
+}
+
+#[derive(Deserialize)]
+pub struct AuthParam {
+    key: String,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct KeyView {
+    name: Option<String>,
+    description: Option<String>,
+    key: String,
+    uid: Uuid,
+    actions: Vec<Action>,
+    indexes: Vec<String>,
+    #[serde(serialize_with = "time::serde::rfc3339::option::serialize")]
+    expires_at: Option<OffsetDateTime>,
+    #[serde(serialize_with = "time::serde::rfc3339::serialize")]
+    created_at: OffsetDateTime,
+    #[serde(serialize_with = "time::serde::rfc3339::serialize")]
+    updated_at: OffsetDateTime,
+}
+
+impl KeyView {
+    fn from_key(key: Key, auth: &AuthController) -> Self {
+        let generated_key = auth.generate_key(key.uid).unwrap_or_default();
+
+        KeyView {
+            name: key.name,
+            description: key.description,
+            key: generated_key,
+            uid: key.uid,
+            actions: key.actions,
+            indexes: key.indexes.into_iter().map(String::from).collect(),
+            expires_at: key.expires_at,
+            created_at: key.created_at,
+            updated_at: key.updated_at,
+        }
+    }
+}

meilisearch-http/src/routes/dump.rs

@@ -1,40 +1,27 @@
-use actix_web::{web, HttpResponse};
+use actix_web::{web, HttpRequest, HttpResponse};
 use log::debug;
-use serde::{Deserialize, Serialize};
+use meilisearch_lib::MeiliSearch;
+use meilisearch_types::error::ResponseError;
+use serde_json::json;
 
-use crate::error::ResponseError;
+use crate::analytics::Analytics;
 use crate::extractors::authentication::{policies::*, GuardedData};
-use crate::Data;
+use crate::extractors::sequential_extractor::SeqHandler;
+use crate::task::SummarizedTaskView;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
-    cfg.service(web::resource("").route(web::post().to(create_dump)))
-        .service(web::resource("/{dump_uid}/status").route(web::get().to(get_dump_status)));
+    cfg.service(web::resource("").route(web::post().to(SeqHandler(create_dump))));
 }
 
-pub async fn create_dump(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
-    let res = data.create_dump().await?;
+pub async fn create_dump(
+    meilisearch: GuardedData<ActionPolicy<{ actions::DUMPS_CREATE }>, MeiliSearch>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
+) -> Result<HttpResponse, ResponseError> {
+    analytics.publish("Dump Created".to_string(), json!({}), Some(&req));
+
+    let res: SummarizedTaskView = meilisearch.register_dump_task().await?.into();
 
     debug!("returns: {:?}", res);
     Ok(HttpResponse::Accepted().json(res))
 }
-
-#[derive(Debug, Serialize)]
-#[serde(rename_all = "camelCase")]
-struct DumpStatusResponse {
-    status: String,
-}
-
-#[derive(Deserialize)]
-struct DumpParam {
-    dump_uid: String,
-}
-
-async fn get_dump_status(
-    data: GuardedData<Private, Data>,
-    path: web::Path<DumpParam>,
-) -> Result<HttpResponse, ResponseError> {
-    let res = data.dump_status(path.dump_uid.clone()).await?;
-
-    debug!("returns: {:?}", res);
-    Ok(HttpResponse::Ok().json(res))
-}

meilisearch-http/src/routes/indexes/documents.rs

@@ -1,50 +1,65 @@
-use actix_web::{web, HttpResponse};
+use actix_web::error::PayloadError;
+use actix_web::http::header::CONTENT_TYPE;
+use actix_web::web::Bytes;
+use actix_web::HttpMessage;
+use actix_web::{web, HttpRequest, HttpResponse};
+use bstr::ByteSlice;
+use futures::{Stream, StreamExt};
 use log::debug;
-use milli::update::{IndexDocumentsMethod, UpdateFormat};
+use meilisearch_lib::index_controller::{DocumentAdditionFormat, Update};
+use meilisearch_lib::milli::update::IndexDocumentsMethod;
+use meilisearch_lib::MeiliSearch;
+use meilisearch_types::error::ResponseError;
+use meilisearch_types::star_or::StarOr;
+use mime::Mime;
+use once_cell::sync::Lazy;
 use serde::Deserialize;
+use serde_cs::vec::CS;
 use serde_json::Value;
+use tokio::sync::mpsc;
 
-use crate::error::ResponseError;
+use crate::analytics::Analytics;
+use crate::error::MeilisearchHttpError;
 use crate::extractors::authentication::{policies::*, GuardedData};
 use crate::extractors::payload::Payload;
-use crate::routes::IndexParam;
-use crate::Data;
+use crate::extractors::sequential_extractor::SeqHandler;
+use crate::routes::{fold_star_or, PaginationView};
+use crate::task::SummarizedTaskView;
 
-const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0;
-const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20;
+static ACCEPTED_CONTENT_TYPE: Lazy<Vec<String>> = Lazy::new(|| {
+    vec![
+        "application/json".to_string(),
+        "application/x-ndjson".to_string(),
+        "text/csv".to_string(),
+    ]
+});
 
-/*
-macro_rules! guard_content_type {
-    ($fn_name:ident, $guard_value:literal) => {
-        fn $fn_name(head: &actix_web::dev::RequestHead) -> bool {
-            if let Some(content_type) = head.headers.get("Content-Type") {
-                content_type
-                    .to_str()
-                    .map(|v| v.contains($guard_value))
-                    .unwrap_or(false)
-            } else {
-                false
-            }
+/// This is required because Payload is not Sync nor Send
+fn payload_to_stream(mut payload: Payload) -> impl Stream<Item = Result<Bytes, PayloadError>> {
+    let (snd, recv) = mpsc::channel(1);
+    tokio::task::spawn_local(async move {
+        while let Some(data) = payload.next().await {
+            let _ = snd.send(data).await;
         }
-    };
+    });
+    tokio_stream::wrappers::ReceiverStream::new(recv)
 }
 
-guard_content_type!(guard_json, "application/json");
-*/
-
-fn guard_json(head: &actix_web::dev::RequestHead) -> bool {
-    if let Some(_content_type) = head.headers.get("Content-Type") {
-        // CURRENTLY AND FOR THIS RELEASE ONLY WE DECIDED TO INTERPRET ALL CONTENT-TYPES AS JSON
-        true
-        /*
-        content_type
-            .to_str()
-            .map(|v| v.contains("application/json"))
-            .unwrap_or(false)
-        */
-    } else {
-        // if no content-type is specified we still accept the data as json!
-        true
+/// Extracts the mime type from the content type and return
+/// a meilisearch error if anything bad happen.
+fn extract_mime_type(req: &HttpRequest) -> Result<Option<Mime>, MeilisearchHttpError> {
+    match req.mime_type() {
+        Ok(Some(mime)) => Ok(Some(mime)),
+        Ok(None) => Ok(None),
+        Err(_) => match req.headers().get(CONTENT_TYPE) {
+            Some(content_type) => Err(MeilisearchHttpError::InvalidContentType(
+                content_type.as_bytes().as_bstr().to_string(),
+                ACCEPTED_CONTENT_TYPE.clone(),
+            )),
+            None => Err(MeilisearchHttpError::MissingContentType(
+                ACCEPTED_CONTENT_TYPE.clone(),
+            )),
+        },
     }
 }
 
@@ -57,136 +72,208 @@ pub struct DocumentParam {
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(
         web::resource("")
-            .route(web::get().to(get_all_documents))
-            .route(web::post().guard(guard_json).to(add_documents))
-            .route(web::put().guard(guard_json).to(update_documents))
-            .route(web::delete().to(clear_all_documents)),
+            .route(web::get().to(SeqHandler(get_all_documents)))
+            .route(web::post().to(SeqHandler(add_documents)))
+            .route(web::put().to(SeqHandler(update_documents)))
+            .route(web::delete().to(SeqHandler(clear_all_documents))),
     )
     // this route needs to be before the /documents/{document_id} to match properly
-    .service(web::resource("/delete-batch").route(web::post().to(delete_documents)))
+    .service(web::resource("/delete-batch").route(web::post().to(SeqHandler(delete_documents))))
     .service(
         web::resource("/{document_id}")
-            .route(web::get().to(get_document))
-            .route(web::delete().to(delete_document)),
+            .route(web::get().to(SeqHandler(get_document)))
+            .route(web::delete().to(SeqHandler(delete_document))),
     );
 }
 
+#[derive(Deserialize, Debug)]
+#[serde(rename_all = "camelCase", deny_unknown_fields)]
+pub struct GetDocument {
+    fields: Option<CS<StarOr<String>>>,
+}
+
 pub async fn get_document(
-    data: GuardedData<Public, Data>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::DOCUMENTS_GET }>, MeiliSearch>,
     path: web::Path<DocumentParam>,
+    params: web::Query<GetDocument>,
 ) -> Result<HttpResponse, ResponseError> {
     let index = path.index_uid.clone();
     let id = path.document_id.clone();
-    let document = data
-        .retrieve_document(index, id, None as Option<Vec<String>>)
+    let GetDocument { fields } = params.into_inner();
+    let attributes_to_retrieve = fields.and_then(fold_star_or);
+
+    let document = meilisearch
+        .document(index, id, attributes_to_retrieve)
         .await?;
     debug!("returns: {:?}", document);
     Ok(HttpResponse::Ok().json(document))
 }
 
 pub async fn delete_document(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::DOCUMENTS_DELETE }>, MeiliSearch>,
     path: web::Path<DocumentParam>,
 ) -> Result<HttpResponse, ResponseError> {
-    let update_status = data
-        .delete_documents(path.index_uid.clone(), vec![path.document_id.clone()])
-        .await?;
-    debug!("returns: {:?}", update_status);
-    Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
+    let DocumentParam {
+        document_id,
+        index_uid,
+    } = path.into_inner();
+    let update = Update::DeleteDocuments(vec![document_id]);
+    let task: SummarizedTaskView = meilisearch.register_update(index_uid, update).await?.into();
+    debug!("returns: {:?}", task);
+    Ok(HttpResponse::Accepted().json(task))
 }
 
 #[derive(Deserialize, Debug)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct BrowseQuery {
-    offset: Option<usize>,
-    limit: Option<usize>,
-    attributes_to_retrieve: Option<String>,
+    #[serde(default)]
+    offset: usize,
+    #[serde(default = "crate::routes::PAGINATION_DEFAULT_LIMIT")]
+    limit: usize,
+    fields: Option<CS<StarOr<String>>>,
 }
 
 pub async fn get_all_documents(
-    data: GuardedData<Public, Data>,
-    path: web::Path<IndexParam>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::DOCUMENTS_GET }>, MeiliSearch>,
+    path: web::Path<String>,
     params: web::Query<BrowseQuery>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!("called with params: {:?}", params);
-    let attributes_to_retrieve = params.attributes_to_retrieve.as_ref().and_then(|attrs| {
-        let mut names = Vec::new();
-        for name in attrs.split(',').map(String::from) {
-            if name == "*" {
-                return None;
-            }
-            names.push(name);
-        }
-        Some(names)
-    });
+    let BrowseQuery {
+        limit,
+        offset,
+        fields,
+    } = params.into_inner();
+    let attributes_to_retrieve = fields.and_then(fold_star_or);
 
-    let documents = data
-        .retrieve_documents(
-            path.index_uid.clone(),
-            params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET),
-            params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT),
-            attributes_to_retrieve,
-        )
+    let (total, documents) = meilisearch
+        .documents(path.into_inner(), offset, limit, attributes_to_retrieve)
         .await?;
-    debug!("returns: {:?}", documents);
-    Ok(HttpResponse::Ok().json(documents))
+
+    let ret = PaginationView::new(offset, limit, total as usize, documents);
+
+    debug!("returns: {:?}", ret);
+    Ok(HttpResponse::Ok().json(ret))
 }
 
 #[derive(Deserialize, Debug)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct UpdateDocumentsQuery {
-    primary_key: Option<String>,
+    pub primary_key: Option<String>,
 }
 
-/// Route used when the payload type is "application/json"
-/// Used to add or replace documents
 pub async fn add_documents(
-    data: GuardedData<Private, Data>,
-    path: web::Path<IndexParam>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::DOCUMENTS_ADD }>, MeiliSearch>,
+    path: web::Path<String>,
     params: web::Query<UpdateDocumentsQuery>,
     body: Payload,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!("called with params: {:?}", params);
-    let update_status = data
-        .add_documents(
-            path.into_inner().index_uid,
-            IndexDocumentsMethod::ReplaceDocuments,
-            UpdateFormat::Json,
-            body,
-            params.primary_key.clone(),
-        )
-        .await?;
+    let params = params.into_inner();
+    let index_uid = path.into_inner();
 
-    debug!("returns: {:?}", update_status);
-    Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
+    analytics.add_documents(
+        &params,
+        meilisearch.get_index(index_uid.clone()).await.is_err(),
+        &req,
+    );
+
+    let allow_index_creation = meilisearch.filters().allow_index_creation;
+    let task = document_addition(
+        extract_mime_type(&req)?,
+        meilisearch,
+        index_uid,
+        params.primary_key,
+        body,
+        IndexDocumentsMethod::ReplaceDocuments,
+        allow_index_creation,
+    )
+    .await?;
+
+    Ok(HttpResponse::Accepted().json(task))
 }
 
-/// Route used when the payload type is "application/json"
-/// Used to add or replace documents
 pub async fn update_documents(
-    data: GuardedData<Private, Data>,
-    path: web::Path<IndexParam>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::DOCUMENTS_ADD }>, MeiliSearch>,
+    path: web::Path<String>,
     params: web::Query<UpdateDocumentsQuery>,
     body: Payload,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!("called with params: {:?}", params);
-    let update = data
-        .add_documents(
-            path.into_inner().index_uid,
-            IndexDocumentsMethod::UpdateDocuments,
-            UpdateFormat::Json,
-            body,
-            params.primary_key.clone(),
-        )
-        .await?;
+    let index_uid = path.into_inner();
 
-    debug!("returns: {:?}", update);
-    Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() })))
+    analytics.update_documents(
+        &params,
+        meilisearch.get_index(index_uid.clone()).await.is_err(),
+        &req,
+    );
+
+    let allow_index_creation = meilisearch.filters().allow_index_creation;
+    let task = document_addition(
+        extract_mime_type(&req)?,
+        meilisearch,
+        index_uid,
+        params.into_inner().primary_key,
+        body,
+        IndexDocumentsMethod::UpdateDocuments,
+        allow_index_creation,
+    )
+    .await?;
+
+    Ok(HttpResponse::Accepted().json(task))
+}
+
+async fn document_addition(
+    mime_type: Option<Mime>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::DOCUMENTS_ADD }>, MeiliSearch>,
+    index_uid: String,
+    primary_key: Option<String>,
+    body: Payload,
+    method: IndexDocumentsMethod,
+    allow_index_creation: bool,
+) -> Result<SummarizedTaskView, ResponseError> {
+    let format = match mime_type
+        .as_ref()
+        .map(|m| (m.type_().as_str(), m.subtype().as_str()))
+    {
+        Some(("application", "json")) => DocumentAdditionFormat::Json,
+        Some(("application", "x-ndjson")) => DocumentAdditionFormat::Ndjson,
+        Some(("text", "csv")) => DocumentAdditionFormat::Csv,
+        Some((type_, subtype)) => {
+            return Err(MeilisearchHttpError::InvalidContentType(
+                format!("{}/{}", type_, subtype),
+                ACCEPTED_CONTENT_TYPE.clone(),
+            )
+            .into())
+        }
+        None => {
+            return Err(
+                MeilisearchHttpError::MissingContentType(ACCEPTED_CONTENT_TYPE.clone()).into(),
+            )
+        }
+    };
+
+    let update = Update::DocumentAddition {
+        payload: Box::new(payload_to_stream(body)),
+        primary_key,
+        method,
+        format,
+        allow_index_creation,
+    };
+
+    let task = meilisearch.register_update(index_uid, update).await?.into();
+
+    debug!("returns: {:?}", task);
+    Ok(task)
 }
 
 pub async fn delete_documents(
-    data: GuardedData<Private, Data>,
-    path: web::Path<IndexParam>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::DOCUMENTS_DELETE }>, MeiliSearch>,
+    path: web::Path<String>,
     body: web::Json<Vec<Value>>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!("called with params: {:?}", body);
@@ -199,16 +286,26 @@ pub async fn delete_documents(
         })
         .collect();
 
-    let update_status = data.delete_documents(path.index_uid.clone(), ids).await?;
-    debug!("returns: {:?}", update_status);
-    Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
+    let update = Update::DeleteDocuments(ids);
+    let task: SummarizedTaskView = meilisearch
+        .register_update(path.into_inner(), update)
+        .await?
+        .into();
+
+    debug!("returns: {:?}", task);
+    Ok(HttpResponse::Accepted().json(task))
 }
 
 pub async fn clear_all_documents(
-    data: GuardedData<Private, Data>,
-    path: web::Path<IndexParam>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::DOCUMENTS_DELETE }>, MeiliSearch>,
+    path: web::Path<String>,
 ) -> Result<HttpResponse, ResponseError> {
-    let update_status = data.clear_documents(path.index_uid.clone()).await?;
-    debug!("returns: {:?}", update_status);
-    Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
+    let update = Update::ClearDocuments;
+    let task: SummarizedTaskView = meilisearch
+        .register_update(path.into_inner(), update)
+        .await?
+        .into();
+
+    debug!("returns: {:?}", task);
+    Ok(HttpResponse::Accepted().json(task))
 }

meilisearch-http/src/routes/indexes/mod.rs

@@ -1,44 +1,60 @@
-use actix_web::{web, HttpResponse};
-use chrono::{DateTime, Utc};
+use actix_web::{web, HttpRequest, HttpResponse};
 use log::debug;
+use meilisearch_lib::index_controller::Update;
+use meilisearch_lib::MeiliSearch;
+use meilisearch_types::error::ResponseError;
 use serde::{Deserialize, Serialize};
+use serde_json::json;
+use time::OffsetDateTime;
 
-use crate::error::ResponseError;
+use crate::analytics::Analytics;
 use crate::extractors::authentication::{policies::*, GuardedData};
-use crate::routes::IndexParam;
-use crate::Data;
+use crate::extractors::sequential_extractor::SeqHandler;
+use crate::task::SummarizedTaskView;
+
+use super::Pagination;
 
 pub mod documents;
 pub mod search;
 pub mod settings;
-pub mod updates;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(
         web::resource("")
             .route(web::get().to(list_indexes))
-            .route(web::post().to(create_index)),
+            .route(web::post().to(SeqHandler(create_index))),
     )
     .service(
         web::scope("/{index_uid}")
             .service(
                 web::resource("")
-                    .route(web::get().to(get_index))
-                    .route(web::put().to(update_index))
-                    .route(web::delete().to(delete_index)),
+                    .route(web::get().to(SeqHandler(get_index)))
+                    .route(web::patch().to(SeqHandler(update_index)))
+                    .route(web::delete().to(SeqHandler(delete_index))),
             )
-            .service(web::resource("/stats").route(web::get().to(get_index_stats)))
+            .service(web::resource("/stats").route(web::get().to(SeqHandler(get_index_stats))))
             .service(web::scope("/documents").configure(documents::configure))
             .service(web::scope("/search").configure(search::configure))
-            .service(web::scope("/updates").configure(updates::configure))
             .service(web::scope("/settings").configure(settings::configure)),
     );
 }
 
-pub async fn list_indexes(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
-    let indexes = data.list_indexes().await?;
-    debug!("returns: {:?}", indexes);
-    Ok(HttpResponse::Ok().json(indexes))
+pub async fn list_indexes(
+    data: GuardedData<ActionPolicy<{ actions::INDEXES_GET }>, MeiliSearch>,
+    paginate: web::Query<Pagination>,
+) -> Result<HttpResponse, ResponseError> {
+    let search_rules = &data.filters().search_rules;
+    let indexes: Vec<_> = data.list_indexes().await?;
+    let nb_indexes = indexes.len();
+    let iter = indexes
+        .into_iter()
+        .filter(|i| search_rules.is_index_authorized(&i.uid));
+    let ret = paginate
+        .into_inner()
+        .auto_paginate_unsized(nb_indexes, iter);
+
+    debug!("returns: {:?}", ret);
+    Ok(HttpResponse::Ok().json(ret))
 }
 
 #[derive(Debug, Deserialize)]
@@ -49,16 +65,30 @@ pub struct IndexCreateRequest {
 }
 
 pub async fn create_index(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::INDEXES_CREATE }>, MeiliSearch>,
     body: web::Json<IndexCreateRequest>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    let body = body.into_inner();
-    let meta = data.create_index(body.uid, body.primary_key).await?;
-    Ok(HttpResponse::Created().json(meta))
+    let IndexCreateRequest {
+        primary_key, uid, ..
+    } = body.into_inner();
+
+    analytics.publish(
+        "Index Created".to_string(),
+        json!({ "primary_key": primary_key }),
+        Some(&req),
+    );
+
+    let update = Update::CreateIndex { primary_key };
+    let task: SummarizedTaskView = meilisearch.register_update(uid, update).await?.into();
+
+    Ok(HttpResponse::Accepted().json(task))
 }
 
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
+#[allow(dead_code)]
 pub struct UpdateIndexRequest {
     uid: Option<String>,
     primary_key: Option<String>,
@@ -69,47 +99,67 @@ pub struct UpdateIndexRequest {
 pub struct UpdateIndexResponse {
     name: String,
     uid: String,
-    created_at: DateTime<Utc>,
-    updated_at: DateTime<Utc>,
-    primary_key: Option<String>,
+    #[serde(serialize_with = "time::serde::rfc3339::serialize")]
+    created_at: OffsetDateTime,
+    #[serde(serialize_with = "time::serde::rfc3339::serialize")]
+    updated_at: OffsetDateTime,
+    #[serde(serialize_with = "time::serde::rfc3339::serialize")]
+    primary_key: OffsetDateTime,
 }
 
 pub async fn get_index(
-    data: GuardedData<Private, Data>,
-    path: web::Path<IndexParam>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::INDEXES_GET }>, MeiliSearch>,
+    path: web::Path<String>,
 ) -> Result<HttpResponse, ResponseError> {
-    let meta = data.index(path.index_uid.clone()).await?;
+    let meta = meilisearch.get_index(path.into_inner()).await?;
     debug!("returns: {:?}", meta);
     Ok(HttpResponse::Ok().json(meta))
 }
 
 pub async fn update_index(
-    data: GuardedData<Private, Data>,
-    path: web::Path<IndexParam>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::INDEXES_UPDATE }>, MeiliSearch>,
+    path: web::Path<String>,
     body: web::Json<UpdateIndexRequest>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!("called with params: {:?}", body);
     let body = body.into_inner();
-    let meta = data
-        .update_index(path.into_inner().index_uid, body.primary_key, body.uid)
-        .await?;
-    debug!("returns: {:?}", meta);
-    Ok(HttpResponse::Ok().json(meta))
+    analytics.publish(
+        "Index Updated".to_string(),
+        json!({ "primary_key": body.primary_key}),
+        Some(&req),
+    );
+
+    let update = Update::UpdateIndex {
+        primary_key: body.primary_key,
+    };
+
+    let task: SummarizedTaskView = meilisearch
+        .register_update(path.into_inner(), update)
+        .await?
+        .into();
+
+    debug!("returns: {:?}", task);
+    Ok(HttpResponse::Accepted().json(task))
 }
 
 pub async fn delete_index(
-    data: GuardedData<Private, Data>,
-    path: web::Path<IndexParam>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::INDEXES_DELETE }>, MeiliSearch>,
+    path: web::Path<String>,
 ) -> Result<HttpResponse, ResponseError> {
-    data.delete_index(path.index_uid.clone()).await?;
-    Ok(HttpResponse::NoContent().finish())
+    let uid = path.into_inner();
+    let update = Update::DeleteIndex;
+    let task: SummarizedTaskView = meilisearch.register_update(uid, update).await?.into();
+
+    Ok(HttpResponse::Accepted().json(task))
 }
 
 pub async fn get_index_stats(
-    data: GuardedData<Private, Data>,
-    path: web::Path<IndexParam>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::STATS_GET }>, MeiliSearch>,
+    path: web::Path<String>,
 ) -> Result<HttpResponse, ResponseError> {
-    let response = data.get_index_stats(path.index_uid.clone()).await?;
+    let response = meilisearch.get_index_stats(path.into_inner()).await?;
 
     debug!("returns: {:?}", response);
     Ok(HttpResponse::Ok().json(response))

meilisearch-http/src/routes/indexes/search.rs

@@ -1,19 +1,25 @@
-use actix_web::{web, HttpResponse};
+use actix_web::{web, HttpRequest, HttpResponse};
 use log::debug;
+use meilisearch_auth::IndexSearchRules;
+use meilisearch_lib::index::{
+    SearchQuery, DEFAULT_CROP_LENGTH, DEFAULT_CROP_MARKER, DEFAULT_HIGHLIGHT_POST_TAG,
+    DEFAULT_HIGHLIGHT_PRE_TAG, DEFAULT_SEARCH_LIMIT,
+};
+use meilisearch_lib::MeiliSearch;
+use meilisearch_types::error::ResponseError;
 use serde::Deserialize;
+use serde_cs::vec::CS;
 use serde_json::Value;
 
-use crate::error::ResponseError;
+use crate::analytics::{Analytics, SearchAggregator};
 use crate::extractors::authentication::{policies::*, GuardedData};
-use crate::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT};
-use crate::routes::IndexParam;
-use crate::Data;
+use crate::extractors::sequential_extractor::SeqHandler;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(
         web::resource("")
-            .route(web::get().to(search_with_url_query))
-            .route(web::post().to(search_with_post)),
+            .route(web::get().to(SeqHandler(search_with_url_query)))
+            .route(web::post().to(SeqHandler(search_with_post))),
     );
 }
 
@@ -23,36 +29,26 @@ pub struct SearchQueryGet {
     q: Option<String>,
     offset: Option<usize>,
     limit: Option<usize>,
-    attributes_to_retrieve: Option<String>,
-    attributes_to_crop: Option<String>,
-    #[serde(default = "default_crop_length")]
+    attributes_to_retrieve: Option<CS<String>>,
+    attributes_to_crop: Option<CS<String>>,
+    #[serde(default = "DEFAULT_CROP_LENGTH")]
     crop_length: usize,
-    attributes_to_highlight: Option<String>,
+    attributes_to_highlight: Option<CS<String>>,
     filter: Option<String>,
     sort: Option<String>,
     #[serde(default = "Default::default")]
-    matches: bool,
-    facets_distribution: Option<String>,
+    show_matches_position: bool,
+    facets: Option<CS<String>>,
+    #[serde(default = "DEFAULT_HIGHLIGHT_PRE_TAG")]
+    highlight_pre_tag: String,
+    #[serde(default = "DEFAULT_HIGHLIGHT_POST_TAG")]
+    highlight_post_tag: String,
+    #[serde(default = "DEFAULT_CROP_MARKER")]
+    crop_marker: String,
 }
 
 impl From<SearchQueryGet> for SearchQuery {
     fn from(other: SearchQueryGet) -> Self {
-        let attributes_to_retrieve = other
-            .attributes_to_retrieve
-            .map(|attrs| attrs.split(',').map(String::from).collect());
-
-        let attributes_to_crop = other
-            .attributes_to_crop
-            .map(|attrs| attrs.split(',').map(String::from).collect());
-
-        let attributes_to_highlight = other
-            .attributes_to_highlight
-            .map(|attrs| attrs.split(',').map(String::from).collect());
-
-        let facets_distribution = other
-            .facets_distribution
-            .map(|attrs| attrs.split(',').map(String::from).collect());
-
         let filter = match other.filter {
             Some(f) => match serde_json::from_str(&f) {
                 Ok(v) => Some(v),
@@ -61,57 +57,177 @@ impl From<SearchQueryGet> for SearchQuery {
             None => None,
         };
 
-        let sort = other
-            .sort
-            .map(|attrs| attrs.split(',').map(String::from).collect());
-
         Self {
             q: other.q,
             offset: other.offset,
-            limit: other.limit.unwrap_or(DEFAULT_SEARCH_LIMIT),
-            attributes_to_retrieve,
-            attributes_to_crop,
+            limit: other.limit.unwrap_or_else(DEFAULT_SEARCH_LIMIT),
+            attributes_to_retrieve: other
+                .attributes_to_retrieve
+                .map(|o| o.into_iter().collect()),
+            attributes_to_crop: other.attributes_to_crop.map(|o| o.into_iter().collect()),
             crop_length: other.crop_length,
-            attributes_to_highlight,
+            attributes_to_highlight: other
+                .attributes_to_highlight
+                .map(|o| o.into_iter().collect()),
             filter,
-            sort,
-            matches: other.matches,
-            facets_distribution,
+            sort: other.sort.map(|attr| fix_sort_query_parameters(&attr)),
+            show_matches_position: other.show_matches_position,
+            facets: other.facets.map(|o| o.into_iter().collect()),
+            highlight_pre_tag: other.highlight_pre_tag,
+            highlight_post_tag: other.highlight_post_tag,
+            crop_marker: other.crop_marker,
         }
     }
 }
 
+/// Incorporate search rules in search query
+fn add_search_rules(query: &mut SearchQuery, rules: IndexSearchRules) {
+    query.filter = match (query.filter.take(), rules.filter) {
+        (None, rules_filter) => rules_filter,
+        (filter, None) => filter,
+        (Some(filter), Some(rules_filter)) => {
+            let filter = match filter {
+                Value::Array(filter) => filter,
+                filter => vec![filter],
+            };
+            let rules_filter = match rules_filter {
+                Value::Array(rules_filter) => rules_filter,
+                rules_filter => vec![rules_filter],
+            };
+
+            Some(Value::Array([filter, rules_filter].concat()))
+        }
+    }
+}
+
+// TODO: TAMO: split on :asc, and :desc, instead of doing some weird things
+
+/// Transform the sort query parameter into something that matches the post expected format.
+fn fix_sort_query_parameters(sort_query: &str) -> Vec<String> {
+    let mut sort_parameters = Vec::new();
+    let mut merge = false;
+    for current_sort in sort_query.trim_matches('"').split(',').map(|s| s.trim()) {
+        if current_sort.starts_with("_geoPoint(") {
+            sort_parameters.push(current_sort.to_string());
+            merge = true;
+        } else if merge && !sort_parameters.is_empty() {
+            let s = sort_parameters.last_mut().unwrap();
+            s.push(',');
+            s.push_str(current_sort);
+            if current_sort.ends_with("):desc") || current_sort.ends_with("):asc") {
+                merge = false;
+            }
+        } else {
+            sort_parameters.push(current_sort.to_string());
+            merge = false;
+        }
+    }
+    sort_parameters
+}
+
 pub async fn search_with_url_query(
-    data: GuardedData<Public, Data>,
-    path: web::Path<IndexParam>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::SEARCH }>, MeiliSearch>,
+    path: web::Path<String>,
     params: web::Query<SearchQueryGet>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!("called with params: {:?}", params);
-    let query = params.into_inner().into();
-    let search_result = data.search(path.into_inner().index_uid, query).await?;
+    let mut query: SearchQuery = params.into_inner().into();
 
-    // Tests that the nb_hits is always set to false
-    #[cfg(test)]
-    assert!(!search_result.exhaustive_nb_hits);
+    let index_uid = path.into_inner();
+    // Tenant token search_rules.
+    if let Some(search_rules) = meilisearch
+        .filters()
+        .search_rules
+        .get_index_search_rules(&index_uid)
+    {
+        add_search_rules(&mut query, search_rules);
+    }
+
+    let mut aggregate = SearchAggregator::from_query(&query, &req);
+
+    let search_result = meilisearch.search(index_uid, query).await;
+    if let Ok(ref search_result) = search_result {
+        aggregate.succeed(search_result);
+    }
+    analytics.get_search(aggregate);
+
+    let search_result = search_result?;
 
     debug!("returns: {:?}", search_result);
     Ok(HttpResponse::Ok().json(search_result))
 }
 
 pub async fn search_with_post(
-    data: GuardedData<Public, Data>,
-    path: web::Path<IndexParam>,
+    meilisearch: GuardedData<ActionPolicy<{ actions::SEARCH }>, MeiliSearch>,
+    path: web::Path<String>,
     params: web::Json<SearchQuery>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    debug!("search called with params: {:?}", params);
-    let search_result = data
-        .search(path.into_inner().index_uid, params.into_inner())
-        .await?;
+    let mut query = params.into_inner();
+    debug!("search called with params: {:?}", query);
 
-    // Tests that the nb_hits is always set to false
-    #[cfg(test)]
-    assert!(!search_result.exhaustive_nb_hits);
+    let index_uid = path.into_inner();
+    // Tenant token search_rules.
+    if let Some(search_rules) = meilisearch
+        .filters()
+        .search_rules
+        .get_index_search_rules(&index_uid)
+    {
+        add_search_rules(&mut query, search_rules);
+    }
+
+    let mut aggregate = SearchAggregator::from_query(&query, &req);
+
+    let search_result = meilisearch.search(index_uid, query).await;
+    if let Ok(ref search_result) = search_result {
+        aggregate.succeed(search_result);
+    }
+    analytics.post_search(aggregate);
+
+    let search_result = search_result?;
 
     debug!("returns: {:?}", search_result);
     Ok(HttpResponse::Ok().json(search_result))
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_fix_sort_query_parameters() {
+        let sort = fix_sort_query_parameters("_geoPoint(12, 13):asc");
+        assert_eq!(sort, vec!["_geoPoint(12,13):asc".to_string()]);
+        let sort = fix_sort_query_parameters("doggo:asc,_geoPoint(12.45,13.56):desc");
+        assert_eq!(
+            sort,
+            vec![
+                "doggo:asc".to_string(),
+                "_geoPoint(12.45,13.56):desc".to_string(),
+            ]
+        );
+        let sort = fix_sort_query_parameters(
+            "doggo:asc , _geoPoint(12.45, 13.56, 2590352):desc , catto:desc",
+        );
+        assert_eq!(
+            sort,
+            vec![
+                "doggo:asc".to_string(),
+                "_geoPoint(12.45,13.56,2590352):desc".to_string(),
+                "catto:desc".to_string(),
+            ]
+        );
+        let sort = fix_sort_query_parameters("doggo:asc , _geoPoint(1, 2), catto:desc");
+        // This is ugly but eh, I don't want to write a full parser just for this unused route
+        assert_eq!(
+            sort,
+            vec![
+                "doggo:asc".to_string(),
+                "_geoPoint(1,2),catto:desc".to_string(),
+            ]
+        );
+    }
+}