Merge #1904

1904: Update mini-dashboard version to v0.1.5 r=curquiza a=curquiza

Update the mini-dashboard with its latest version (v0.1.5)

Check with `@mdubus,` replaces https://github.com/meilisearch/MeiliSearch/pull/1903

Fixes #1898 

Co-authored-by: Clémentine Urquizar <clementine@meilisearch.com>

.github/workflows/coverage.yml

@@ -1,7 +1,6 @@
 ---
 on:
-  pull_request:
-    types: [review_requested, ready_for_review]
+  workflow_dispatch:
 
 name: Execute code coverage
 

.github/workflows/publish-binaries.yml

@@ -42,11 +42,13 @@ jobs:
     runs-on: ubuntu-18.04
     steps:
       - uses: actions/checkout@v2
-      - uses: uraimo/run-on-arch-action@v1.0.7
+      - uses: uraimo/run-on-arch-action@v2.1.1
         id: runcmd
         with:
-          architecture: aarch64 # aka ARMv8
-          distribution: ubuntu18.04
+          arch: aarch64 # aka ARMv8
+          distro: ubuntu18.04
+          env: |
+            JEMALLOC_SYS_WITH_LG_PAGE: 16
           run: |
             apt update
             apt install -y curl gcc make

.github/workflows/publish-crossbuild.yml

@@ -0,0 +1,76 @@
+name: Publish aarch64 binary
+
+on:
+  release:
+    types: [published]
+
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  publish-aarch64:
+    name: Publish to Github
+    runs-on: ${{ matrix.os }}
+    continue-on-error: false
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - build: aarch64
+            os: ubuntu-18.04
+            target: aarch64-unknown-linux-gnu
+            linker: gcc-aarch64-linux-gnu
+            use-cross: true
+            asset_name: meilisearch-linux-aarch64
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Installing Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          profile: minimal
+          target: ${{ matrix.target }}
+          override: true
+
+      - name: APT update
+        run: |
+          sudo apt update
+
+      - name: Install target specific tools
+        if: matrix.use-cross
+        run: |
+          sudo apt-get install -y ${{ matrix.linker }}
+
+      - name: Configure target aarch64 GNU
+        if: matrix.target == 'aarch64-unknown-linux-gnu'
+        ## Environment variable is not passed using env:
+        ## LD gold won't work with MUSL
+        # env:
+        #   JEMALLOC_SYS_WITH_LG_PAGE: 16
+        #   RUSTFLAGS: '-Clink-arg=-fuse-ld=gold'
+        run: |
+          echo '[target.aarch64-unknown-linux-gnu]' >> ~/.cargo/config
+          echo 'linker = "aarch64-linux-gnu-gcc"' >> ~/.cargo/config
+          echo 'JEMALLOC_SYS_WITH_LG_PAGE=16' >> $GITHUB_ENV
+          echo RUSTFLAGS="-Clink-arg=-fuse-ld=gold" >> $GITHUB_ENV
+
+      - name: Cargo build
+        uses: actions-rs/cargo@v1
+        with:
+          command: build
+          use-cross: ${{ matrix.use-cross }}
+          args: --release --target ${{ matrix.target }}
+
+      - name: List target output files
+        run: ls -lR ./target
+
+      - name: Upload the binary to release
+        uses: svenstaro/upload-release-action@v1-release
+        with:
+          repo_token: ${{ secrets.PUBLISH_TOKEN }}
+          file: target/${{ matrix.target }}/release/meilisearch
+          asset_name: ${{ matrix.asset_name }}
+          tag: ${{ github.ref }}

.github/workflows/rust.yml

@@ -23,12 +23,7 @@ jobs:
     steps:
     - uses: actions/checkout@v2
     - name: Cache dependencies
-      uses: actions/cache@v2
-      with:
-        path: |
-          ~/.cargo
-          ./target
-        key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
+      uses: Swatinem/rust-cache@v1.3.0
     - name: Run cargo check without any default features
       uses: actions-rs/cargo@v1
       with:
@@ -45,19 +40,14 @@ jobs:
     runs-on: ubuntu-18.04
     steps:
       - uses: actions/checkout@v2
-      - name: Cache dependencies
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo
-            ./target
-          key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
           toolchain: stable
           override: true
           components: clippy
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v1.3.0
       - name: Run cargo clippy
         uses: actions-rs/cargo@v1
         with:
@@ -69,18 +59,13 @@ jobs:
     runs-on: ubuntu-18.04
     steps:
       - uses: actions/checkout@v2
-      - name: Cache dependencies
-        uses: actions/cache@v2
-        with:
-          path: |
-            ~/.cargo
-            ./target
-          key: ${{ matrix.os }}-${{ hashFiles('Cargo.lock') }}
       - uses: actions-rs/toolchain@v1
         with:
           profile: minimal
           toolchain: nightly
           override: true
           components: rustfmt
+      - name: Cache dependencies
+        uses: Swatinem/rust-cache@v1.3.0
       - name: Run cargo fmt
         run: cargo fmt --all -- --check

CONTRIBUTING.md

@@ -1,112 +1,91 @@
 # Contributing
 
-First, thank you for contributing to MeiliSearch! The goal of this document is to
-provide everything you need to start contributing to MeiliSearch. The
-following TOC is sorted progressively, starting with the basics and
-expanding into more specifics.
+First, thank you for contributing to MeiliSearch! The goal of this document is to provide everything you need to start contributing to MeiliSearch.
 
-<!-- MarkdownTOC autolink="true" style="ordered" indent="   " -->
+- [Hacktoberfest](#hacktoberfest)
+- [Assumptions](#assumptions)
+- [How to Contribute](#how-to-contribute)
+- [Development Workflow](#development-workflow)
+- [Git Guidelines](#git-guidelines)
 
-1. [Assumptions](#assumptions)
-1. [Your First Contribution](#your-first-contribution)
-1. [Change Control](#change-control)
-   1. [Git Branches](#git-branches)
-   1. [Git Commits](#git-commits)
-      1. [Style](#style)
-   1. [Github Pull Requests](#github-pull-requests)
-      1. [Reviews & Approvals](#reviews--approvals)
-      1. [Merge Style](#merge-style)
-   1. [CI](#ci)
-1. [Development](#development)
-   1. [Setup](#setup)
-   1. [Testing](#testing)
-   1. [Benchmarking](#benchmarking--profiling)
-1. [Humans](#humans)
-   1. [Documentation](#documentation)
-   1. [Changelog](#changelog)
+## Hacktoberfest
 
-<!-- /MarkdownTOC -->
+It's [Hacktoberfest month](https://blog.meilisearch.com/contribute-hacktoberfest-2021/)! 🥳
+
+🚀 If your PR gets accepted it will count into your participation to Hacktoberfest!
+
+✅ To be accepted it has either to have been merged, approved or tagged with the `hacktoberfest-accepted` label.
+
+🧐 Don't forget to check the [quality standards](https://hacktoberfest.digitalocean.com/resources/qualitystandards)! Low-quality PRs might get marked as `spam` or `invalid`, and will not count toward your participation in Hacktoberfest.
 
 ## Assumptions
 
-1. **You're familiar with [Github](https://github.com) and the [pull request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)
-   workflow.**
-2. **You've read the MeiliSearch [docs](https://docs.meilisearch.com).**
+1. **You're familiar with [Github](https://github.com) and the [Pull Requests](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)(PR) workflow.**
+2. **You've read the MeiliSearch [documentation](https://docs.meilisearch.com).**
 3. **You know about the [MeiliSearch community](https://docs.meilisearch.com/learn/what_is_meilisearch/contact.html).
    Please use this for help.**
 
-## Your First Contribution
+## How to Contribute
 
 1. Ensure your change has an issue! Find an
    [existing issue](https://github.com/meilisearch/meilisearch/issues/) or [open a new issue](https://github.com/meilisearch/meilisearch/issues/new).
    * This is where you can get a feel if the change will be accepted or not.
-2. Once approved, [fork the MeiliSearch repository](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) in your own
-   Github account.
+2. Once approved, [fork the MeiliSearch repository](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) in your own Github account.
 3. [Create a new Git branch](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-and-deleting-branches-within-your-repository)
-4. Review the MeiliSearch [workflow](#workflow) and [development](#development).
-5. Make your changes.
-6. [Submit the branch as a pull request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) to the main MeiliSearch
-   repo. A MeiliSearch team member should comment and/or review your pull request
-   with a few days. Although, depending on the circumstances, it may take
-   longer.
+4. Review the [Development Workflow](#development-workflow) section that describes the steps to maintain the repository.
+5. Make your changes on your branch.
+6. [Submit the branch as a Pull Request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) pointing to the `main` branch of the MeiliSearch repository. A maintainer should comment and/or review your Pull Request within a few days. Although depending on the circumstances, it may take longer.
 
-## Change Control
+## Development Workflow
+
+### Setup and run MeiliSearch
+
+```bash
+cargo run --release
+```
+
+We recommend using the `--release` flag to test the full performance of MeiliSearch.
+
+### Test
+
+```bash
+cargo test
+```
+
+If you get a "Too many open files" error you might want to increase the open file limit using this command:
+
+```bash
+ulimit -Sn 3000
+```
+
+## Git Guidelines
 
 ### Git Branches
 
-_All_ changes must be made in a branch and submitted as [pull requests](#pull-requests).
-MeiliSearch does not adopt any type of branch naming style, but please use something
-descriptive of your changes.
+All changes must be made in a branch and submitted as PR.
+
+We do not enforce any branch naming style, but please use something descriptive of your changes.
 
 ### Git Commits
 
-#### Style
+As minimal requirements, your commit message should:
+- be capitalized
+- not finish by a dot or any other punctuation character (!,?)
+- start with a verb so that we can read your commit message this way: "This commit will ...", where "..." is the commit message.
+  e.g.: "Fix the home page button" or "Add more tests for create_index method"
 
-Please ensure your commits are small and focused; they should tell a story of
-your change. This helps reviewers to follow your changes, especially for more
-complex changes.
-
-Familiarise yourself with [How to Write a Git Commit Message](https://chris.beams.io/posts/git-commit/).
+We don't follow any other convention, but if you want to use one, we recommend [the Chris Beams one](https://chris.beams.io/posts/git-commit/).
 
 ### Github Pull Requests
 
-Once your changes are ready you must submit your branch as a pull request.
+Some notes on GitHub PRs:
 
-#### Reviews & Approvals
+- All PRs must be reviewed and approved by at least one maintainer.
+- The PR title should be accurate and descriptive of the changes.
+- [Convert your PR as a draft](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/changing-the-stage-of-a-pull-request) if your changes are a work in progress: no one will review it until you pass your PR as ready for review.<br>
+  The draft PRs are recommended when you want to show that you are working on something and make your work visible.
+- The branch related to the PR must be **up-to-date with `main`** before merging. Fortunately, this project uses [Bors](https://github.com/bors-ng/bors-ng) to automatically enforce this requirement without the PR author having to rebase manually.
 
-All pull requests must be reviewed and approved by at least one MeiliSearch team
-member.
+<hr>
 
-#### Merge Style
-
-All pull requests are squashed and merged. We generally discourage large pull
-requests that are over 300-500 lines of diff. If you would like to propose
-a change that is larger we suggest coming onto our chat channel and
-discuss it with one of our engineers. This way we can talk through the
-solution and discuss if a change that large is even needed! This overall
-will produce a quicker response to the change and likely produce code that
-aligns better with our process.
-
-## Development
-
-### Setup
-
-See the [MeiliSearch Docs](https://docs.meilisearch.com/reference/features/installation.html) for how to set up a development environment.
-
-### Benchmarking & Profiling
-
-We do not yet do any benchmarking, nor have we formalised our profiling. If you'd like to work on this please get in touch!
-
-## Humans
-
-After making your change, you'll want to prepare it for MeiliSearch users (mostly humans). This usually entails updating documentation and announcing your feature.
-
-### Documentation
-
-Documentation is very important to MeiliSearch. All contributions that
-alter user-facing behavior MUST include documentation changes. Please see
-[GitHub.com/meilisearch/documentation](https://github.com/meilisearch/documentation) for more info.
-
-### Changelog
-
-Until we have guidelines in place, updating the [`Changelog`](/CHANGELOG.md) is solely the responsibility of MeiliSearch team members.
+Thank you again for reading this through, we can not wait to begin to work with you if you made your way through this contributing guide ❤️

Cargo.toml

@@ -2,7 +2,9 @@
 members = [
     "meilisearch-http",
     "meilisearch-error",
+    "meilisearch-lib",
 ]
+resolver = "2"
 
-[profile.release]
-debug = true
+[patch.crates-io]
+pest = { git = "https://github.com/pest-parser/pest.git", rev = "51fd1d49f1041f7839975664ef71fe15c7dcaf67" }

Cross.toml

@@ -0,0 +1,7 @@
+[build.env]
+passthrough = [
+    "RUST_BACKTRACE",
+    "CARGO_TERM_COLOR",
+    "RUSTFLAGS",
+    "JEMALLOC_SYS_WITH_LG_PAGE"
+]

Dockerfile

@@ -1,9 +1,8 @@
 # Compile
 FROM    alpine:3.14 AS compiler
 
-RUN     apk update --quiet
-RUN     apk add curl
-RUN     apk add build-base
+RUN     apk update --quiet \
+        && apk add -q --no-cache curl build-base
 
 RUN     curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y
 
@@ -14,6 +13,7 @@ COPY    Cargo.toml .
 
 COPY    meilisearch-error/Cargo.toml meilisearch-error/
 COPY    meilisearch-http/Cargo.toml meilisearch-http/
+COPY    meilisearch-lib/Cargo.toml meilisearch-lib/
 
 ENV     RUSTFLAGS="-C target-feature=-crt-static"
 
@@ -34,11 +34,20 @@ RUN     $HOME/.cargo/bin/cargo build --release
 # Run
 FROM    alpine:3.14
 
-RUN     apk add -q --no-cache libgcc tini
+ARG     USER=meili
+ENV     HOME /home/${USER}
+ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
+ENV     MEILI_SERVER_PROVIDER docker
 
+# download runtime deps as root and create ${USER}
+RUN     apk update --quiet \
+        && apk add -q --no-cache libgcc tini curl \
+        && adduser -D ${USER}
+WORKDIR ${HOME}
+USER    ${USER}
+# copy file as ${USER} to ${HOME}
 COPY    --from=compiler /meilisearch/target/release/meilisearch .
 
-ENV     MEILI_HTTP_ADDR 0.0.0.0:7700
 EXPOSE  7700/tcp
 
 ENTRYPOINT ["tini", "--"]

README.md

@@ -61,6 +61,10 @@ meilisearch
 docker run -p 7700:7700 -v "$(pwd)/data.ms:/data.ms" getmeili/meilisearch
 ```
 
+#### Announcing a cloud-hosted MeiliSearch
+
+Join the closed beta by filling out this [form](https://meilisearch.typeform.com/to/FtnzvZfh).
+
 #### Try MeiliSearch in our Sandbox
 
 Create a MeiliSearch instance in [MeiliSearch Sandbox](https://sandbox.meilisearch.com/). This instance is free, and will be active for 48 hours.
@@ -183,7 +187,7 @@ Search and indexation are the domain of our core engine, [`milli`](https://githu
 MeiliSearch collects anonymous data regarding general usage.
 This helps us better understand developers' usage of MeiliSearch features.
 
-To see what information we're retrieving, please see the complete list [on the dedicated issue](https://github.com/meilisearch/MeiliSearch/issues/720).
+To find out more on what information we're retrieving, please see our documentation on [Telemetry](https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html).
 
 This program is optional, you can disable these analytics by using the `MEILI_NO_ANALYTICS` env variable.
 

SECURITY.md

@@ -0,0 +1,33 @@
+# Security
+
+MeiliSearch takes the security of our software products and services seriously.
+
+If you believe you have found a security vulnerability in any MeiliSearch-owned repository, please report it to us as described below.
+
+## Suported versions
+
+As long as we are pre-v1.0, only the latest version of MeiliSearch will be supported with security updates. 
+
+## Reporting security issues
+
+⚠️ Please do not report security vulnerabilities through public GitHub issues. ⚠️ 
+
+Instead, please kindly email us at security@meilisearch.com
+
+Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue:
+
+- Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.)
+- Full paths of source file(s) related to the manifestation of the issue
+- The location of the affected source code (tag/branch/commit or direct URL)
+- Any special configuration required to reproduce the issue
+- Step-by-step instructions to reproduce the issue
+- Proof-of-concept or exploit code (if possible)
+- Impact of the issue, including how an attacker might exploit the issue
+
+This information will help us triage your report more quickly.
+
+You will receive a response from us within 72 hours. If the issue is confirmed, we will release a patch as soon as possible depending on complexity.
+
+## Preferred languages
+
+We prefer all communications to be in English.

bors.toml

@@ -5,5 +5,6 @@ status = [
     'Run Clippy',
     'Run Rustfmt'
 ]
+pr_status = ['Milestone Check']
 # 3 hours timeout
 timeout-sec = 10800

datasets/movies/README.md

@@ -1 +0,0 @@
-_datas in movies.json are from https://www.themoviedb.org/_

download-latest.sh

@@ -51,13 +51,13 @@ semverLT() {
     if [ $MAJOR_A -le $MAJOR_B ] && [ $MINOR_A -le $MINOR_B ] && [ $PATCH_A -lt $PATCH_B ]; then
         return 0
     fi
-    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+    if [ "_$SPECIAL_A"  == '_' ] && [ "_$SPECIAL_B"  == '_' ] ; then
         return 1
     fi
-    if [ "_$SPECIAL_A"  == "_" ] && [ "_$SPECIAL_B"  != "_" ] ; then
+    if [ "_$SPECIAL_A"  == '_' ] && [ "_$SPECIAL_B"  != '_' ] ; then
         return 1
     fi
-    if [ "_$SPECIAL_A"  != "_" ] && [ "_$SPECIAL_B"  == "_" ] ; then
+    if [ "_$SPECIAL_A"  != '_' ] && [ "_$SPECIAL_B"  == '_' ] ; then
         return 0
     fi
     if [ "_$SPECIAL_A" < "_$SPECIAL_B" ]; then
@@ -67,10 +67,18 @@ semverLT() {
     return 1
 }
 
+# Get a token from https://github.com/settings/tokens to increasae rate limit (from 60 to 5000), make sure the token scope is set to 'public_repo'
+# Create GITHUB_PAT enviroment variable once you aquired the token to start using it
 # Returns the tag of the latest stable release (in terms of semver and not of release date)
 get_latest() {
     temp_file='temp_file' # temp_file needed because the grep would start before the download is over
-    curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file" || return 1
+
+        if [ -z "$GITHUB_PAT" ]; then
+        curl -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file" || return 1
+    else
+        curl -H "Authorization: token $GITHUB_PAT" -s 'https://api.github.com/repos/meilisearch/MeiliSearch/releases' > "$temp_file" || return 1
+    fi
+
     releases=$(cat "$temp_file" | \
         grep -E "tag_name|draft|prerelease" \
         | tr -d ',"' | cut -d ':' -f2 | tr -d ' ')
@@ -78,28 +86,28 @@ get_latest() {
         # Ex: v0.10.1 false false v0.9.1-rc.1 false true v0.9.0 false false...
 
     i=0
-    latest=""
-    current_tag=""
+    latest=''
+    current_tag=''
     for release_info in $releases; do
         if [ $i -eq 0 ]; then # Cheking tag_name
             if echo "$release_info" | grep -q "$GREP_SEMVER_REGEXP"; then # If it's not an alpha or beta release
                 current_tag=$release_info
             else
-                current_tag=""
+                current_tag=''
             fi
             i=1
         elif [ $i -eq 1 ]; then # Checking draft boolean
-            if [ "$release_info" = "true" ]; then
-                current_tag=""
+            if [ "$release_info" = 'true' ]; then
+                current_tag=''
             fi
             i=2
         elif [ $i -eq 2 ]; then # Checking prerelease boolean
-            if [ "$release_info" = "true" ]; then
-                current_tag=""
+            if [ "$release_info" = 'true' ]; then
+                current_tag=''
             fi
             i=0
-            if [ "$current_tag" != "" ]; then # If the current_tag is valid
-                if [ "$latest" = "" ]; then # If there is no latest yet
+            if [ "$current_tag" != '' ]; then # If the current_tag is valid
+                if [ "$latest" = '' ]; then # If there is no latest yet
                     latest="$current_tag"
                 else
                     semverLT $current_tag $latest # Comparing latest and the current tag
@@ -140,7 +148,7 @@ get_os() {
 get_archi() {
     architecture=$(uname -m)
     case "$architecture" in
-    'x86_64' | 'amd64')
+    'x86_64' | 'amd64' | 'arm64')
         archi='amd64'
         ;;
     'aarch64')
@@ -153,7 +161,7 @@ get_archi() {
 }
 
 success_usage() {
-    printf "$GREEN%s\n$DEFAULT" "MeiliSearch binary successfully downloaded as '$BINARY_NAME' file."
+    printf "$GREEN%s\n$DEFAULT" "MeiliSearch $latest binary successfully downloaded as '$binary_name' file."
     echo ''
     echo 'Run it:'
     echo '    $ ./meilisearch'
@@ -171,6 +179,13 @@ failure_usage() {
 # MAIN
 latest="$(get_latest)"
 
+if [ "$latest" = '' ]; then
+    echo ''
+    echo 'Impossible to get the latest stable version of MeiliSearch.'
+    echo 'Please let us know about this issue: https://github.com/meilisearch/meilisearch-swift/issues/new/choose'
+    exit 1
+fi
+
 if ! get_os; then
     failure_usage
     exit 1
@@ -185,16 +200,16 @@ echo "Downloading MeiliSearch binary $latest for $os, architecture $archi..."
 case "$os" in
     'windows')
         release_file="meilisearch-$os-$archi.exe"
-		BINARY_NAME='meilisearch.exe'
+		binary_name='meilisearch.exe'
 
         ;;
-	*) 
+	*)
 		release_file="meilisearch-$os-$archi"
-		BINARY_NAME='meilisearch'
+		binary_name='meilisearch'
 
 esac
 link="https://github.com/meilisearch/MeiliSearch/releases/download/$latest/$release_file"
 curl -OL "$link"
-mv "$release_file" "$BINARY_NAME"
-chmod 744 "$BINARY_NAME"
+mv "$release_file" "$binary_name"
+chmod 744 "$binary_name"
 success_usage

meilisearch-error/Cargo.toml

@@ -1,8 +1,9 @@
 [package]
 name = "meilisearch-error"
-version = "0.22.0"
+version = "0.24.0"
 authors = ["marin <postma.marin@protonmail.com>"]
 edition = "2018"
 
 [dependencies]
-actix-http = "=3.0.0-beta.6"
+actix-http = "=3.0.0-beta.10"
+serde = { version = "1.0.130", features = ["derive"] }

meilisearch-error/src/lib.rs

@@ -1,6 +1,7 @@
 use std::fmt;
 
 use actix_http::http::StatusCode;
+use serde::{Deserialize, Serialize};
 
 pub trait ErrorCode: std::error::Error {
     fn error_code(&self) -> Code;
@@ -38,20 +39,20 @@ impl fmt::Display for ErrorType {
         use ErrorType::*;
 
         match self {
-            InternalError => write!(f, "internal_error"),
-            InvalidRequestError => write!(f, "invalid_request_error"),
-            AuthenticationError => write!(f, "authentication_error"),
+            InternalError => write!(f, "internal"),
+            InvalidRequestError => write!(f, "invalid_request"),
+            AuthenticationError => write!(f, "auth"),
         }
     }
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone, Copy)]
 pub enum Code {
     // index related error
     CreateIndex,
     IndexAlreadyExists,
     IndexNotFound,
     InvalidIndexUid,
-    OpenIndex,
 
     // invalid state error
     InvalidState,
@@ -60,18 +61,24 @@ pub enum Code {
 
     MaxFieldsLimitExceeded,
     MissingDocumentId,
+    InvalidDocumentId,
 
-    Facet,
     Filter,
     Sort,
 
     BadParameter,
     BadRequest,
+    DatabaseSizeLimitReached,
     DocumentNotFound,
     Internal,
+    InvalidGeoField,
+    InvalidRankingRule,
+    InvalidStore,
     InvalidToken,
     MissingAuthorizationHeader,
-    NotFound,
+    NoSpaceLeftOnDevice,
+    DumpNotFound,
+    TaskNotFound,
     PayloadTooLarge,
     RetrieveDocument,
     SearchDocuments,
@@ -79,6 +86,11 @@ pub enum Code {
 
     DumpAlreadyInProgress,
     DumpProcessFailed,
+
+    InvalidContentType,
+    MissingContentType,
+    MalformedPayload,
+    MissingPayload,
 }
 
 impl Code {
@@ -89,22 +101,30 @@ impl Code {
         match self {
             // index related errors
             // create index is thrown on internal error while creating an index.
-            CreateIndex => ErrCode::internal("index_creation_failed", StatusCode::BAD_REQUEST),
-            IndexAlreadyExists => ErrCode::invalid("index_already_exists", StatusCode::BAD_REQUEST),
+            CreateIndex => {
+                ErrCode::internal("index_creation_failed", StatusCode::INTERNAL_SERVER_ERROR)
+            }
+            IndexAlreadyExists => ErrCode::invalid("index_already_exists", StatusCode::CONFLICT),
             // thrown when requesting an unexisting index
             IndexNotFound => ErrCode::invalid("index_not_found", StatusCode::NOT_FOUND),
             InvalidIndexUid => ErrCode::invalid("invalid_index_uid", StatusCode::BAD_REQUEST),
-            OpenIndex => {
-                ErrCode::internal("index_not_accessible", StatusCode::INTERNAL_SERVER_ERROR)
-            }
 
             // invalid state error
             InvalidState => ErrCode::internal("invalid_state", StatusCode::INTERNAL_SERVER_ERROR),
             // thrown when no primary key has been set
-            MissingPrimaryKey => ErrCode::invalid("missing_primary_key", StatusCode::BAD_REQUEST),
+            MissingPrimaryKey => {
+                ErrCode::invalid("primary_key_inference_failed", StatusCode::BAD_REQUEST)
+            }
             // error thrown when trying to set an already existing primary key
             PrimaryKeyAlreadyPresent => {
-                ErrCode::invalid("primary_key_already_present", StatusCode::BAD_REQUEST)
+                ErrCode::invalid("index_primary_key_already_exists", StatusCode::BAD_REQUEST)
+            }
+            // invalid ranking rule
+            InvalidRankingRule => ErrCode::invalid("invalid_ranking_rule", StatusCode::BAD_REQUEST),
+
+            // invalid database
+            InvalidStore => {
+                ErrCode::internal("invalid_store_file", StatusCode::INTERNAL_SERVER_ERROR)
             }
 
             // invalid document
@@ -112,9 +132,8 @@ impl Code {
                 ErrCode::invalid("max_fields_limit_exceeded", StatusCode::BAD_REQUEST)
             }
             MissingDocumentId => ErrCode::invalid("missing_document_id", StatusCode::BAD_REQUEST),
+            InvalidDocumentId => ErrCode::invalid("invalid_document_id", StatusCode::BAD_REQUEST),
 
-            // error related to facets
-            Facet => ErrCode::invalid("invalid_facet", StatusCode::BAD_REQUEST),
             // error related to filters
             Filter => ErrCode::invalid("invalid_filter", StatusCode::BAD_REQUEST),
             // error related to sorts
@@ -122,13 +141,22 @@ impl Code {
 
             BadParameter => ErrCode::invalid("bad_parameter", StatusCode::BAD_REQUEST),
             BadRequest => ErrCode::invalid("bad_request", StatusCode::BAD_REQUEST),
+            DatabaseSizeLimitReached => ErrCode::internal(
+                "database_size_limit_reached",
+                StatusCode::INTERNAL_SERVER_ERROR,
+            ),
             DocumentNotFound => ErrCode::invalid("document_not_found", StatusCode::NOT_FOUND),
             Internal => ErrCode::internal("internal", StatusCode::INTERNAL_SERVER_ERROR),
-            InvalidToken => ErrCode::authentication("invalid_token", StatusCode::FORBIDDEN),
+            InvalidGeoField => ErrCode::invalid("invalid_geo_field", StatusCode::BAD_REQUEST),
+            InvalidToken => ErrCode::authentication("invalid_api_key", StatusCode::FORBIDDEN),
             MissingAuthorizationHeader => {
                 ErrCode::authentication("missing_authorization_header", StatusCode::UNAUTHORIZED)
             }
-            NotFound => ErrCode::invalid("not_found", StatusCode::NOT_FOUND),
+            TaskNotFound => ErrCode::invalid("task_not_found", StatusCode::NOT_FOUND),
+            DumpNotFound => ErrCode::invalid("dump_not_found", StatusCode::NOT_FOUND),
+            NoSpaceLeftOnDevice => {
+                ErrCode::internal("no_space_left_on_device", StatusCode::INTERNAL_SERVER_ERROR)
+            }
             PayloadTooLarge => ErrCode::invalid("payload_too_large", StatusCode::PAYLOAD_TOO_LARGE),
             RetrieveDocument => {
                 ErrCode::internal("unretrievable_document", StatusCode::BAD_REQUEST)
@@ -140,11 +168,19 @@ impl Code {
 
             // error related to dump
             DumpAlreadyInProgress => {
-                ErrCode::invalid("dump_already_in_progress", StatusCode::CONFLICT)
+                ErrCode::invalid("dump_already_processing", StatusCode::CONFLICT)
             }
             DumpProcessFailed => {
                 ErrCode::internal("dump_process_failed", StatusCode::INTERNAL_SERVER_ERROR)
             }
+            MissingContentType => {
+                ErrCode::invalid("missing_content_type", StatusCode::UNSUPPORTED_MEDIA_TYPE)
+            }
+            MalformedPayload => ErrCode::invalid("malformed_payload", StatusCode::BAD_REQUEST),
+            InvalidContentType => {
+                ErrCode::invalid("invalid_content_type", StatusCode::UNSUPPORTED_MEDIA_TYPE)
+            }
+            MissingPayload => ErrCode::invalid("missing_payload", StatusCode::BAD_REQUEST),
         }
     }
 

meilisearch-http/Cargo.toml

@@ -4,88 +4,81 @@ description = "MeiliSearch HTTP server"
 edition = "2018"
 license = "MIT"
 name = "meilisearch-http"
-version = "0.22.0"
+version = "0.24.0"
 
 [[bin]]
 name = "meilisearch"
 path = "src/main.rs"
 
 [build-dependencies]
-actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true }
-anyhow = { version = "*", optional = true }
-cargo_toml = { version = "0.9.0", optional = true }
+actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "39d8006", optional = true }
+anyhow = { version = "1.0.43", optional = true }
+cargo_toml = { version = "0.9", optional = true }
 hex = { version = "0.4.3", optional = true }
-reqwest = { version = "0.11.3", features = ["blocking", "rustls-tls"], default-features = false, optional = true }
-sha-1 = { version = "0.9.4", optional = true }
-tempfile = { version = "3.1.0", optional = true }
+reqwest = { version = "0.11.4", features = ["blocking", "rustls-tls"], default-features = false, optional = true }
+sha-1 = { version = "0.9.8", optional = true }
+tempfile = { version = "3.2.0", optional = true }
 vergen = { version = "5.1.15", default-features = false, features = ["git"] }
-zip = { version = "0.5.12", optional = true }
+zip = { version = "0.5.13", optional = true }
 
 [dependencies]
-actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "2dac1a4"}
-actix-http = { version = "=3.0.0-beta.6" }
-actix-service = "2.0.0"
-actix-web = { version = "=4.0.0-beta.6", features = ["rustls"] }
-actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "6db8c3e", optional = true }
-anyhow = "1.0.36"
-async-stream = "0.3.0"
-async-trait = "0.1.42"
-arc-swap = "1.2.0"
-byte-unit = { version = "4.0.9", default-features = false, features = ["std"] }
-bytes = "0.6.0"
+actix-cors = { git = "https://github.com/MarinPostma/actix-extras.git", rev = "963ac94d" }
+actix-web = { version = "4.0.0-beta.9", features = ["rustls"] }
+actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "39d8006", optional = true }
+anyhow = { version = "1.0.43", features = ["backtrace"] }
+async-stream = "0.3.2"
+async-trait = "0.1.51"
+arc-swap = "1.3.2"
+byte-unit = { version = "4.0.12", default-features = false, features = ["std"] }
+bytes = "1.1.0"
 chrono = { version = "0.4.19", features = ["serde"] }
-crossbeam-channel = "0.5.0"
+crossbeam-channel = "0.5.1"
 either = "1.6.1"
-env_logger = "0.8.2"
-flate2 = "1.0.19"
-fst = "0.4.5"
-futures = "0.3.7"
-futures-util = "0.3.8"
+env_logger = "0.9.0"
+flate2 = "1.0.21"
+fst = "0.4.7"
+futures = "0.3.17"
+futures-util = "0.3.17"
 heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
-http = "0.2.1"
-indexmap = { version = "1.3.2", features = ["serde-1"] }
-itertools = "0.10.0"
-log = "0.4.8"
-main_error = "0.1.0"
+http = "0.2.4"
+indexmap = { version = "1.7.0", features = ["serde-1"] }
+itertools = "0.10.1"
+log = "0.4.14"
+meilisearch-lib = { path = "../meilisearch-lib" }
 meilisearch-error = { path = "../meilisearch-error" }
 meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" }
-memmap = "0.7.0"
-milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.13.1" }
 mime = "0.3.16"
 num_cpus = "1.13.0"
-once_cell = "1.5.2"
-parking_lot = "0.11.1"
-rand = "0.7.3"
-rayon = "1.5.0"
-regex = "1.4.2"
-rustls = "0.19"
-serde = { version = "1.0", features = ["derive"] }
-serde_json = { version = "1.0.59", features = ["preserve_order"] }
-sha2 = "0.9.1"
-siphasher = "0.3.2"
+once_cell = "1.8.0"
+parking_lot = "0.11.2"
+platform-dirs = "0.3.0"
+rand = "0.8.4"
+rayon = "1.5.1"
+regex = "1.5.4"
+rustls = "0.19.1"
+segment = { version = "0.1.2", optional = true }
+serde = { version = "1.0.130", features = ["derive"] }
+serde_json = { version = "1.0.67", features = ["preserve_order"] }
+sha2 = "0.9.6"
+siphasher = "0.3.7"
 slice-group-by = "0.2.6"
-structopt = "0.3.20"
-tar = "0.4.29"
-tempfile = "3.1.0"
-thiserror = "1.0.24"
-tokio = { version = "1", features = ["full"] }
-uuid = { version =  "0.8.2", features = ["serde"] }
+structopt = "0.3.23"
+tar = "0.4.37"
+tempfile = "3.2.0"
+thiserror = "1.0.28"
+tokio = { version = "1.11.0", features = ["full"] }
+uuid = { version = "0.8.2", features = ["serde"] }
 walkdir = "2.3.2"
 obkv = "0.2.0"
-pin-project = "1.0.7"
-whoami = { version = "1.1.2", optional = true }
-reqwest = { version = "0.11.3", features = ["json", "rustls-tls"], default-features = false, optional = true }
-serdeval = "0.1.0"
-sysinfo = "0.20.0"
+pin-project = "1.0.8"
+sysinfo = "0.20.2"
+tokio-stream = "0.1.7"
 
 [dev-dependencies]
-actix-rt = "2.1.0"
-assert-json-diff = { branch = "master", git = "https://github.com/qdequele/assert-json-diff" }
-mockall = "0.9.1"
+actix-rt = "2.2.0"
 paste = "1.0.5"
 serde_url_params = "0.2.1"
-tempdir = "0.3.7"
-urlencoding = "1.1.1"
+urlencoding = "2.1.0"
 
 [features]
 mini-dashboard = [
@@ -98,12 +91,12 @@ mini-dashboard = [
     "tempfile",
     "zip",
 ]
-analytics = ["whoami", "reqwest"]
+analytics = ["segment"]
 default = ["analytics", "mini-dashboard"]
 
 [target.'cfg(target_os = "linux")'.dependencies]
-jemallocator = "0.3.2"
+tikv-jemallocator = "0.4.1"
 
 [package.metadata.mini-dashboard]
-assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.4/build.zip"
-sha1 = "750e8a8e56cfa61fbf9ead14b08a5f17ad3f3d37"
+assets-url = "https://github.com/meilisearch/mini-dashboard/releases/download/v0.1.5/build.zip"
+sha1 = "1d955ea91b7691bd6fc207cb39866b82210783f0"

meilisearch-http/src/analytics.rs

@@ -1,126 +0,0 @@
-use std::hash::{Hash, Hasher};
-use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
-
-use log::debug;
-use serde::Serialize;
-use siphasher::sip::SipHasher;
-
-use crate::Data;
-use crate::Opt;
-
-const AMPLITUDE_API_KEY: &str = "f7fba398780e06d8fe6666a9be7e3d47";
-
-#[derive(Debug, Serialize)]
-struct EventProperties {
-    database_size: u64,
-    last_update_timestamp: Option<i64>, //timestamp
-    number_of_documents: Vec<u64>,
-}
-
-impl EventProperties {
-    async fn from(data: Data) -> anyhow::Result<EventProperties> {
-        let stats = data.index_controller.get_all_stats().await?;
-
-        let database_size = stats.database_size;
-        let last_update_timestamp = stats.last_update.map(|u| u.timestamp());
-        let number_of_documents = stats
-            .indexes
-            .values()
-            .map(|index| index.number_of_documents)
-            .collect();
-
-        Ok(EventProperties {
-            database_size,
-            last_update_timestamp,
-            number_of_documents,
-        })
-    }
-}
-
-#[derive(Debug, Serialize)]
-struct UserProperties<'a> {
-    env: &'a str,
-    start_since_days: u64,
-    user_email: Option<String>,
-    server_provider: Option<String>,
-}
-
-#[derive(Debug, Serialize)]
-struct Event<'a> {
-    user_id: &'a str,
-    event_type: &'a str,
-    device_id: &'a str,
-    time: u64,
-    app_version: &'a str,
-    user_properties: UserProperties<'a>,
-    event_properties: Option<EventProperties>,
-}
-
-#[derive(Debug, Serialize)]
-struct AmplitudeRequest<'a> {
-    api_key: &'a str,
-    events: Vec<Event<'a>>,
-}
-
-pub async fn analytics_sender(data: Data, opt: Opt) {
-    let username = whoami::username();
-    let hostname = whoami::hostname();
-    let platform = whoami::platform();
-
-    let uid = username + &hostname + &platform.to_string();
-
-    let mut hasher = SipHasher::new();
-    uid.hash(&mut hasher);
-    let hash = hasher.finish();
-
-    let uid = format!("{:X}", hash);
-    let platform = platform.to_string();
-    let first_start = Instant::now();
-
-    loop {
-        let n = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
-        let user_id = &uid;
-        let device_id = &platform;
-        let time = n.as_secs();
-        let event_type = "runtime_tick";
-        let elapsed_since_start = first_start.elapsed().as_secs() / 86_400; // One day
-        let event_properties = EventProperties::from(data.clone()).await.ok();
-        let app_version = env!("CARGO_PKG_VERSION").to_string();
-        let app_version = app_version.as_str();
-        let user_email = std::env::var("MEILI_USER_EMAIL").ok();
-        let server_provider = std::env::var("MEILI_SERVER_PROVIDER").ok();
-        let user_properties = UserProperties {
-            env: &opt.env,
-            start_since_days: elapsed_since_start,
-            user_email,
-            server_provider,
-        };
-
-        let event = Event {
-            user_id,
-            event_type,
-            device_id,
-            time,
-            app_version,
-            user_properties,
-            event_properties,
-        };
-
-        let request = AmplitudeRequest {
-            api_key: AMPLITUDE_API_KEY,
-            events: vec![event],
-        };
-
-        let response = reqwest::Client::new()
-            .post("https://api2.amplitude.com/2/httpapi")
-            .timeout(Duration::from_secs(60)) // 1 minute max
-            .json(&request)
-            .send()
-            .await;
-        if let Err(e) = response {
-            debug!("Unsuccessful call to Amplitude: {}", e);
-        }
-
-        tokio::time::sleep(Duration::from_secs(3600)).await;
-    }
-}

meilisearch-http/src/analytics/mock_analytics.rs

@@ -0,0 +1,51 @@
+use std::{any::Any, sync::Arc};
+
+use actix_web::HttpRequest;
+use serde_json::Value;
+
+use crate::{routes::indexes::documents::UpdateDocumentsQuery, Opt};
+
+use super::{find_user_id, Analytics};
+
+pub struct MockAnalytics;
+
+#[derive(Default)]
+pub struct SearchAggregator {}
+
+#[allow(dead_code)]
+impl SearchAggregator {
+    pub fn from_query(_: &dyn Any, _: &dyn Any) -> Self {
+        Self::default()
+    }
+
+    pub fn succeed(&mut self, _: &dyn Any) {}
+}
+
+impl MockAnalytics {
+    #[allow(clippy::new_ret_no_self)]
+    pub fn new(opt: &Opt) -> (Arc<dyn Analytics>, String) {
+        let user = find_user_id(&opt.db_path).unwrap_or_default();
+        (Arc::new(Self), user)
+    }
+}
+
+impl Analytics for MockAnalytics {
+    // These methods are noop and should be optimized out
+    fn publish(&self, _event_name: String, _send: Value, _request: Option<&HttpRequest>) {}
+    fn get_search(&self, _aggregate: super::SearchAggregator) {}
+    fn post_search(&self, _aggregate: super::SearchAggregator) {}
+    fn add_documents(
+        &self,
+        _documents_query: &UpdateDocumentsQuery,
+        _index_creation: bool,
+        _request: &HttpRequest,
+    ) {
+    }
+    fn update_documents(
+        &self,
+        _documents_query: &UpdateDocumentsQuery,
+        _index_creation: bool,
+        _request: &HttpRequest,
+    ) {
+    }
+}

meilisearch-http/src/analytics/mod.rs

@@ -0,0 +1,84 @@
+mod mock_analytics;
+// if we are in release mode and the feature analytics was enabled
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+mod segment_analytics;
+
+use std::fs;
+use std::path::{Path, PathBuf};
+
+use actix_web::HttpRequest;
+use once_cell::sync::Lazy;
+use platform_dirs::AppDirs;
+use serde_json::Value;
+
+use crate::routes::indexes::documents::UpdateDocumentsQuery;
+
+pub use mock_analytics::MockAnalytics;
+
+// if we are in debug mode OR the analytics feature is disabled
+// the `SegmentAnalytics` point to the mock instead of the real analytics
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
+pub type SegmentAnalytics = mock_analytics::MockAnalytics;
+#[cfg(any(debug_assertions, not(feature = "analytics")))]
+pub type SearchAggregator = mock_analytics::SearchAggregator;
+
+// if we are in release mode and the feature analytics was enabled
+// we use the real analytics
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+pub type SegmentAnalytics = segment_analytics::SegmentAnalytics;
+#[cfg(all(not(debug_assertions), feature = "analytics"))]
+pub type SearchAggregator = segment_analytics::SearchAggregator;
+
+/// The MeiliSearch config dir:
+/// `~/.config/MeiliSearch` on *NIX or *BSD.
+/// `~/Library/ApplicationSupport` on macOS.
+/// `%APPDATA` (= `C:\Users%USERNAME%\AppData\Roaming`) on windows.
+static MEILISEARCH_CONFIG_PATH: Lazy<Option<PathBuf>> =
+    Lazy::new(|| AppDirs::new(Some("MeiliSearch"), false).map(|appdir| appdir.config_dir));
+
+fn config_user_id_path(db_path: &Path) -> Option<PathBuf> {
+    db_path
+        .canonicalize()
+        .ok()
+        .map(|path| {
+            path.join("instance-uid")
+                .display()
+                .to_string()
+                .replace("/", "-")
+        })
+        .zip(MEILISEARCH_CONFIG_PATH.as_ref())
+        .map(|(filename, config_path)| config_path.join(filename.trim_start_matches('-')))
+}
+
+/// Look for the instance-uid in the `data.ms` or in `~/.config/MeiliSearch/path-to-db-instance-uid`
+fn find_user_id(db_path: &Path) -> Option<String> {
+    fs::read_to_string(db_path.join("instance-uid"))
+        .ok()
+        .or_else(|| fs::read_to_string(&config_user_id_path(db_path)?).ok())
+}
+
+pub trait Analytics: Sync + Send {
+    /// The method used to publish most analytics that do not need to be batched every hours
+    fn publish(&self, event_name: String, send: Value, request: Option<&HttpRequest>);
+
+    /// This method should be called to aggergate a get search
+    fn get_search(&self, aggregate: SearchAggregator);
+
+    /// This method should be called to aggregate a post search
+    fn post_search(&self, aggregate: SearchAggregator);
+
+    // this method should be called to aggregate a add documents request
+    fn add_documents(
+        &self,
+        documents_query: &UpdateDocumentsQuery,
+        index_creation: bool,
+        request: &HttpRequest,
+    );
+    // this method should be called to batch a update documents request
+    fn update_documents(
+        &self,
+        documents_query: &UpdateDocumentsQuery,
+        index_creation: bool,
+        request: &HttpRequest,
+    );
+}

meilisearch-http/src/analytics/segment_analytics.rs

@@ -0,0 +1,521 @@
+use std::collections::{BinaryHeap, HashMap, HashSet};
+use std::fs;
+use std::path::Path;
+use std::sync::Arc;
+use std::time::{Duration, Instant};
+
+use actix_web::http::header::USER_AGENT;
+use actix_web::HttpRequest;
+use http::header::CONTENT_TYPE;
+use meilisearch_lib::index::{SearchQuery, SearchResult};
+use meilisearch_lib::index_controller::Stats;
+use meilisearch_lib::MeiliSearch;
+use once_cell::sync::Lazy;
+use regex::Regex;
+use segment::message::{Identify, Track, User};
+use segment::{AutoBatcher, Batcher, HttpClient};
+use serde_json::{json, Value};
+use sysinfo::{DiskExt, System, SystemExt};
+use tokio::select;
+use tokio::sync::mpsc::{self, Receiver, Sender};
+use uuid::Uuid;
+
+use crate::analytics::Analytics;
+use crate::routes::indexes::documents::UpdateDocumentsQuery;
+use crate::Opt;
+
+use super::{config_user_id_path, MEILISEARCH_CONFIG_PATH};
+
+/// Write the instance-uid in the `data.ms` and in `~/.config/MeiliSearch/path-to-db-instance-uid`. Ignore the errors.
+fn write_user_id(db_path: &Path, user_id: &str) {
+    let _ = fs::write(db_path.join("instance-uid"), user_id.as_bytes());
+    if let Some((meilisearch_config_path, user_id_path)) = MEILISEARCH_CONFIG_PATH
+        .as_ref()
+        .zip(config_user_id_path(db_path))
+    {
+        let _ = fs::create_dir_all(&meilisearch_config_path);
+        let _ = fs::write(user_id_path, user_id.as_bytes());
+    }
+}
+
+const SEGMENT_API_KEY: &str = "P3FWhhEsJiEDCuEHpmcN9DHcK4hVfBvb";
+
+pub fn extract_user_agents(request: &HttpRequest) -> Vec<String> {
+    request
+        .headers()
+        .get(USER_AGENT)
+        .map(|header| header.to_str().ok())
+        .flatten()
+        .unwrap_or("unknown")
+        .split(';')
+        .map(str::trim)
+        .map(ToString::to_string)
+        .collect()
+}
+
+pub enum AnalyticsMsg {
+    BatchMessage(Track),
+    AggregateGetSearch(SearchAggregator),
+    AggregatePostSearch(SearchAggregator),
+    AggregateAddDocuments(DocumentsAggregator),
+    AggregateUpdateDocuments(DocumentsAggregator),
+}
+
+pub struct SegmentAnalytics {
+    sender: Sender<AnalyticsMsg>,
+    user: User,
+}
+
+impl SegmentAnalytics {
+    pub async fn new(opt: &Opt, meilisearch: &MeiliSearch) -> (Arc<dyn Analytics>, String) {
+        let user_id = super::find_user_id(&opt.db_path);
+        let first_time_run = user_id.is_none();
+        let user_id = user_id.unwrap_or_else(|| Uuid::new_v4().to_string());
+        write_user_id(&opt.db_path, &user_id);
+
+        let client = HttpClient::default();
+        let user = User::UserId { user_id };
+        let batcher = AutoBatcher::new(client, Batcher::new(None), SEGMENT_API_KEY.to_string());
+
+        let (sender, inbox) = mpsc::channel(100); // How many analytics can we bufferize
+
+        let segment = Box::new(Segment {
+            inbox,
+            user: user.clone(),
+            opt: opt.clone(),
+            batcher,
+            post_search_aggregator: SearchAggregator::default(),
+            get_search_aggregator: SearchAggregator::default(),
+            add_documents_aggregator: DocumentsAggregator::default(),
+            update_documents_aggregator: DocumentsAggregator::default(),
+        });
+        tokio::spawn(segment.run(meilisearch.clone()));
+
+        let this = Self {
+            sender,
+            user: user.clone(),
+        };
+        // batch the launched for the first time track event
+        if first_time_run {
+            this.publish("Launched".to_string(), json!({}), None);
+        }
+
+        (Arc::new(this), user.to_string())
+    }
+}
+
+impl super::Analytics for SegmentAnalytics {
+    fn publish(&self, event_name: String, mut send: Value, request: Option<&HttpRequest>) {
+        let user_agent = request
+            .map(|req| req.headers().get(USER_AGENT))
+            .flatten()
+            .map(|header| header.to_str().unwrap_or("unknown"))
+            .map(|s| s.split(';').map(str::trim).collect::<Vec<&str>>());
+
+        send["user-agent"] = json!(user_agent);
+        let event = Track {
+            user: self.user.clone(),
+            event: event_name.clone(),
+            properties: send,
+            ..Default::default()
+        };
+        let _ = self
+            .sender
+            .try_send(AnalyticsMsg::BatchMessage(event.into()));
+    }
+
+    fn get_search(&self, aggregate: SearchAggregator) {
+        let _ = self
+            .sender
+            .try_send(AnalyticsMsg::AggregateGetSearch(aggregate));
+    }
+
+    fn post_search(&self, aggregate: SearchAggregator) {
+        let _ = self
+            .sender
+            .try_send(AnalyticsMsg::AggregatePostSearch(aggregate));
+    }
+
+    fn add_documents(
+        &self,
+        documents_query: &UpdateDocumentsQuery,
+        index_creation: bool,
+        request: &HttpRequest,
+    ) {
+        let aggregate = DocumentsAggregator::from_query(documents_query, index_creation, request);
+        let _ = self
+            .sender
+            .try_send(AnalyticsMsg::AggregateAddDocuments(aggregate));
+    }
+
+    fn update_documents(
+        &self,
+        documents_query: &UpdateDocumentsQuery,
+        index_creation: bool,
+        request: &HttpRequest,
+    ) {
+        let aggregate = DocumentsAggregator::from_query(documents_query, index_creation, request);
+        let _ = self
+            .sender
+            .try_send(AnalyticsMsg::AggregateUpdateDocuments(aggregate));
+    }
+}
+
+pub struct Segment {
+    inbox: Receiver<AnalyticsMsg>,
+    user: User,
+    opt: Opt,
+    batcher: AutoBatcher,
+    get_search_aggregator: SearchAggregator,
+    post_search_aggregator: SearchAggregator,
+    add_documents_aggregator: DocumentsAggregator,
+    update_documents_aggregator: DocumentsAggregator,
+}
+
+impl Segment {
+    fn compute_traits(opt: &Opt, stats: Stats) -> Value {
+        static FIRST_START_TIMESTAMP: Lazy<Instant> = Lazy::new(Instant::now);
+        static SYSTEM: Lazy<Value> = Lazy::new(|| {
+            let mut sys = System::new_all();
+            sys.refresh_all();
+            let kernel_version = sys
+                .kernel_version()
+                .map(|k| k.split_once("-").map(|(k, _)| k.to_string()))
+                .flatten();
+            json!({
+                    "distribution": sys.name(),
+                    "kernel_version": kernel_version,
+                    "cores": sys.processors().len(),
+                    "ram_size": sys.total_memory(),
+                    "disk_size": sys.disks().iter().map(|disk| disk.total_space()).max(),
+                    "server_provider": std::env::var("MEILI_SERVER_PROVIDER").ok(),
+            })
+        });
+        let infos = json!({
+            "env": opt.env.clone(),
+            "has_snapshot": opt.schedule_snapshot,
+        });
+
+        let number_of_documents = stats
+            .indexes
+            .values()
+            .map(|index| index.number_of_documents)
+            .collect::<Vec<u64>>();
+
+        json!({
+            "start_since_days": FIRST_START_TIMESTAMP.elapsed().as_secs() / (60 * 60 * 24), // one day
+            "system": *SYSTEM,
+            "stats": {
+                "database_size": stats.database_size,
+                "indexes_number": stats.indexes.len(),
+                "documents_number": number_of_documents,
+            },
+            "infos": infos,
+        })
+    }
+
+    async fn run(mut self, meilisearch: MeiliSearch) {
+        const INTERVAL: Duration = Duration::from_secs(60 * 60); // one hour
+        let mut interval = tokio::time::interval(INTERVAL);
+
+        loop {
+            select! {
+                _ = interval.tick() => {
+                    self.tick(meilisearch.clone()).await;
+                },
+                msg = self.inbox.recv() => {
+                    match msg {
+                        Some(AnalyticsMsg::BatchMessage(msg)) => drop(self.batcher.push(msg).await),
+                        Some(AnalyticsMsg::AggregateGetSearch(agreg)) => self.get_search_aggregator.aggregate(agreg),
+                        Some(AnalyticsMsg::AggregatePostSearch(agreg)) => self.post_search_aggregator.aggregate(agreg),
+                        Some(AnalyticsMsg::AggregateAddDocuments(agreg)) => self.add_documents_aggregator.aggregate(agreg),
+                        Some(AnalyticsMsg::AggregateUpdateDocuments(agreg)) => self.update_documents_aggregator.aggregate(agreg),
+                        None => (),
+                    }
+                }
+            }
+        }
+    }
+
+    async fn tick(&mut self, meilisearch: MeiliSearch) {
+        if let Ok(stats) = meilisearch.get_all_stats().await {
+            let _ = self
+                .batcher
+                .push(Identify {
+                    context: Some(json!({
+                        "app": {
+                            "version": env!("CARGO_PKG_VERSION").to_string(),
+                        },
+                    })),
+                    user: self.user.clone(),
+                    traits: Self::compute_traits(&self.opt, stats),
+                    ..Default::default()
+                })
+                .await;
+        }
+        let get_search = std::mem::take(&mut self.get_search_aggregator)
+            .into_event(&self.user, "Documents Searched GET");
+        let post_search = std::mem::take(&mut self.post_search_aggregator)
+            .into_event(&self.user, "Documents Searched POST");
+        let add_documents = std::mem::take(&mut self.add_documents_aggregator)
+            .into_event(&self.user, "Documents Added");
+        let update_documents = std::mem::take(&mut self.update_documents_aggregator)
+            .into_event(&self.user, "Documents Updated");
+
+        if let Some(get_search) = get_search {
+            let _ = self.batcher.push(get_search).await;
+        }
+        if let Some(post_search) = post_search {
+            let _ = self.batcher.push(post_search).await;
+        }
+        if let Some(add_documents) = add_documents {
+            let _ = self.batcher.push(add_documents).await;
+        }
+        if let Some(update_documents) = update_documents {
+            let _ = self.batcher.push(update_documents).await;
+        }
+        let _ = self.batcher.flush().await;
+    }
+}
+
+#[derive(Default)]
+pub struct SearchAggregator {
+    // context
+    user_agents: HashSet<String>,
+
+    // requests
+    total_received: usize,
+    total_succeeded: usize,
+    time_spent: BinaryHeap<usize>,
+
+    // sort
+    sort_with_geo_point: bool,
+    // everytime a request has a filter, this field must be incremented by the number of terms it contains
+    sort_sum_of_criteria_terms: usize,
+    // everytime a request has a filter, this field must be incremented by one
+    sort_total_number_of_criteria: usize,
+
+    // filter
+    filter_with_geo_radius: bool,
+    // everytime a request has a filter, this field must be incremented by the number of terms it contains
+    filter_sum_of_criteria_terms: usize,
+    // everytime a request has a filter, this field must be incremented by one
+    filter_total_number_of_criteria: usize,
+    used_syntax: HashMap<String, usize>,
+
+    // q
+    // everytime a request has a q field, this field must be incremented by the number of terms
+    sum_of_terms_count: usize,
+    // everytime a request has a q field, this field must be incremented by one
+    total_number_of_q: usize,
+
+    // pagination
+    max_limit: usize,
+    max_offset: usize,
+}
+
+impl SearchAggregator {
+    pub fn from_query(query: &SearchQuery, request: &HttpRequest) -> Self {
+        let mut ret = Self::default();
+        ret.total_received = 1;
+        ret.user_agents = extract_user_agents(request).into_iter().collect();
+
+        if let Some(ref sort) = query.sort {
+            ret.sort_total_number_of_criteria = 1;
+            ret.sort_with_geo_point = sort.iter().any(|s| s.contains("_geoPoint("));
+            ret.sort_sum_of_criteria_terms = sort.len();
+        }
+
+        if let Some(ref filter) = query.filter {
+            static RE: Lazy<Regex> = Lazy::new(|| Regex::new("AND | OR").unwrap());
+            ret.filter_total_number_of_criteria = 1;
+
+            let syntax = match filter {
+                Value::String(_) => "string".to_string(),
+                Value::Array(values) => {
+                    if values
+                        .iter()
+                        .map(|v| v.to_string())
+                        .any(|s| RE.is_match(&s))
+                    {
+                        "mixed".to_string()
+                    } else {
+                        "array".to_string()
+                    }
+                }
+                _ => "none".to_string(),
+            };
+            // convert the string to a HashMap
+            ret.used_syntax.insert(syntax, 1);
+
+            let stringified_filters = filter.to_string();
+            ret.filter_with_geo_radius = stringified_filters.contains("_geoRadius(");
+            ret.filter_sum_of_criteria_terms = RE.split(&stringified_filters).count();
+        }
+
+        if let Some(ref q) = query.q {
+            ret.total_number_of_q = 1;
+            ret.sum_of_terms_count = q.split_whitespace().count();
+        }
+
+        ret.max_limit = query.limit;
+        ret.max_offset = query.offset.unwrap_or_default();
+
+        ret
+    }
+
+    pub fn succeed(&mut self, result: &SearchResult) {
+        self.total_succeeded += 1;
+        self.time_spent.push(result.processing_time_ms as usize);
+    }
+
+    /// Aggregate one [SearchAggregator] into another.
+    pub fn aggregate(&mut self, mut other: Self) {
+        // context
+        for user_agent in other.user_agents.into_iter() {
+            self.user_agents.insert(user_agent);
+        }
+        // request
+        self.total_received += other.total_received;
+        self.total_succeeded += other.total_succeeded;
+        self.time_spent.append(&mut other.time_spent);
+        // sort
+        self.sort_with_geo_point |= other.sort_with_geo_point;
+        self.sort_sum_of_criteria_terms += other.sort_sum_of_criteria_terms;
+        self.sort_total_number_of_criteria += other.sort_total_number_of_criteria;
+        // filter
+        self.filter_with_geo_radius |= other.filter_with_geo_radius;
+        self.filter_sum_of_criteria_terms += other.filter_sum_of_criteria_terms;
+        self.filter_total_number_of_criteria += other.filter_total_number_of_criteria;
+        for (key, value) in other.used_syntax.into_iter() {
+            *self.used_syntax.entry(key).or_insert(0) += value;
+        }
+        // q
+        self.sum_of_terms_count += other.sum_of_terms_count;
+        self.total_number_of_q += other.total_number_of_q;
+        // pagination
+        self.max_limit = self.max_limit.max(other.max_limit);
+        self.max_offset = self.max_offset.max(other.max_offset);
+    }
+
+    pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
+        if self.total_received == 0 {
+            None
+        } else {
+            // the index of the 99th percentage of value
+            let percentile_99th = 0.99 * (self.total_succeeded as f64 - 1.) + 1.;
+            // we get all the values in a sorted manner
+            let time_spent = self.time_spent.into_sorted_vec();
+            // We are only intersted by the slowest value of the 99th fastest results
+            let time_spent = time_spent[percentile_99th as usize];
+
+            let properties = json!({
+                "user-agent": self.user_agents,
+                "requests": {
+                    "99th_response_time":  format!("{:.2}", time_spent),
+                    "total_succeeded": self.total_succeeded,
+                    "total_failed": self.total_received.saturating_sub(self.total_succeeded), // just to be sure we never panics
+                    "total_received": self.total_received,
+                },
+                "sort": {
+                    "with_geoPoint": self.sort_with_geo_point,
+                    "avg_criteria_number": format!("{:.2}", self.sort_sum_of_criteria_terms as f64 / self.sort_total_number_of_criteria as f64),
+                },
+                "filter": {
+                   "with_geoRadius": self.filter_with_geo_radius,
+                   "avg_criteria_number": format!("{:.2}", self.filter_sum_of_criteria_terms as f64 / self.filter_total_number_of_criteria as f64),
+                   "most_used_syntax": self.used_syntax.iter().max_by_key(|(_, v)| *v).map(|(k, _)| json!(k)).unwrap_or_else(|| json!(null)),
+                },
+                "q": {
+                   "avg_terms_number": format!("{:.2}", self.sum_of_terms_count as f64 / self.total_number_of_q as f64),
+                },
+                "pagination": {
+                   "max_limit": self.max_limit,
+                   "max_offset": self.max_offset,
+                },
+            });
+
+            Some(Track {
+                user: user.clone(),
+                event: event_name.to_string(),
+                properties,
+                ..Default::default()
+            })
+        }
+    }
+}
+
+#[derive(Default)]
+pub struct DocumentsAggregator {
+    // set to true when at least one request was received
+    updated: bool,
+
+    // context
+    user_agents: HashSet<String>,
+
+    content_types: HashSet<String>,
+    primary_keys: HashSet<String>,
+    index_creation: bool,
+}
+
+impl DocumentsAggregator {
+    pub fn from_query(
+        documents_query: &UpdateDocumentsQuery,
+        index_creation: bool,
+        request: &HttpRequest,
+    ) -> Self {
+        let mut ret = Self::default();
+
+        ret.updated = true;
+        ret.user_agents = extract_user_agents(request).into_iter().collect();
+        if let Some(primary_key) = documents_query.primary_key.clone() {
+            ret.primary_keys.insert(primary_key);
+        }
+        let content_type = request
+            .headers()
+            .get(CONTENT_TYPE)
+            .map(|s| s.to_str().unwrap_or("unkown"))
+            .unwrap()
+            .to_string();
+        ret.content_types.insert(content_type);
+        ret.index_creation = index_creation;
+
+        ret
+    }
+
+    /// Aggregate one [DocumentsAggregator] into another.
+    pub fn aggregate(&mut self, other: Self) {
+        self.updated |= other.updated;
+        // we can't create a union because there is no `into_union` method
+        for user_agent in other.user_agents.into_iter() {
+            self.user_agents.insert(user_agent);
+        }
+        for primary_key in other.primary_keys.into_iter() {
+            self.primary_keys.insert(primary_key);
+        }
+        for content_type in other.content_types.into_iter() {
+            self.content_types.insert(content_type);
+        }
+        self.index_creation |= other.index_creation;
+    }
+
+    pub fn into_event(self, user: &User, event_name: &str) -> Option<Track> {
+        if !self.updated {
+            None
+        } else {
+            let properties = json!({
+                "user-agent": self.user_agents,
+                "payload_type": self.content_types,
+                "primary_key": self.primary_keys,
+                "index_creation": self.index_creation,
+            });
+
+            Some(Track {
+                user: user.clone(),
+                event: event_name.to_string(),
+                properties,
+                ..Default::default()
+            })
+        }
+    }
+}

meilisearch-http/src/data/mod.rs

@@ -1,133 +0,0 @@
-use std::ops::Deref;
-use std::sync::Arc;
-
-use sha2::Digest;
-
-use crate::index::{Checked, Settings};
-use crate::index_controller::{
-    error::Result, DumpInfo, IndexController, IndexMetadata, IndexSettings, IndexStats, Stats,
-};
-use crate::option::Opt;
-
-pub mod search;
-mod updates;
-
-#[derive(Clone)]
-pub struct Data {
-    inner: Arc<DataInner>,
-}
-
-impl Deref for Data {
-    type Target = DataInner;
-
-    fn deref(&self) -> &Self::Target {
-        &self.inner
-    }
-}
-
-pub struct DataInner {
-    pub index_controller: IndexController,
-    pub api_keys: ApiKeys,
-    options: Opt,
-}
-
-#[derive(Clone)]
-pub struct ApiKeys {
-    pub public: Option<String>,
-    pub private: Option<String>,
-    pub master: Option<String>,
-}
-
-impl ApiKeys {
-    pub fn generate_missing_api_keys(&mut self) {
-        if let Some(master_key) = &self.master {
-            if self.private.is_none() {
-                let key = format!("{}-private", master_key);
-                let sha = sha2::Sha256::digest(key.as_bytes());
-                self.private = Some(format!("{:x}", sha));
-            }
-            if self.public.is_none() {
-                let key = format!("{}-public", master_key);
-                let sha = sha2::Sha256::digest(key.as_bytes());
-                self.public = Some(format!("{:x}", sha));
-            }
-        }
-    }
-}
-
-impl Data {
-    pub fn new(options: Opt) -> anyhow::Result<Data> {
-        let path = options.db_path.clone();
-
-        let index_controller = IndexController::new(&path, &options)?;
-
-        let mut api_keys = ApiKeys {
-            master: options.clone().master_key,
-            private: None,
-            public: None,
-        };
-
-        api_keys.generate_missing_api_keys();
-
-        let inner = DataInner {
-            index_controller,
-            api_keys,
-            options,
-        };
-        let inner = Arc::new(inner);
-
-        Ok(Data { inner })
-    }
-
-    pub async fn settings(&self, uid: String) -> Result<Settings<Checked>> {
-        self.index_controller.settings(uid).await
-    }
-
-    pub async fn list_indexes(&self) -> Result<Vec<IndexMetadata>> {
-        self.index_controller.list_indexes().await
-    }
-
-    pub async fn index(&self, uid: String) -> Result<IndexMetadata> {
-        self.index_controller.get_index(uid).await
-    }
-
-    pub async fn create_index(
-        &self,
-        uid: String,
-        primary_key: Option<String>,
-    ) -> Result<IndexMetadata> {
-        let settings = IndexSettings {
-            uid: Some(uid),
-            primary_key,
-        };
-
-        let meta = self.index_controller.create_index(settings).await?;
-        Ok(meta)
-    }
-
-    pub async fn get_index_stats(&self, uid: String) -> Result<IndexStats> {
-        Ok(self.index_controller.get_index_stats(uid).await?)
-    }
-
-    pub async fn get_all_stats(&self) -> Result<Stats> {
-        Ok(self.index_controller.get_all_stats().await?)
-    }
-
-    pub async fn create_dump(&self) -> Result<DumpInfo> {
-        Ok(self.index_controller.create_dump().await?)
-    }
-
-    pub async fn dump_status(&self, uid: String) -> Result<DumpInfo> {
-        Ok(self.index_controller.dump_info(uid).await?)
-    }
-
-    #[inline]
-    pub fn http_payload_size_limit(&self) -> usize {
-        self.options.http_payload_size_limit.get_bytes() as usize
-    }
-
-    #[inline]
-    pub fn api_keys(&self) -> &ApiKeys {
-        &self.api_keys
-    }
-}

meilisearch-http/src/data/search.rs

@@ -1,34 +0,0 @@
-use serde_json::{Map, Value};
-
-use super::Data;
-use crate::index::{SearchQuery, SearchResult};
-use crate::index_controller::error::Result;
-
-impl Data {
-    pub async fn search(&self, index: String, search_query: SearchQuery) -> Result<SearchResult> {
-        self.index_controller.search(index, search_query).await
-    }
-
-    pub async fn retrieve_documents(
-        &self,
-        index: String,
-        offset: usize,
-        limit: usize,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Vec<Map<String, Value>>> {
-        self.index_controller
-            .documents(index, offset, limit, attributes_to_retrieve)
-            .await
-    }
-
-    pub async fn retrieve_document(
-        &self,
-        index: String,
-        document_id: String,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Map<String, Value>> {
-        self.index_controller
-            .document(index, document_id, attributes_to_retrieve)
-            .await
-    }
-}

meilisearch-http/src/data/updates.rs

@@ -1,80 +0,0 @@
-use milli::update::{IndexDocumentsMethod, UpdateFormat};
-
-use crate::extractors::payload::Payload;
-use crate::index::{Checked, Settings};
-use crate::index_controller::{error::Result, IndexMetadata, IndexSettings, UpdateStatus};
-use crate::Data;
-
-impl Data {
-    pub async fn add_documents(
-        &self,
-        index: String,
-        method: IndexDocumentsMethod,
-        format: UpdateFormat,
-        stream: Payload,
-        primary_key: Option<String>,
-    ) -> Result<UpdateStatus> {
-        let update_status = self
-            .index_controller
-            .add_documents(index, method, format, stream, primary_key)
-            .await?;
-        Ok(update_status)
-    }
-
-    pub async fn update_settings(
-        &self,
-        index: String,
-        settings: Settings<Checked>,
-        create: bool,
-    ) -> Result<UpdateStatus> {
-        let update = self
-            .index_controller
-            .update_settings(index, settings, create)
-            .await?;
-        Ok(update)
-    }
-
-    pub async fn clear_documents(&self, index: String) -> Result<UpdateStatus> {
-        let update = self.index_controller.clear_documents(index).await?;
-        Ok(update)
-    }
-
-    pub async fn delete_documents(
-        &self,
-        index: String,
-        document_ids: Vec<String>,
-    ) -> Result<UpdateStatus> {
-        let update = self
-            .index_controller
-            .delete_documents(index, document_ids)
-            .await?;
-        Ok(update)
-    }
-
-    pub async fn delete_index(&self, index: String) -> Result<()> {
-        self.index_controller.delete_index(index).await?;
-        Ok(())
-    }
-
-    pub async fn get_update_status(&self, index: String, uid: u64) -> Result<UpdateStatus> {
-        self.index_controller.update_status(index, uid).await
-    }
-
-    pub async fn get_updates_status(&self, index: String) -> Result<Vec<UpdateStatus>> {
-        self.index_controller.all_update_status(index).await
-    }
-
-    pub async fn update_index(
-        &self,
-        uid: String,
-        primary_key: Option<String>,
-        new_uid: Option<String>,
-    ) -> Result<IndexMetadata> {
-        let settings = IndexSettings {
-            uid: new_uid,
-            primary_key,
-        };
-
-        self.index_controller.update_index(uid, settings).await
-    }
-}

meilisearch-http/src/error.rs

@@ -3,21 +3,50 @@ use std::fmt;
 
 use actix_web as aweb;
 use actix_web::body::Body;
-use actix_web::dev::BaseHttpResponseBuilder;
 use actix_web::http::StatusCode;
+use actix_web::HttpResponseBuilder;
 use aweb::error::{JsonPayloadError, QueryPayloadError};
 use meilisearch_error::{Code, ErrorCode};
-use milli::UserError;
 use serde::{Deserialize, Serialize};
 
+#[derive(Debug, thiserror::Error)]
+pub enum MeilisearchHttpError {
+    #[error("A Content-Type header is missing. Accepted values for the Content-Type header are: {}",
+            .0.iter().map(|s| format!("`{}`", s)).collect::<Vec<_>>().join(", "))]
+    MissingContentType(Vec<String>),
+    #[error(
+        "The Content-Type `{0}` is invalid. Accepted values for the Content-Type header are: {}",
+        .1.iter().map(|s| format!("`{}`", s)).collect::<Vec<_>>().join(", ")
+    )]
+    InvalidContentType(String, Vec<String>),
+}
+
+impl ErrorCode for MeilisearchHttpError {
+    fn error_code(&self) -> Code {
+        match self {
+            MeilisearchHttpError::MissingContentType(_) => Code::MissingContentType,
+            MeilisearchHttpError::InvalidContentType(_, _) => Code::InvalidContentType,
+        }
+    }
+}
+
+impl From<MeilisearchHttpError> for aweb::Error {
+    fn from(other: MeilisearchHttpError) -> Self {
+        aweb::Error::from(ResponseError::from(other))
+    }
+}
+
 #[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(rename_all = "camelCase")]
 pub struct ResponseError {
     #[serde(skip)]
     code: StatusCode,
     message: String,
+    #[serde(rename = "code")]
     error_code: String,
+    #[serde(rename = "type")]
     error_type: String,
+    #[serde(rename = "link")]
     error_link: String,
 }
 
@@ -43,9 +72,9 @@ where
 }
 
 impl aweb::error::ResponseError for ResponseError {
-    fn error_response(&self) -> aweb::BaseHttpResponse<Body> {
+    fn error_response(&self) -> aweb::HttpResponse<Body> {
         let json = serde_json::to_vec(self).unwrap();
-        BaseHttpResponseBuilder::new(self.status_code())
+        HttpResponseBuilder::new(self.status_code())
             .content_type("application/json")
             .body(json)
     }
@@ -55,64 +84,6 @@ impl aweb::error::ResponseError for ResponseError {
     }
 }
 
-macro_rules! internal_error {
-    ($target:ty : $($other:path), *) => {
-        $(
-            impl From<$other> for $target {
-                fn from(other: $other) -> Self {
-                    Self::Internal(Box::new(other))
-                }
-            }
-        )*
-    }
-}
-
-#[derive(Debug)]
-pub struct MilliError<'a>(pub &'a milli::Error);
-
-impl Error for MilliError<'_> {}
-
-impl fmt::Display for MilliError<'_> {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        self.0.fmt(f)
-    }
-}
-
-impl ErrorCode for MilliError<'_> {
-    fn error_code(&self) -> Code {
-        match self.0 {
-            milli::Error::InternalError(_) => Code::Internal,
-            milli::Error::IoError(_) => Code::Internal,
-            milli::Error::UserError(ref error) => {
-                match error {
-                    // TODO: wait for spec for new error codes.
-                    UserError::Csv(_)
-                    | UserError::SerdeJson(_)
-                    | UserError::MaxDatabaseSizeReached
-                    | UserError::InvalidCriterionName { .. }
-                    | UserError::InvalidDocumentId { .. }
-                    | UserError::InvalidStoreFile
-                    | UserError::NoSpaceLeftOnDevice
-                    | UserError::InvalidAscDescSyntax { .. }
-                    | UserError::DocumentLimitReached => Code::Internal,
-                    UserError::AttributeLimitReached => Code::MaxFieldsLimitExceeded,
-                    UserError::InvalidFilter(_) => Code::Filter,
-                    UserError::InvalidFilterAttribute(_) => Code::Filter,
-                    UserError::InvalidSortName { .. } => Code::Sort,
-                    UserError::MissingDocumentId { .. } => Code::MissingDocumentId,
-                    UserError::MissingPrimaryKey => Code::MissingPrimaryKey,
-                    UserError::PrimaryKeyCannotBeChanged => Code::PrimaryKeyAlreadyPresent,
-                    UserError::PrimaryKeyCannotBeReset => Code::PrimaryKeyAlreadyPresent,
-                    UserError::SortRankingRuleMissing => Code::Sort,
-                    UserError::UnknownInternalDocumentId { .. } => Code::DocumentNotFound,
-                    UserError::InvalidFacetsDistribution { .. } => Code::BadRequest,
-                    UserError::InvalidSortableAttribute { .. } => Code::Sort,
-                }
-            }
-        }
-    }
-}
-
 impl fmt::Display for PayloadError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
@@ -134,7 +105,7 @@ impl ErrorCode for PayloadError {
     fn error_code(&self) -> Code {
         match self {
             PayloadError::Json(err) => match err {
-                JsonPayloadError::Overflow => Code::PayloadTooLarge,
+                JsonPayloadError::Overflow { .. } => Code::PayloadTooLarge,
                 JsonPayloadError::ContentType => Code::UnsupportedMediaType,
                 JsonPayloadError::Payload(aweb::error::PayloadError::Overflow) => {
                     Code::PayloadTooLarge
@@ -163,9 +134,8 @@ impl From<QueryPayloadError> for PayloadError {
     }
 }
 
-pub fn payload_error_handler<E>(err: E) -> ResponseError
-where
-    E: Into<PayloadError>,
-{
-    err.into().into()
+impl From<PayloadError> for aweb::Error {
+    fn from(other: PayloadError) -> Self {
+        aweb::Error::from(ResponseError::from(other))
+    }
 }

meilisearch-http/src/extractors/authentication/error.rs

@@ -2,14 +2,14 @@ use meilisearch_error::{Code, ErrorCode};
 
 #[derive(Debug, thiserror::Error)]
 pub enum AuthenticationError {
-    #[error("You must have an authorization token")]
+    #[error("The X-MEILI-API-KEY header is missing.")]
     MissingAuthorizationHeader,
-    #[error("Invalid API key")]
+    #[error("The provided API key is invalid.")]
     InvalidToken(String),
     // Triggered on configuration error.
-    #[error("Irretrievable state")]
+    #[error("An internal error has occurred. `Irretrievable state`.")]
     IrretrievableState,
-    #[error("Unknown authentication policy")]
+    #[error("An internal error has occurred. `Unknown authentication policy`.")]
     UnknownPolicy,
 }
 

meilisearch-http/src/extractors/authentication/mod.rs

@@ -145,7 +145,7 @@ impl<P: Policy + 'static, D: 'static + Clone> FromRequest for GuardedData<P, D>
 
     fn from_request(
         req: &actix_web::HttpRequest,
-        _payload: &mut actix_http::Payload,
+        _payload: &mut actix_web::dev::Payload,
     ) -> Self::Future {
         match req.app_data::<Self::Config>() {
             Some(config) => match config {
@@ -168,7 +168,8 @@ impl<P: Policy + 'static, D: 'static + Clone> FromRequest for GuardedData<P, D>
                                     None => err(AuthenticationError::IrretrievableState.into()),
                                 }
                             } else {
-                                err(AuthenticationError::InvalidToken(String::from("hello")).into())
+                                let token = token.to_str().unwrap_or("unknown").to_string();
+                                err(AuthenticationError::InvalidToken(token).into())
                             }
                         }
                         None => err(AuthenticationError::MissingAuthorizationHeader.into()),

meilisearch-http/src/extractors/payload.rs

@@ -1,7 +1,7 @@
 use std::pin::Pin;
 use std::task::{Context, Poll};
 
-use actix_http::error::PayloadError;
+use actix_web::error::PayloadError;
 use actix_web::{dev, web, FromRequest, HttpRequest};
 use futures::future::{ready, Ready};
 use futures::Stream;

meilisearch-http/src/helpers/mod.rs

@@ -1,4 +1,3 @@
-pub mod compression;
 mod env;
 
 pub use env::EnvSizer;

meilisearch-http/src/index/dump.rs

@@ -1,170 +0,0 @@
-use std::fs::{create_dir_all, File};
-use std::io::{BufRead, BufReader, Write};
-use std::path::Path;
-use std::sync::Arc;
-
-use anyhow::{bail, Context};
-use heed::RoTxn;
-use indexmap::IndexMap;
-use milli::update::{IndexDocumentsMethod, UpdateFormat::JsonStream};
-use serde::{Deserialize, Serialize};
-use serde_json::Value;
-
-use crate::index_controller::{asc_ranking_rule, desc_ranking_rule};
-use crate::option::IndexerOpts;
-
-use super::error::Result;
-use super::{update_handler::UpdateHandler, Index, Settings, Unchecked};
-
-#[derive(Serialize, Deserialize)]
-struct DumpMeta {
-    settings: Settings<Unchecked>,
-    primary_key: Option<String>,
-}
-
-const META_FILE_NAME: &str = "meta.json";
-const DATA_FILE_NAME: &str = "documents.jsonl";
-
-impl Index {
-    pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
-        // acquire write txn make sure any ongoing write is finished before we start.
-        let txn = self.env.write_txn()?;
-
-        self.dump_documents(&txn, &path)?;
-        self.dump_meta(&txn, &path)?;
-
-        Ok(())
-    }
-
-    fn dump_documents(&self, txn: &RoTxn, path: impl AsRef<Path>) -> Result<()> {
-        let document_file_path = path.as_ref().join(DATA_FILE_NAME);
-        let mut document_file = File::create(&document_file_path)?;
-
-        let documents = self.all_documents(txn)?;
-        let fields_ids_map = self.fields_ids_map(txn)?;
-
-        // dump documents
-        let mut json_map = IndexMap::new();
-        for document in documents {
-            let (_, reader) = document?;
-
-            for (fid, bytes) in reader.iter() {
-                if let Some(name) = fields_ids_map.name(fid) {
-                    json_map.insert(name, serde_json::from_slice::<serde_json::Value>(bytes)?);
-                }
-            }
-
-            serde_json::to_writer(&mut document_file, &json_map)?;
-            document_file.write_all(b"\n")?;
-
-            json_map.clear();
-        }
-
-        Ok(())
-    }
-
-    fn dump_meta(&self, txn: &RoTxn, path: impl AsRef<Path>) -> Result<()> {
-        let meta_file_path = path.as_ref().join(META_FILE_NAME);
-        let mut meta_file = File::create(&meta_file_path)?;
-
-        let settings = self.settings_txn(txn)?.into_unchecked();
-        let primary_key = self.primary_key(txn)?.map(String::from);
-        let meta = DumpMeta {
-            settings,
-            primary_key,
-        };
-
-        serde_json::to_writer(&mut meta_file, &meta)?;
-
-        Ok(())
-    }
-
-    pub fn load_dump(
-        src: impl AsRef<Path>,
-        dst: impl AsRef<Path>,
-        size: usize,
-        indexing_options: &IndexerOpts,
-    ) -> anyhow::Result<()> {
-        let dir_name = src
-            .as_ref()
-            .file_name()
-            .with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?;
-
-        let dst_dir_path = dst.as_ref().join("indexes").join(dir_name);
-        create_dir_all(&dst_dir_path)?;
-
-        let meta_path = src.as_ref().join(META_FILE_NAME);
-        let mut meta_file = File::open(meta_path)?;
-
-        // We first deserialize the dump meta into a serde_json::Value and change
-        // the custom ranking rules settings from the old format to the new format.
-        let mut meta: Value = serde_json::from_reader(&mut meta_file)?;
-        if let Some(ranking_rules) = meta.pointer_mut("/settings/rankingRules") {
-            convert_custom_ranking_rules(ranking_rules);
-        }
-
-        // Then we serialize it back into a vec to deserialize it
-        // into a `DumpMeta` struct with the newly patched `rankingRules` format.
-        let patched_meta = serde_json::to_vec(&meta)?;
-
-        let DumpMeta {
-            settings,
-            primary_key,
-        } = serde_json::from_slice(&patched_meta)?;
-        let settings = settings.check();
-        let index = Self::open(&dst_dir_path, size)?;
-        let mut txn = index.write_txn()?;
-
-        let handler = UpdateHandler::new(indexing_options)?;
-
-        index.update_settings_txn(&mut txn, &settings, handler.update_builder(0))?;
-
-        let document_file_path = src.as_ref().join(DATA_FILE_NAME);
-        let reader = File::open(&document_file_path)?;
-        let mut reader = BufReader::new(reader);
-        reader.fill_buf()?;
-        // If the document file is empty, we don't perform the document addition, to prevent
-        // a primary key error to be thrown.
-        if !reader.buffer().is_empty() {
-            index.update_documents_txn(
-                &mut txn,
-                JsonStream,
-                IndexDocumentsMethod::UpdateDocuments,
-                Some(reader),
-                handler.update_builder(0),
-                primary_key.as_deref(),
-            )?;
-        }
-
-        txn.commit()?;
-
-        match Arc::try_unwrap(index.0) {
-            Ok(inner) => inner.prepare_for_closing().wait(),
-            Err(_) => bail!("Could not close index properly."),
-        }
-
-        Ok(())
-    }
-}
-
-/// Converts the ranking rules from the format `asc(_)`, `desc(_)` to the format `_:asc`, `_:desc`.
-///
-/// This is done for compatibility reasons, and to avoid a new dump version,
-/// since the new syntax was introduced soon after the new dump version.
-fn convert_custom_ranking_rules(ranking_rules: &mut Value) {
-    *ranking_rules = match ranking_rules.take() {
-        Value::Array(values) => values
-            .into_iter()
-            .filter_map(|value| match value {
-                Value::String(s) if s.starts_with("asc") => asc_ranking_rule(&s)
-                    .map(|f| format!("{}:asc", f))
-                    .map(Value::String),
-                Value::String(s) if s.starts_with("desc") => desc_ranking_rule(&s)
-                    .map(|f| format!("{}:desc", f))
-                    .map(Value::String),
-                otherwise => Some(otherwise),
-            })
-            .collect(),
-        otherwise => otherwise,
-    }
-}

meilisearch-http/src/index_controller/dump_actor/loaders/mod.rs

@@ -1,2 +0,0 @@
-pub mod v1;
-pub mod v2;

meilisearch-http/src/index_controller/dump_actor/loaders/v2.rs

@@ -1,59 +0,0 @@
-use std::path::Path;
-
-use chrono::{DateTime, Utc};
-use log::info;
-use serde::{Deserialize, Serialize};
-
-use crate::index::Index;
-use crate::index_controller::{update_actor::UpdateStore, uuid_resolver::HeedUuidStore};
-use crate::option::IndexerOpts;
-
-#[derive(Serialize, Deserialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct MetadataV2 {
-    db_version: String,
-    index_db_size: usize,
-    update_db_size: usize,
-    dump_date: DateTime<Utc>,
-}
-
-impl MetadataV2 {
-    pub fn new(index_db_size: usize, update_db_size: usize) -> Self {
-        Self {
-            db_version: env!("CARGO_PKG_VERSION").to_string(),
-            index_db_size,
-            update_db_size,
-            dump_date: Utc::now(),
-        }
-    }
-
-    pub fn load_dump(
-        self,
-        src: impl AsRef<Path>,
-        dst: impl AsRef<Path>,
-        index_db_size: usize,
-        update_db_size: usize,
-        indexing_options: &IndexerOpts,
-    ) -> anyhow::Result<()> {
-        info!(
-            "Loading dump from {}, dump database version: {}, dump version: V2",
-            self.dump_date, self.db_version
-        );
-
-        info!("Loading index database.");
-        HeedUuidStore::load_dump(src.as_ref(), &dst)?;
-
-        info!("Loading updates.");
-        UpdateStore::load_dump(&src, &dst, update_db_size)?;
-
-        info!("Loading indexes.");
-        let indexes_path = src.as_ref().join("indexes");
-        let indexes = indexes_path.read_dir()?;
-        for index in indexes {
-            let index = index?;
-            Index::load_dump(&index.path(), &dst, index_db_size, indexing_options)?;
-        }
-
-        Ok(())
-    }
-}

meilisearch-http/src/index_controller/dump_actor/mod.rs

@@ -1,203 +0,0 @@
-use std::fs::File;
-use std::path::{Path, PathBuf};
-
-use anyhow::Context;
-use chrono::{DateTime, Utc};
-use log::{info, trace, warn};
-#[cfg(test)]
-use mockall::automock;
-use serde::{Deserialize, Serialize};
-use tokio::fs::create_dir_all;
-
-use loaders::v1::MetadataV1;
-use loaders::v2::MetadataV2;
-
-pub use actor::DumpActor;
-pub use handle_impl::*;
-pub use message::DumpMsg;
-
-use super::{update_actor::UpdateActorHandle, uuid_resolver::UuidResolverHandle};
-use crate::index_controller::dump_actor::error::DumpActorError;
-use crate::{helpers::compression, option::IndexerOpts};
-use error::Result;
-
-mod actor;
-pub mod error;
-mod handle_impl;
-mod loaders;
-mod message;
-
-const META_FILE_NAME: &str = "metadata.json";
-
-#[async_trait::async_trait]
-#[cfg_attr(test, automock)]
-pub trait DumpActorHandle {
-    /// Start the creation of a dump
-    /// Implementation: [handle_impl::DumpActorHandleImpl::create_dump]
-    async fn create_dump(&self) -> Result<DumpInfo>;
-
-    /// Return the status of an already created dump
-    /// Implementation: [handle_impl::DumpActorHandleImpl::dump_info]
-    async fn dump_info(&self, uid: String) -> Result<DumpInfo>;
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(tag = "dumpVersion")]
-pub enum Metadata {
-    V1(MetadataV1),
-    V2(MetadataV2),
-}
-
-impl Metadata {
-    pub fn new_v2(index_db_size: usize, update_db_size: usize) -> Self {
-        let meta = MetadataV2::new(index_db_size, update_db_size);
-        Self::V2(meta)
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
-#[serde(rename_all = "snake_case")]
-pub enum DumpStatus {
-    Done,
-    InProgress,
-    Failed,
-}
-
-#[derive(Debug, Serialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct DumpInfo {
-    pub uid: String,
-    pub status: DumpStatus,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub error: Option<String>,
-    started_at: DateTime<Utc>,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    finished_at: Option<DateTime<Utc>>,
-}
-
-impl DumpInfo {
-    pub fn new(uid: String, status: DumpStatus) -> Self {
-        Self {
-            uid,
-            status,
-            error: None,
-            started_at: Utc::now(),
-            finished_at: None,
-        }
-    }
-
-    pub fn with_error(&mut self, error: String) {
-        self.status = DumpStatus::Failed;
-        self.finished_at = Some(Utc::now());
-        self.error = Some(error);
-    }
-
-    pub fn done(&mut self) {
-        self.finished_at = Some(Utc::now());
-        self.status = DumpStatus::Done;
-    }
-
-    pub fn dump_already_in_progress(&self) -> bool {
-        self.status == DumpStatus::InProgress
-    }
-}
-
-pub fn load_dump(
-    dst_path: impl AsRef<Path>,
-    src_path: impl AsRef<Path>,
-    index_db_size: usize,
-    update_db_size: usize,
-    indexer_opts: &IndexerOpts,
-) -> anyhow::Result<()> {
-    let tmp_src = tempfile::tempdir_in(".")?;
-    let tmp_src_path = tmp_src.path();
-
-    compression::from_tar_gz(&src_path, tmp_src_path)?;
-
-    let meta_path = tmp_src_path.join(META_FILE_NAME);
-    let mut meta_file = File::open(&meta_path)?;
-    let meta: Metadata = serde_json::from_reader(&mut meta_file)?;
-
-    let dst_dir = dst_path
-        .as_ref()
-        .parent()
-        .with_context(|| format!("Invalid db path: {}", dst_path.as_ref().display()))?;
-
-    let tmp_dst = tempfile::tempdir_in(dst_dir)?;
-
-    match meta {
-        Metadata::V1(meta) => {
-            meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size, indexer_opts)?
-        }
-        Metadata::V2(meta) => meta.load_dump(
-            &tmp_src_path,
-            tmp_dst.path(),
-            index_db_size,
-            update_db_size,
-            indexer_opts,
-        )?,
-    }
-    // Persist and atomically rename the db
-    let persisted_dump = tmp_dst.into_path();
-    if dst_path.as_ref().exists() {
-        warn!("Overwriting database at {}", dst_path.as_ref().display());
-        std::fs::remove_dir_all(&dst_path)?;
-    }
-
-    std::fs::rename(&persisted_dump, &dst_path)?;
-
-    Ok(())
-}
-
-struct DumpTask<U, P> {
-    path: PathBuf,
-    uuid_resolver: U,
-    update_handle: P,
-    uid: String,
-    update_db_size: usize,
-    index_db_size: usize,
-}
-
-impl<U, P> DumpTask<U, P>
-where
-    U: UuidResolverHandle + Send + Sync + Clone + 'static,
-    P: UpdateActorHandle + Send + Sync + Clone + 'static,
-{
-    async fn run(self) -> Result<()> {
-        trace!("Performing dump.");
-
-        create_dir_all(&self.path).await?;
-
-        let path_clone = self.path.clone();
-        let temp_dump_dir =
-            tokio::task::spawn_blocking(|| tempfile::TempDir::new_in(path_clone)).await??;
-        let temp_dump_path = temp_dump_dir.path().to_owned();
-
-        let meta = Metadata::new_v2(self.index_db_size, self.update_db_size);
-        let meta_path = temp_dump_path.join(META_FILE_NAME);
-        let mut meta_file = File::create(&meta_path)?;
-        serde_json::to_writer(&mut meta_file, &meta)?;
-
-        let uuids = self.uuid_resolver.dump(temp_dump_path.clone()).await?;
-
-        self.update_handle
-            .dump(uuids, temp_dump_path.clone())
-            .await?;
-
-        let dump_path = tokio::task::spawn_blocking(move || -> Result<PathBuf> {
-            let temp_dump_file = tempfile::NamedTempFile::new_in(&self.path)?;
-            compression::to_tar_gz(temp_dump_path, temp_dump_file.path())
-                .map_err(|e| DumpActorError::Internal(e.into()))?;
-
-            let dump_path = self.path.join(self.uid).with_extension("dump");
-            temp_dump_file.persist(&dump_path)?;
-
-            Ok(dump_path)
-        })
-        .await??;
-
-        info!("Created dump in {:?}.", dump_path);
-
-        Ok(())
-    }
-}

meilisearch-http/src/index_controller/index_actor/actor.rs

@@ -1,351 +0,0 @@
-use std::fs::File;
-use std::path::PathBuf;
-use std::sync::Arc;
-
-use async_stream::stream;
-use futures::stream::StreamExt;
-use heed::CompactionOption;
-use log::debug;
-use milli::update::UpdateBuilder;
-use tokio::task::spawn_blocking;
-use tokio::{fs, sync::mpsc};
-use uuid::Uuid;
-
-use crate::index::{
-    update_handler::UpdateHandler, Checked, Document, SearchQuery, SearchResult, Settings,
-};
-use crate::index_controller::{
-    get_arc_ownership_blocking, Failed, IndexStats, Processed, Processing,
-};
-use crate::option::IndexerOpts;
-
-use super::error::{IndexActorError, Result};
-use super::{IndexMeta, IndexMsg, IndexSettings, IndexStore};
-
-pub const CONCURRENT_INDEX_MSG: usize = 10;
-
-pub struct IndexActor<S> {
-    receiver: Option<mpsc::Receiver<IndexMsg>>,
-    update_handler: Arc<UpdateHandler>,
-    store: S,
-}
-
-impl<S: IndexStore + Sync + Send> IndexActor<S> {
-    pub fn new(
-        receiver: mpsc::Receiver<IndexMsg>,
-        store: S,
-        options: &IndexerOpts,
-    ) -> anyhow::Result<Self> {
-        let update_handler = UpdateHandler::new(options)?;
-        let update_handler = Arc::new(update_handler);
-        let receiver = Some(receiver);
-        Ok(Self {
-            receiver,
-            update_handler,
-            store,
-        })
-    }
-
-    /// `run` poll the write_receiver and read_receiver concurrently, but while messages send
-    /// through the read channel are processed concurrently, the messages sent through the write
-    /// channel are processed one at a time.
-    pub async fn run(mut self) {
-        let mut receiver = self
-            .receiver
-            .take()
-            .expect("Index Actor must have a inbox at this point.");
-
-        let stream = stream! {
-            loop {
-                match receiver.recv().await {
-                    Some(msg) => yield msg,
-                    None => break,
-                }
-            }
-        };
-
-        stream
-            .for_each_concurrent(Some(CONCURRENT_INDEX_MSG), |msg| self.handle_message(msg))
-            .await;
-    }
-
-    async fn handle_message(&self, msg: IndexMsg) {
-        use IndexMsg::*;
-        match msg {
-            CreateIndex {
-                uuid,
-                primary_key,
-                ret,
-            } => {
-                let _ = ret.send(self.handle_create_index(uuid, primary_key).await);
-            }
-            Update {
-                ret,
-                meta,
-                data,
-                uuid,
-            } => {
-                let _ = ret.send(self.handle_update(uuid, meta, data).await);
-            }
-            Search { ret, query, uuid } => {
-                let _ = ret.send(self.handle_search(uuid, query).await);
-            }
-            Settings { ret, uuid } => {
-                let _ = ret.send(self.handle_settings(uuid).await);
-            }
-            Documents {
-                ret,
-                uuid,
-                attributes_to_retrieve,
-                offset,
-                limit,
-            } => {
-                let _ = ret.send(
-                    self.handle_fetch_documents(uuid, offset, limit, attributes_to_retrieve)
-                        .await,
-                );
-            }
-            Document {
-                uuid,
-                attributes_to_retrieve,
-                doc_id,
-                ret,
-            } => {
-                let _ = ret.send(
-                    self.handle_fetch_document(uuid, doc_id, attributes_to_retrieve)
-                        .await,
-                );
-            }
-            Delete { uuid, ret } => {
-                let _ = ret.send(self.handle_delete(uuid).await);
-            }
-            GetMeta { uuid, ret } => {
-                let _ = ret.send(self.handle_get_meta(uuid).await);
-            }
-            UpdateIndex {
-                uuid,
-                index_settings,
-                ret,
-            } => {
-                let _ = ret.send(self.handle_update_index(uuid, index_settings).await);
-            }
-            Snapshot { uuid, path, ret } => {
-                let _ = ret.send(self.handle_snapshot(uuid, path).await);
-            }
-            Dump { uuid, path, ret } => {
-                let _ = ret.send(self.handle_dump(uuid, path).await);
-            }
-            GetStats { uuid, ret } => {
-                let _ = ret.send(self.handle_get_stats(uuid).await);
-            }
-        }
-    }
-
-    async fn handle_search(&self, uuid: Uuid, query: SearchQuery) -> Result<SearchResult> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-        let result = spawn_blocking(move || index.perform_search(query)).await??;
-        Ok(result)
-    }
-
-    async fn handle_create_index(
-        &self,
-        uuid: Uuid,
-        primary_key: Option<String>,
-    ) -> Result<IndexMeta> {
-        let index = self.store.create(uuid, primary_key).await?;
-        let meta = spawn_blocking(move || IndexMeta::new(&index)).await??;
-        Ok(meta)
-    }
-
-    async fn handle_update(
-        &self,
-        uuid: Uuid,
-        meta: Processing,
-        data: Option<File>,
-    ) -> Result<std::result::Result<Processed, Failed>> {
-        debug!("Processing update {}", meta.id());
-        let update_handler = self.update_handler.clone();
-        let index = match self.store.get(uuid).await? {
-            Some(index) => index,
-            None => self.store.create(uuid, None).await?,
-        };
-
-        Ok(spawn_blocking(move || update_handler.handle_update(meta, data, index)).await?)
-    }
-
-    async fn handle_settings(&self, uuid: Uuid) -> Result<Settings<Checked>> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-        let result = spawn_blocking(move || index.settings()).await??;
-        Ok(result)
-    }
-
-    async fn handle_fetch_documents(
-        &self,
-        uuid: Uuid,
-        offset: usize,
-        limit: usize,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Vec<Document>> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-        let result =
-            spawn_blocking(move || index.retrieve_documents(offset, limit, attributes_to_retrieve))
-                .await??;
-
-        Ok(result)
-    }
-
-    async fn handle_fetch_document(
-        &self,
-        uuid: Uuid,
-        doc_id: String,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Document> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-
-        let result =
-            spawn_blocking(move || index.retrieve_document(doc_id, attributes_to_retrieve))
-                .await??;
-
-        Ok(result)
-    }
-
-    async fn handle_delete(&self, uuid: Uuid) -> Result<()> {
-        let index = self.store.delete(uuid).await?;
-
-        if let Some(index) = index {
-            tokio::task::spawn(async move {
-                let index = index.0;
-                let store = get_arc_ownership_blocking(index).await;
-                spawn_blocking(move || {
-                    store.prepare_for_closing().wait();
-                    debug!("Index closed");
-                });
-            });
-        }
-
-        Ok(())
-    }
-
-    async fn handle_get_meta(&self, uuid: Uuid) -> Result<IndexMeta> {
-        match self.store.get(uuid).await? {
-            Some(index) => {
-                let meta = spawn_blocking(move || IndexMeta::new(&index)).await??;
-                Ok(meta)
-            }
-            None => Err(IndexActorError::UnexistingIndex),
-        }
-    }
-
-    async fn handle_update_index(
-        &self,
-        uuid: Uuid,
-        index_settings: IndexSettings,
-    ) -> Result<IndexMeta> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-
-        let result = spawn_blocking(move || match index_settings.primary_key {
-            Some(primary_key) => {
-                let mut txn = index.write_txn()?;
-                if index.primary_key(&txn)?.is_some() {
-                    return Err(IndexActorError::ExistingPrimaryKey);
-                }
-                let mut builder = UpdateBuilder::new(0).settings(&mut txn, &index);
-                builder.set_primary_key(primary_key);
-                builder.execute(|_, _| ())?;
-                let meta = IndexMeta::new_txn(&index, &txn)?;
-                txn.commit()?;
-                Ok(meta)
-            }
-            None => {
-                let meta = IndexMeta::new(&index)?;
-                Ok(meta)
-            }
-        })
-        .await??;
-
-        Ok(result)
-    }
-
-    async fn handle_snapshot(&self, uuid: Uuid, mut path: PathBuf) -> Result<()> {
-        use tokio::fs::create_dir_all;
-
-        path.push("indexes");
-        create_dir_all(&path).await?;
-
-        if let Some(index) = self.store.get(uuid).await? {
-            let mut index_path = path.join(format!("index-{}", uuid));
-
-            create_dir_all(&index_path).await?;
-
-            index_path.push("data.mdb");
-            spawn_blocking(move || -> Result<()> {
-                // Get write txn to wait for ongoing write transaction before snapshot.
-                let _txn = index.write_txn()?;
-                index
-                    .env
-                    .copy_to_path(index_path, CompactionOption::Enabled)?;
-                Ok(())
-            })
-            .await??;
-        }
-
-        Ok(())
-    }
-
-    /// Create a `documents.jsonl` and a `settings.json` in `path/uid/` with a dump of all the
-    /// documents and all the settings.
-    async fn handle_dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-
-        let path = path.join(format!("indexes/index-{}/", uuid));
-        fs::create_dir_all(&path).await?;
-
-        tokio::task::spawn_blocking(move || index.dump(path)).await??;
-
-        Ok(())
-    }
-
-    async fn handle_get_stats(&self, uuid: Uuid) -> Result<IndexStats> {
-        let index = self
-            .store
-            .get(uuid)
-            .await?
-            .ok_or(IndexActorError::UnexistingIndex)?;
-
-        spawn_blocking(move || {
-            let rtxn = index.read_txn()?;
-
-            Ok(IndexStats {
-                size: index.size(),
-                number_of_documents: index.number_of_documents(&rtxn)?,
-                is_indexing: None,
-                field_distribution: index.field_distribution(&rtxn)?,
-            })
-        })
-        .await?
-    }
-}

meilisearch-http/src/index_controller/index_actor/error.rs

@@ -1,48 +0,0 @@
-use meilisearch_error::{Code, ErrorCode};
-
-use crate::{error::MilliError, index::error::IndexError};
-
-pub type Result<T> = std::result::Result<T, IndexActorError>;
-
-#[derive(thiserror::Error, Debug)]
-pub enum IndexActorError {
-    #[error("{0}")]
-    IndexError(#[from] IndexError),
-    #[error("Index already exists")]
-    IndexAlreadyExists,
-    #[error("Index not found")]
-    UnexistingIndex,
-    #[error("A primary key is already present. It's impossible to update it")]
-    ExistingPrimaryKey,
-    #[error("Internal Error: {0}")]
-    Internal(Box<dyn std::error::Error + Send + Sync + 'static>),
-    #[error("{0}")]
-    Milli(#[from] milli::Error),
-}
-
-macro_rules! internal_error {
-    ($($other:path), *) => {
-        $(
-            impl From<$other> for IndexActorError {
-                fn from(other: $other) -> Self {
-                    Self::Internal(Box::new(other))
-                }
-            }
-        )*
-    }
-}
-
-internal_error!(heed::Error, tokio::task::JoinError, std::io::Error);
-
-impl ErrorCode for IndexActorError {
-    fn error_code(&self) -> Code {
-        match self {
-            IndexActorError::IndexError(e) => e.error_code(),
-            IndexActorError::IndexAlreadyExists => Code::IndexAlreadyExists,
-            IndexActorError::UnexistingIndex => Code::IndexNotFound,
-            IndexActorError::ExistingPrimaryKey => Code::PrimaryKeyAlreadyPresent,
-            IndexActorError::Internal(_) => Code::Internal,
-            IndexActorError::Milli(e) => MilliError(e).error_code(),
-        }
-    }
-}

meilisearch-http/src/index_controller/index_actor/handle_impl.rs

@@ -1,164 +0,0 @@
-use crate::option::IndexerOpts;
-use std::path::{Path, PathBuf};
-
-use tokio::sync::{mpsc, oneshot};
-use uuid::Uuid;
-
-use crate::{
-    index::Checked,
-    index_controller::{IndexSettings, IndexStats, Processing},
-};
-use crate::{
-    index::{Document, SearchQuery, SearchResult, Settings},
-    index_controller::{Failed, Processed},
-};
-
-use super::error::Result;
-use super::{IndexActor, IndexActorHandle, IndexMeta, IndexMsg, MapIndexStore};
-
-#[derive(Clone)]
-pub struct IndexActorHandleImpl {
-    sender: mpsc::Sender<IndexMsg>,
-}
-
-#[async_trait::async_trait]
-impl IndexActorHandle for IndexActorHandleImpl {
-    async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::CreateIndex {
-            ret,
-            uuid,
-            primary_key,
-        };
-        let _ = self.sender.send(msg).await;
-        receiver.await.expect("IndexActor has been killed")
-    }
-
-    async fn update(
-        &self,
-        uuid: Uuid,
-        meta: Processing,
-        data: Option<std::fs::File>,
-    ) -> Result<std::result::Result<Processed, Failed>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Update {
-            ret,
-            meta,
-            data,
-            uuid,
-        };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result<SearchResult> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Search { uuid, query, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn settings(&self, uuid: Uuid) -> Result<Settings<Checked>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Settings { uuid, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn documents(
-        &self,
-        uuid: Uuid,
-        offset: usize,
-        limit: usize,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Vec<Document>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Documents {
-            uuid,
-            ret,
-            offset,
-            attributes_to_retrieve,
-            limit,
-        };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn document(
-        &self,
-        uuid: Uuid,
-        doc_id: String,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Document> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Document {
-            uuid,
-            ret,
-            doc_id,
-            attributes_to_retrieve,
-        };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn delete(&self, uuid: Uuid) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Delete { uuid, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn get_index_meta(&self, uuid: Uuid) -> Result<IndexMeta> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::GetMeta { uuid, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result<IndexMeta> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::UpdateIndex {
-            uuid,
-            index_settings,
-            ret,
-        };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Snapshot { uuid, path, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::Dump { uuid, path, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-
-    async fn get_index_stats(&self, uuid: Uuid) -> Result<IndexStats> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = IndexMsg::GetStats { uuid, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver.await.expect("IndexActor has been killed")?)
-    }
-}
-
-impl IndexActorHandleImpl {
-    pub fn new(
-        path: impl AsRef<Path>,
-        index_size: usize,
-        options: &IndexerOpts,
-    ) -> anyhow::Result<Self> {
-        let (sender, receiver) = mpsc::channel(100);
-
-        let store = MapIndexStore::new(path, index_size);
-        let actor = IndexActor::new(receiver, store, options)?;
-        tokio::task::spawn(actor.run());
-        Ok(Self { sender })
-    }
-}

meilisearch-http/src/index_controller/index_actor/message.rs

@@ -1,74 +0,0 @@
-use std::path::PathBuf;
-
-use tokio::sync::oneshot;
-use uuid::Uuid;
-
-use super::error::Result as IndexResult;
-use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings};
-use crate::index_controller::{Failed, IndexStats, Processed, Processing};
-
-use super::{IndexMeta, IndexSettings};
-
-#[allow(clippy::large_enum_variant)]
-pub enum IndexMsg {
-    CreateIndex {
-        uuid: Uuid,
-        primary_key: Option<String>,
-        ret: oneshot::Sender<IndexResult<IndexMeta>>,
-    },
-    Update {
-        uuid: Uuid,
-        meta: Processing,
-        data: Option<std::fs::File>,
-        ret: oneshot::Sender<IndexResult<Result<Processed, Failed>>>,
-    },
-    Search {
-        uuid: Uuid,
-        query: SearchQuery,
-        ret: oneshot::Sender<IndexResult<SearchResult>>,
-    },
-    Settings {
-        uuid: Uuid,
-        ret: oneshot::Sender<IndexResult<Settings<Checked>>>,
-    },
-    Documents {
-        uuid: Uuid,
-        attributes_to_retrieve: Option<Vec<String>>,
-        offset: usize,
-        limit: usize,
-        ret: oneshot::Sender<IndexResult<Vec<Document>>>,
-    },
-    Document {
-        uuid: Uuid,
-        attributes_to_retrieve: Option<Vec<String>>,
-        doc_id: String,
-        ret: oneshot::Sender<IndexResult<Document>>,
-    },
-    Delete {
-        uuid: Uuid,
-        ret: oneshot::Sender<IndexResult<()>>,
-    },
-    GetMeta {
-        uuid: Uuid,
-        ret: oneshot::Sender<IndexResult<IndexMeta>>,
-    },
-    UpdateIndex {
-        uuid: Uuid,
-        index_settings: IndexSettings,
-        ret: oneshot::Sender<IndexResult<IndexMeta>>,
-    },
-    Snapshot {
-        uuid: Uuid,
-        path: PathBuf,
-        ret: oneshot::Sender<IndexResult<()>>,
-    },
-    Dump {
-        uuid: Uuid,
-        path: PathBuf,
-        ret: oneshot::Sender<IndexResult<()>>,
-    },
-    GetStats {
-        uuid: Uuid,
-        ret: oneshot::Sender<IndexResult<IndexStats>>,
-    },
-}

meilisearch-http/src/index_controller/index_actor/mod.rs

@@ -1,169 +0,0 @@
-use std::fs::File;
-use std::path::PathBuf;
-
-use chrono::{DateTime, Utc};
-#[cfg(test)]
-use mockall::automock;
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-
-use actor::IndexActor;
-pub use actor::CONCURRENT_INDEX_MSG;
-pub use handle_impl::IndexActorHandleImpl;
-use message::IndexMsg;
-use store::{IndexStore, MapIndexStore};
-
-use crate::index::{Checked, Document, Index, SearchQuery, SearchResult, Settings};
-use crate::index_controller::{Failed, IndexStats, Processed, Processing};
-use error::Result;
-
-use super::IndexSettings;
-
-mod actor;
-pub mod error;
-mod handle_impl;
-mod message;
-mod store;
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct IndexMeta {
-    created_at: DateTime<Utc>,
-    pub updated_at: DateTime<Utc>,
-    pub primary_key: Option<String>,
-}
-
-impl IndexMeta {
-    fn new(index: &Index) -> Result<Self> {
-        let txn = index.read_txn()?;
-        Self::new_txn(index, &txn)
-    }
-
-    fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result<Self> {
-        let created_at = index.created_at(txn)?;
-        let updated_at = index.updated_at(txn)?;
-        let primary_key = index.primary_key(txn)?.map(String::from);
-        Ok(Self {
-            created_at,
-            updated_at,
-            primary_key,
-        })
-    }
-}
-
-#[async_trait::async_trait]
-#[cfg_attr(test, automock)]
-pub trait IndexActorHandle {
-    async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta>;
-    async fn update(
-        &self,
-        uuid: Uuid,
-        meta: Processing,
-        data: Option<File>,
-    ) -> Result<std::result::Result<Processed, Failed>>;
-    async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result<SearchResult>;
-    async fn settings(&self, uuid: Uuid) -> Result<Settings<Checked>>;
-
-    async fn documents(
-        &self,
-        uuid: Uuid,
-        offset: usize,
-        limit: usize,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Vec<Document>>;
-    async fn document(
-        &self,
-        uuid: Uuid,
-        doc_id: String,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Document>;
-    async fn delete(&self, uuid: Uuid) -> Result<()>;
-    async fn get_index_meta(&self, uuid: Uuid) -> Result<IndexMeta>;
-    async fn update_index(&self, uuid: Uuid, index_settings: IndexSettings) -> Result<IndexMeta>;
-    async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()>;
-    async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()>;
-    async fn get_index_stats(&self, uuid: Uuid) -> Result<IndexStats>;
-}
-
-#[cfg(test)]
-mod test {
-    use std::sync::Arc;
-
-    use super::*;
-
-    #[async_trait::async_trait]
-    /// Useful for passing around an `Arc<MockIndexActorHandle>` in tests.
-    impl IndexActorHandle for Arc<MockIndexActorHandle> {
-        async fn create_index(&self, uuid: Uuid, primary_key: Option<String>) -> Result<IndexMeta> {
-            self.as_ref().create_index(uuid, primary_key).await
-        }
-
-        async fn update(
-            &self,
-            uuid: Uuid,
-            meta: Processing,
-            data: Option<std::fs::File>,
-        ) -> Result<std::result::Result<Processed, Failed>> {
-            self.as_ref().update(uuid, meta, data).await
-        }
-
-        async fn search(&self, uuid: Uuid, query: SearchQuery) -> Result<SearchResult> {
-            self.as_ref().search(uuid, query).await
-        }
-
-        async fn settings(&self, uuid: Uuid) -> Result<Settings<Checked>> {
-            self.as_ref().settings(uuid).await
-        }
-
-        async fn documents(
-            &self,
-            uuid: Uuid,
-            offset: usize,
-            limit: usize,
-            attributes_to_retrieve: Option<Vec<String>>,
-        ) -> Result<Vec<Document>> {
-            self.as_ref()
-                .documents(uuid, offset, limit, attributes_to_retrieve)
-                .await
-        }
-
-        async fn document(
-            &self,
-            uuid: Uuid,
-            doc_id: String,
-            attributes_to_retrieve: Option<Vec<String>>,
-        ) -> Result<Document> {
-            self.as_ref()
-                .document(uuid, doc_id, attributes_to_retrieve)
-                .await
-        }
-
-        async fn delete(&self, uuid: Uuid) -> Result<()> {
-            self.as_ref().delete(uuid).await
-        }
-
-        async fn get_index_meta(&self, uuid: Uuid) -> Result<IndexMeta> {
-            self.as_ref().get_index_meta(uuid).await
-        }
-
-        async fn update_index(
-            &self,
-            uuid: Uuid,
-            index_settings: IndexSettings,
-        ) -> Result<IndexMeta> {
-            self.as_ref().update_index(uuid, index_settings).await
-        }
-
-        async fn snapshot(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
-            self.as_ref().snapshot(uuid, path).await
-        }
-
-        async fn dump(&self, uuid: Uuid, path: PathBuf) -> Result<()> {
-            self.as_ref().dump(uuid, path).await
-        }
-
-        async fn get_index_stats(&self, uuid: Uuid) -> Result<IndexStats> {
-            self.as_ref().get_index_stats(uuid).await
-        }
-    }
-}

meilisearch-http/src/index_controller/mod.rs

@@ -1,456 +0,0 @@
-use std::collections::BTreeMap;
-use std::path::Path;
-use std::sync::Arc;
-use std::time::Duration;
-
-use actix_web::web::Bytes;
-use chrono::{DateTime, Utc};
-use futures::stream::StreamExt;
-use log::error;
-use log::info;
-use milli::FieldDistribution;
-use serde::{Deserialize, Serialize};
-use tokio::sync::mpsc;
-use tokio::time::sleep;
-use uuid::Uuid;
-
-use dump_actor::DumpActorHandle;
-pub use dump_actor::{DumpInfo, DumpStatus};
-use index_actor::IndexActorHandle;
-use snapshot::{load_snapshot, SnapshotService};
-use update_actor::UpdateActorHandle;
-pub use updates::*;
-use uuid_resolver::{error::UuidResolverError, UuidResolverHandle};
-
-use crate::extractors::payload::Payload;
-use crate::index::{Checked, Document, SearchQuery, SearchResult, Settings};
-use crate::option::Opt;
-use error::Result;
-
-use self::dump_actor::load_dump;
-use self::error::IndexControllerError;
-
-mod dump_actor;
-pub mod error;
-pub mod index_actor;
-mod snapshot;
-mod update_actor;
-mod updates;
-mod uuid_resolver;
-
-#[derive(Debug, Serialize, Deserialize, Clone)]
-#[serde(rename_all = "camelCase")]
-pub struct IndexMetadata {
-    #[serde(skip)]
-    pub uuid: Uuid,
-    pub uid: String,
-    name: String,
-    #[serde(flatten)]
-    pub meta: index_actor::IndexMeta,
-}
-
-#[derive(Clone, Debug)]
-pub struct IndexSettings {
-    pub uid: Option<String>,
-    pub primary_key: Option<String>,
-}
-
-#[derive(Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct IndexStats {
-    #[serde(skip)]
-    pub size: u64,
-    pub number_of_documents: u64,
-    /// Whether the current index is performing an update. It is initially `None` when the
-    /// index returns it, since it is the `UpdateStore` that knows what index is currently indexing. It is
-    /// later set to either true or false, we we retrieve the information from the `UpdateStore`
-    pub is_indexing: Option<bool>,
-    pub field_distribution: FieldDistribution,
-}
-
-#[derive(Clone)]
-pub struct IndexController {
-    uuid_resolver: uuid_resolver::UuidResolverHandleImpl,
-    index_handle: index_actor::IndexActorHandleImpl,
-    update_handle: update_actor::UpdateActorHandleImpl<Bytes>,
-    dump_handle: dump_actor::DumpActorHandleImpl,
-}
-
-#[derive(Serialize, Debug)]
-#[serde(rename_all = "camelCase")]
-pub struct Stats {
-    pub database_size: u64,
-    pub last_update: Option<DateTime<Utc>>,
-    pub indexes: BTreeMap<String, IndexStats>,
-}
-
-impl IndexController {
-    pub fn new(path: impl AsRef<Path>, options: &Opt) -> anyhow::Result<Self> {
-        let index_size = options.max_index_size.get_bytes() as usize;
-        let update_store_size = options.max_index_size.get_bytes() as usize;
-
-        if let Some(ref path) = options.import_snapshot {
-            info!("Loading from snapshot {:?}", path);
-            load_snapshot(
-                &options.db_path,
-                path,
-                options.ignore_snapshot_if_db_exists,
-                options.ignore_missing_snapshot,
-            )?;
-        } else if let Some(ref src_path) = options.import_dump {
-            load_dump(
-                &options.db_path,
-                src_path,
-                options.max_index_size.get_bytes() as usize,
-                options.max_udb_size.get_bytes() as usize,
-                &options.indexer_options,
-            )?;
-        }
-
-        std::fs::create_dir_all(&path)?;
-
-        let uuid_resolver = uuid_resolver::UuidResolverHandleImpl::new(&path)?;
-        let index_handle =
-            index_actor::IndexActorHandleImpl::new(&path, index_size, &options.indexer_options)?;
-        let update_handle = update_actor::UpdateActorHandleImpl::new(
-            index_handle.clone(),
-            &path,
-            update_store_size,
-        )?;
-        let dump_handle = dump_actor::DumpActorHandleImpl::new(
-            &options.dumps_dir,
-            uuid_resolver.clone(),
-            update_handle.clone(),
-            options.max_index_size.get_bytes() as usize,
-            options.max_udb_size.get_bytes() as usize,
-        )?;
-
-        if options.schedule_snapshot {
-            let snapshot_service = SnapshotService::new(
-                uuid_resolver.clone(),
-                update_handle.clone(),
-                Duration::from_secs(options.snapshot_interval_sec),
-                options.snapshot_dir.clone(),
-                options
-                    .db_path
-                    .file_name()
-                    .map(|n| n.to_owned().into_string().expect("invalid path"))
-                    .unwrap_or_else(|| String::from("data.ms")),
-            );
-
-            tokio::task::spawn(snapshot_service.run());
-        }
-
-        Ok(Self {
-            uuid_resolver,
-            index_handle,
-            update_handle,
-            dump_handle,
-        })
-    }
-
-    pub async fn add_documents(
-        &self,
-        uid: String,
-        method: milli::update::IndexDocumentsMethod,
-        format: milli::update::UpdateFormat,
-        payload: Payload,
-        primary_key: Option<String>,
-    ) -> Result<UpdateStatus> {
-        let perform_update = |uuid| async move {
-            let meta = UpdateMeta::DocumentsAddition {
-                method,
-                format,
-                primary_key,
-            };
-            let (sender, receiver) = mpsc::channel(10);
-
-            // It is necessary to spawn a local task to send the payload to the update handle to
-            // prevent dead_locking between the update_handle::update that waits for the update to be
-            // registered and the update_actor that waits for the the payload to be sent to it.
-            tokio::task::spawn_local(async move {
-                payload
-                    .for_each(|r| async {
-                        let _ = sender.send(r).await;
-                    })
-                    .await
-            });
-
-            // This must be done *AFTER* spawning the task.
-            self.update_handle.update(meta, receiver, uuid).await
-        };
-
-        match self.uuid_resolver.get(uid).await {
-            Ok(uuid) => Ok(perform_update(uuid).await?),
-            Err(UuidResolverError::UnexistingIndex(name)) => {
-                let uuid = Uuid::new_v4();
-                let status = perform_update(uuid).await?;
-                // ignore if index creation fails now, since it may already have been created
-                let _ = self.index_handle.create_index(uuid, None).await;
-                self.uuid_resolver.insert(name, uuid).await?;
-                Ok(status)
-            }
-            Err(e) => Err(e.into()),
-        }
-    }
-
-    pub async fn clear_documents(&self, uid: String) -> Result<UpdateStatus> {
-        let uuid = self.uuid_resolver.get(uid).await?;
-        let meta = UpdateMeta::ClearDocuments;
-        let (_, receiver) = mpsc::channel(1);
-        let status = self.update_handle.update(meta, receiver, uuid).await?;
-        Ok(status)
-    }
-
-    pub async fn delete_documents(
-        &self,
-        uid: String,
-        documents: Vec<String>,
-    ) -> Result<UpdateStatus> {
-        let uuid = self.uuid_resolver.get(uid).await?;
-        let meta = UpdateMeta::DeleteDocuments { ids: documents };
-        let (_, receiver) = mpsc::channel(1);
-        let status = self.update_handle.update(meta, receiver, uuid).await?;
-        Ok(status)
-    }
-
-    pub async fn update_settings(
-        &self,
-        uid: String,
-        settings: Settings<Checked>,
-        create: bool,
-    ) -> Result<UpdateStatus> {
-        let perform_udpate = |uuid| async move {
-            let meta = UpdateMeta::Settings(settings.into_unchecked());
-            // Nothing so send, drop the sender right away, as not to block the update actor.
-            let (_, receiver) = mpsc::channel(1);
-            self.update_handle.update(meta, receiver, uuid).await
-        };
-
-        match self.uuid_resolver.get(uid).await {
-            Ok(uuid) => Ok(perform_udpate(uuid).await?),
-            Err(UuidResolverError::UnexistingIndex(name)) if create => {
-                let uuid = Uuid::new_v4();
-                let status = perform_udpate(uuid).await?;
-                // ignore if index creation fails now, since it may already have been created
-                let _ = self.index_handle.create_index(uuid, None).await;
-                self.uuid_resolver.insert(name, uuid).await?;
-                Ok(status)
-            }
-            Err(e) => Err(e.into()),
-        }
-    }
-
-    pub async fn create_index(&self, index_settings: IndexSettings) -> Result<IndexMetadata> {
-        let IndexSettings { uid, primary_key } = index_settings;
-        let uid = uid.ok_or(IndexControllerError::MissingUid)?;
-        let uuid = Uuid::new_v4();
-        let meta = self.index_handle.create_index(uuid, primary_key).await?;
-        self.uuid_resolver.insert(uid.clone(), uuid).await?;
-        let meta = IndexMetadata {
-            uuid,
-            name: uid.clone(),
-            uid,
-            meta,
-        };
-
-        Ok(meta)
-    }
-
-    pub async fn delete_index(&self, uid: String) -> Result<()> {
-        let uuid = self.uuid_resolver.delete(uid).await?;
-
-        // We remove the index from the resolver synchronously, and effectively perform the index
-        // deletion as a background task.
-        let update_handle = self.update_handle.clone();
-        let index_handle = self.index_handle.clone();
-        tokio::spawn(async move {
-            if let Err(e) = update_handle.delete(uuid).await {
-                error!("Error while deleting index: {}", e);
-            }
-            if let Err(e) = index_handle.delete(uuid).await {
-                error!("Error while deleting index: {}", e);
-            }
-        });
-
-        Ok(())
-    }
-
-    pub async fn update_status(&self, uid: String, id: u64) -> Result<UpdateStatus> {
-        let uuid = self.uuid_resolver.get(uid).await?;
-        let result = self.update_handle.update_status(uuid, id).await?;
-        Ok(result)
-    }
-
-    pub async fn all_update_status(&self, uid: String) -> Result<Vec<UpdateStatus>> {
-        let uuid = self.uuid_resolver.get(uid).await?;
-        let result = self.update_handle.get_all_updates_status(uuid).await?;
-        Ok(result)
-    }
-
-    pub async fn list_indexes(&self) -> Result<Vec<IndexMetadata>> {
-        let uuids = self.uuid_resolver.list().await?;
-
-        let mut ret = Vec::new();
-
-        for (uid, uuid) in uuids {
-            let meta = self.index_handle.get_index_meta(uuid).await?;
-            let meta = IndexMetadata {
-                uuid,
-                name: uid.clone(),
-                uid,
-                meta,
-            };
-            ret.push(meta);
-        }
-
-        Ok(ret)
-    }
-
-    pub async fn settings(&self, uid: String) -> Result<Settings<Checked>> {
-        let uuid = self.uuid_resolver.get(uid.clone()).await?;
-        let settings = self.index_handle.settings(uuid).await?;
-        Ok(settings)
-    }
-
-    pub async fn documents(
-        &self,
-        uid: String,
-        offset: usize,
-        limit: usize,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Vec<Document>> {
-        let uuid = self.uuid_resolver.get(uid.clone()).await?;
-        let documents = self
-            .index_handle
-            .documents(uuid, offset, limit, attributes_to_retrieve)
-            .await?;
-        Ok(documents)
-    }
-
-    pub async fn document(
-        &self,
-        uid: String,
-        doc_id: String,
-        attributes_to_retrieve: Option<Vec<String>>,
-    ) -> Result<Document> {
-        let uuid = self.uuid_resolver.get(uid.clone()).await?;
-        let document = self
-            .index_handle
-            .document(uuid, doc_id, attributes_to_retrieve)
-            .await?;
-        Ok(document)
-    }
-
-    pub async fn update_index(
-        &self,
-        uid: String,
-        mut index_settings: IndexSettings,
-    ) -> Result<IndexMetadata> {
-        if index_settings.uid.is_some() {
-            index_settings.uid.take();
-        }
-
-        let uuid = self.uuid_resolver.get(uid.clone()).await?;
-        let meta = self.index_handle.update_index(uuid, index_settings).await?;
-        let meta = IndexMetadata {
-            uuid,
-            name: uid.clone(),
-            uid,
-            meta,
-        };
-        Ok(meta)
-    }
-
-    pub async fn search(&self, uid: String, query: SearchQuery) -> Result<SearchResult> {
-        let uuid = self.uuid_resolver.get(uid).await?;
-        let result = self.index_handle.search(uuid, query).await?;
-        Ok(result)
-    }
-
-    pub async fn get_index(&self, uid: String) -> Result<IndexMetadata> {
-        let uuid = self.uuid_resolver.get(uid.clone()).await?;
-        let meta = self.index_handle.get_index_meta(uuid).await?;
-        let meta = IndexMetadata {
-            uuid,
-            name: uid.clone(),
-            uid,
-            meta,
-        };
-        Ok(meta)
-    }
-
-    pub async fn get_uuids_size(&self) -> Result<u64> {
-        Ok(self.uuid_resolver.get_size().await?)
-    }
-
-    pub async fn get_index_stats(&self, uid: String) -> Result<IndexStats> {
-        let uuid = self.uuid_resolver.get(uid).await?;
-        let update_infos = self.update_handle.get_info().await?;
-        let mut stats = self.index_handle.get_index_stats(uuid).await?;
-        // Check if the currently indexing update is from out index.
-        stats.is_indexing = Some(Some(uuid) == update_infos.processing);
-        Ok(stats)
-    }
-
-    pub async fn get_all_stats(&self) -> Result<Stats> {
-        let update_infos = self.update_handle.get_info().await?;
-        let mut database_size = self.get_uuids_size().await? + update_infos.size;
-        let mut last_update: Option<DateTime<_>> = None;
-        let mut indexes = BTreeMap::new();
-
-        for index in self.list_indexes().await? {
-            let mut index_stats = self.index_handle.get_index_stats(index.uuid).await?;
-            database_size += index_stats.size;
-
-            last_update = last_update.map_or(Some(index.meta.updated_at), |last| {
-                Some(last.max(index.meta.updated_at))
-            });
-
-            index_stats.is_indexing = Some(Some(index.uuid) == update_infos.processing);
-
-            indexes.insert(index.uid, index_stats);
-        }
-
-        Ok(Stats {
-            database_size,
-            last_update,
-            indexes,
-        })
-    }
-
-    pub async fn create_dump(&self) -> Result<DumpInfo> {
-        Ok(self.dump_handle.create_dump().await?)
-    }
-
-    pub async fn dump_info(&self, uid: String) -> Result<DumpInfo> {
-        Ok(self.dump_handle.dump_info(uid).await?)
-    }
-}
-
-pub async fn get_arc_ownership_blocking<T>(mut item: Arc<T>) -> T {
-    loop {
-        match Arc::try_unwrap(item) {
-            Ok(item) => return item,
-            Err(item_arc) => {
-                item = item_arc;
-                sleep(Duration::from_millis(100)).await;
-                continue;
-            }
-        }
-    }
-}
-
-/// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name.
-pub fn asc_ranking_rule(text: &str) -> Option<&str> {
-    text.split_once("asc(")
-        .and_then(|(_, tail)| tail.rsplit_once(")"))
-        .map(|(field, _)| field)
-}
-
-/// Parses the v1 version of the Desc ranking rules `asc(price)`and returns the field name.
-pub fn desc_ranking_rule(text: &str) -> Option<&str> {
-    text.split_once("desc(")
-        .and_then(|(_, tail)| tail.rsplit_once(")"))
-        .map(|(field, _)| field)
-}

meilisearch-http/src/index_controller/snapshot.rs

@@ -1,268 +0,0 @@
-use std::path::{Path, PathBuf};
-use std::time::Duration;
-
-use anyhow::bail;
-use log::{error, info, trace};
-use tokio::fs;
-use tokio::task::spawn_blocking;
-use tokio::time::sleep;
-
-use super::update_actor::UpdateActorHandle;
-use super::uuid_resolver::UuidResolverHandle;
-use crate::helpers::compression;
-
-pub struct SnapshotService<U, R> {
-    uuid_resolver_handle: R,
-    update_handle: U,
-    snapshot_period: Duration,
-    snapshot_path: PathBuf,
-    db_name: String,
-}
-
-impl<U, R> SnapshotService<U, R>
-where
-    U: UpdateActorHandle,
-    R: UuidResolverHandle,
-{
-    pub fn new(
-        uuid_resolver_handle: R,
-        update_handle: U,
-        snapshot_period: Duration,
-        snapshot_path: PathBuf,
-        db_name: String,
-    ) -> Self {
-        Self {
-            uuid_resolver_handle,
-            update_handle,
-            snapshot_period,
-            snapshot_path,
-            db_name,
-        }
-    }
-
-    pub async fn run(self) {
-        info!(
-            "Snapshot scheduled every {}s.",
-            self.snapshot_period.as_secs()
-        );
-        loop {
-            if let Err(e) = self.perform_snapshot().await {
-                error!("Error while performing snapshot: {}", e);
-            }
-            sleep(self.snapshot_period).await;
-        }
-    }
-
-    async fn perform_snapshot(&self) -> anyhow::Result<()> {
-        trace!("Performing snapshot.");
-
-        let snapshot_dir = self.snapshot_path.clone();
-        fs::create_dir_all(&snapshot_dir).await?;
-        let temp_snapshot_dir =
-            spawn_blocking(move || tempfile::tempdir_in(snapshot_dir)).await??;
-        let temp_snapshot_path = temp_snapshot_dir.path().to_owned();
-
-        let uuids = self
-            .uuid_resolver_handle
-            .snapshot(temp_snapshot_path.clone())
-            .await?;
-
-        if uuids.is_empty() {
-            return Ok(());
-        }
-
-        self.update_handle
-            .snapshot(uuids, temp_snapshot_path.clone())
-            .await?;
-        let snapshot_dir = self.snapshot_path.clone();
-        let snapshot_path = self
-            .snapshot_path
-            .join(format!("{}.snapshot", self.db_name));
-        let snapshot_path = spawn_blocking(move || -> anyhow::Result<PathBuf> {
-            let temp_snapshot_file = tempfile::NamedTempFile::new_in(snapshot_dir)?;
-            let temp_snapshot_file_path = temp_snapshot_file.path().to_owned();
-            compression::to_tar_gz(temp_snapshot_path, temp_snapshot_file_path)?;
-            temp_snapshot_file.persist(&snapshot_path)?;
-            Ok(snapshot_path)
-        })
-        .await??;
-
-        trace!("Created snapshot in {:?}.", snapshot_path);
-
-        Ok(())
-    }
-}
-
-pub fn load_snapshot(
-    db_path: impl AsRef<Path>,
-    snapshot_path: impl AsRef<Path>,
-    ignore_snapshot_if_db_exists: bool,
-    ignore_missing_snapshot: bool,
-) -> anyhow::Result<()> {
-    if !db_path.as_ref().exists() && snapshot_path.as_ref().exists() {
-        match compression::from_tar_gz(snapshot_path, &db_path) {
-            Ok(()) => Ok(()),
-            Err(e) => {
-                // clean created db folder
-                std::fs::remove_dir_all(&db_path)?;
-                Err(e)
-            }
-        }
-    } else if db_path.as_ref().exists() && !ignore_snapshot_if_db_exists {
-        bail!(
-            "database already exists at {:?}, try to delete it or rename it",
-            db_path
-                .as_ref()
-                .canonicalize()
-                .unwrap_or_else(|_| db_path.as_ref().to_owned())
-        )
-    } else if !snapshot_path.as_ref().exists() && !ignore_missing_snapshot {
-        bail!(
-            "snapshot doesn't exist at {:?}",
-            snapshot_path
-                .as_ref()
-                .canonicalize()
-                .unwrap_or_else(|_| snapshot_path.as_ref().to_owned())
-        )
-    } else {
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod test {
-    use std::iter::FromIterator;
-    use std::{collections::HashSet, sync::Arc};
-
-    use futures::future::{err, ok};
-    use rand::Rng;
-    use tokio::time::timeout;
-    use uuid::Uuid;
-
-    use super::*;
-    use crate::index_controller::index_actor::MockIndexActorHandle;
-    use crate::index_controller::update_actor::{
-        error::UpdateActorError, MockUpdateActorHandle, UpdateActorHandleImpl,
-    };
-    use crate::index_controller::uuid_resolver::{
-        error::UuidResolverError, MockUuidResolverHandle,
-    };
-
-    #[actix_rt::test]
-    async fn test_normal() {
-        let mut rng = rand::thread_rng();
-        let uuids_num: usize = rng.gen_range(5, 10);
-        let uuids = (0..uuids_num)
-            .map(|_| Uuid::new_v4())
-            .collect::<HashSet<_>>();
-
-        let mut uuid_resolver = MockUuidResolverHandle::new();
-        let uuids_clone = uuids.clone();
-        uuid_resolver
-            .expect_snapshot()
-            .times(1)
-            .returning(move |_| Box::pin(ok(uuids_clone.clone())));
-
-        let uuids_clone = uuids.clone();
-        let mut index_handle = MockIndexActorHandle::new();
-        index_handle
-            .expect_snapshot()
-            .withf(move |uuid, _path| uuids_clone.contains(uuid))
-            .times(uuids_num)
-            .returning(move |_, _| Box::pin(ok(())));
-
-        let dir = tempfile::tempdir_in(".").unwrap();
-        let handle = Arc::new(index_handle);
-        let update_handle =
-            UpdateActorHandleImpl::<Vec<u8>>::new(handle.clone(), dir.path(), 4096 * 100).unwrap();
-
-        let snapshot_path = tempfile::tempdir_in(".").unwrap();
-        let snapshot_service = SnapshotService::new(
-            uuid_resolver,
-            update_handle,
-            Duration::from_millis(100),
-            snapshot_path.path().to_owned(),
-            "data.ms".to_string(),
-        );
-
-        snapshot_service.perform_snapshot().await.unwrap();
-    }
-
-    #[actix_rt::test]
-    async fn error_performing_uuid_snapshot() {
-        let mut uuid_resolver = MockUuidResolverHandle::new();
-        uuid_resolver
-            .expect_snapshot()
-            .times(1)
-            // abitrary error
-            .returning(|_| Box::pin(err(UuidResolverError::NameAlreadyExist)));
-
-        let update_handle = MockUpdateActorHandle::new();
-
-        let snapshot_path = tempfile::tempdir_in(".").unwrap();
-        let snapshot_service = SnapshotService::new(
-            uuid_resolver,
-            update_handle,
-            Duration::from_millis(100),
-            snapshot_path.path().to_owned(),
-            "data.ms".to_string(),
-        );
-
-        assert!(snapshot_service.perform_snapshot().await.is_err());
-        // Nothing was written to the file
-        assert!(!snapshot_path.path().join("data.ms.snapshot").exists());
-    }
-
-    #[actix_rt::test]
-    async fn error_performing_index_snapshot() {
-        let uuid = Uuid::new_v4();
-        let mut uuid_resolver = MockUuidResolverHandle::new();
-        uuid_resolver
-            .expect_snapshot()
-            .times(1)
-            .returning(move |_| Box::pin(ok(HashSet::from_iter(Some(uuid)))));
-
-        let mut update_handle = MockUpdateActorHandle::new();
-        update_handle
-            .expect_snapshot()
-            // abitrary error
-            .returning(|_, _| Box::pin(err(UpdateActorError::UnexistingUpdate(0))));
-
-        let snapshot_path = tempfile::tempdir_in(".").unwrap();
-        let snapshot_service = SnapshotService::new(
-            uuid_resolver,
-            update_handle,
-            Duration::from_millis(100),
-            snapshot_path.path().to_owned(),
-            "data.ms".to_string(),
-        );
-
-        assert!(snapshot_service.perform_snapshot().await.is_err());
-        // Nothing was written to the file
-        assert!(!snapshot_path.path().join("data.ms.snapshot").exists());
-    }
-
-    #[actix_rt::test]
-    async fn test_loop() {
-        let mut uuid_resolver = MockUuidResolverHandle::new();
-        uuid_resolver
-            .expect_snapshot()
-            // we expect the funtion to be called between 2 and 3 time in the given interval.
-            .times(2..4)
-            // abitrary error, to short-circuit the function
-            .returning(move |_| Box::pin(err(UuidResolverError::NameAlreadyExist)));
-
-        let update_handle = MockUpdateActorHandle::new();
-
-        let snapshot_path = tempfile::tempdir_in(".").unwrap();
-        let snapshot_service = SnapshotService::new(
-            uuid_resolver,
-            update_handle,
-            Duration::from_millis(100),
-            snapshot_path.path().to_owned(),
-            "data.ms".to_string(),
-        );
-
-        let _ = timeout(Duration::from_millis(300), snapshot_service.run()).await;
-    }
-}

meilisearch-http/src/index_controller/update_actor/actor.rs

@@ -1,270 +0,0 @@
-use std::collections::HashSet;
-use std::io::SeekFrom;
-use std::path::{Path, PathBuf};
-use std::sync::atomic::AtomicBool;
-use std::sync::Arc;
-
-use async_stream::stream;
-use futures::StreamExt;
-use log::trace;
-use serdeval::*;
-use tokio::fs;
-use tokio::io::AsyncWriteExt;
-use tokio::sync::mpsc;
-use uuid::Uuid;
-
-use super::error::{Result, UpdateActorError};
-use super::{PayloadData, UpdateMsg, UpdateStore, UpdateStoreInfo};
-use crate::index_controller::index_actor::IndexActorHandle;
-use crate::index_controller::{UpdateMeta, UpdateStatus};
-
-pub struct UpdateActor<D, I> {
-    path: PathBuf,
-    store: Arc<UpdateStore>,
-    inbox: Option<mpsc::Receiver<UpdateMsg<D>>>,
-    index_handle: I,
-    must_exit: Arc<AtomicBool>,
-}
-
-impl<D, I> UpdateActor<D, I>
-where
-    D: AsRef<[u8]> + Sized + 'static,
-    I: IndexActorHandle + Clone + Send + Sync + 'static,
-{
-    pub fn new(
-        update_db_size: usize,
-        inbox: mpsc::Receiver<UpdateMsg<D>>,
-        path: impl AsRef<Path>,
-        index_handle: I,
-    ) -> anyhow::Result<Self> {
-        let path = path.as_ref().join("updates");
-
-        std::fs::create_dir_all(&path)?;
-
-        let mut options = heed::EnvOpenOptions::new();
-        options.map_size(update_db_size);
-
-        let must_exit = Arc::new(AtomicBool::new(false));
-
-        let store = UpdateStore::open(options, &path, index_handle.clone(), must_exit.clone())?;
-        std::fs::create_dir_all(path.join("update_files"))?;
-        let inbox = Some(inbox);
-        Ok(Self {
-            path,
-            store,
-            inbox,
-            index_handle,
-            must_exit,
-        })
-    }
-
-    pub async fn run(mut self) {
-        use UpdateMsg::*;
-
-        trace!("Started update actor.");
-
-        let mut inbox = self
-            .inbox
-            .take()
-            .expect("A receiver should be present by now.");
-
-        let must_exit = self.must_exit.clone();
-        let stream = stream! {
-            loop {
-                let msg = inbox.recv().await;
-
-                if must_exit.load(std::sync::atomic::Ordering::Relaxed) {
-                    break;
-                }
-
-                match msg {
-                    Some(msg) => yield msg,
-                    None => break,
-                }
-            }
-        };
-
-        stream
-            .for_each_concurrent(Some(10), |msg| async {
-                match msg {
-                    Update {
-                        uuid,
-                        meta,
-                        data,
-                        ret,
-                    } => {
-                        let _ = ret.send(self.handle_update(uuid, meta, data).await);
-                    }
-                    ListUpdates { uuid, ret } => {
-                        let _ = ret.send(self.handle_list_updates(uuid).await);
-                    }
-                    GetUpdate { uuid, ret, id } => {
-                        let _ = ret.send(self.handle_get_update(uuid, id).await);
-                    }
-                    Delete { uuid, ret } => {
-                        let _ = ret.send(self.handle_delete(uuid).await);
-                    }
-                    Snapshot { uuids, path, ret } => {
-                        let _ = ret.send(self.handle_snapshot(uuids, path).await);
-                    }
-                    GetInfo { ret } => {
-                        let _ = ret.send(self.handle_get_info().await);
-                    }
-                    Dump { uuids, path, ret } => {
-                        let _ = ret.send(self.handle_dump(uuids, path).await);
-                    }
-                }
-            })
-            .await;
-    }
-
-    async fn handle_update(
-        &self,
-        uuid: Uuid,
-        meta: UpdateMeta,
-        payload: mpsc::Receiver<PayloadData<D>>,
-    ) -> Result<UpdateStatus> {
-        let file_path = match meta {
-            UpdateMeta::DocumentsAddition { .. } => {
-                let update_file_id = uuid::Uuid::new_v4();
-                let path = self
-                    .path
-                    .join(format!("update_files/update_{}", update_file_id));
-                let mut file = fs::OpenOptions::new()
-                    .read(true)
-                    .write(true)
-                    .create(true)
-                    .open(&path)
-                    .await?;
-
-                async fn write_to_file<D>(
-                    file: &mut fs::File,
-                    mut payload: mpsc::Receiver<PayloadData<D>>,
-                ) -> Result<usize>
-                where
-                    D: AsRef<[u8]> + Sized + 'static,
-                {
-                    let mut file_len = 0;
-
-                    while let Some(bytes) = payload.recv().await {
-                        let bytes = bytes?;
-                        file_len += bytes.as_ref().len();
-                        file.write_all(bytes.as_ref()).await?;
-                    }
-
-                    file.flush().await?;
-
-                    Ok(file_len)
-                }
-
-                let file_len = write_to_file(&mut file, payload).await;
-
-                match file_len {
-                    Ok(len) if len > 0 => {
-                        let file = file.into_std().await;
-                        Some((file, update_file_id))
-                    }
-                    Err(e) => {
-                        fs::remove_file(&path).await?;
-                        return Err(e);
-                    }
-                    _ => {
-                        fs::remove_file(&path).await?;
-                        None
-                    }
-                }
-            }
-            _ => None,
-        };
-
-        let update_store = self.store.clone();
-
-        tokio::task::spawn_blocking(move || {
-            use std::io::{BufReader, Seek};
-
-            // If the payload is empty, ignore the check.
-            let update_uuid = if let Some((mut file, uuid)) = file_path {
-                // set the file back to the beginning
-                file.seek(SeekFrom::Start(0))?;
-                // Check that the json payload is valid:
-                let reader = BufReader::new(&mut file);
-                // Validate that the payload is in the correct format.
-                let _: Seq<Map<Str, Any>> = serde_json::from_reader(reader)
-                    .map_err(|e| UpdateActorError::InvalidPayload(Box::new(e)))?;
-
-                Some(uuid)
-            } else {
-                None
-            };
-
-            // The payload is valid, we can register it to the update store.
-            let status = update_store
-                .register_update(meta, update_uuid, uuid)
-                .map(UpdateStatus::Enqueued)?;
-            Ok(status)
-        })
-        .await?
-    }
-
-    async fn handle_list_updates(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>> {
-        let update_store = self.store.clone();
-        tokio::task::spawn_blocking(move || {
-            let result = update_store.list(uuid)?;
-            Ok(result)
-        })
-        .await?
-    }
-
-    async fn handle_get_update(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus> {
-        let store = self.store.clone();
-        tokio::task::spawn_blocking(move || {
-            let result = store
-                .meta(uuid, id)?
-                .ok_or(UpdateActorError::UnexistingUpdate(id))?;
-            Ok(result)
-        })
-        .await?
-    }
-
-    async fn handle_delete(&self, uuid: Uuid) -> Result<()> {
-        let store = self.store.clone();
-
-        tokio::task::spawn_blocking(move || store.delete_all(uuid)).await??;
-
-        Ok(())
-    }
-
-    async fn handle_snapshot(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
-        let index_handle = self.index_handle.clone();
-        let update_store = self.store.clone();
-
-        tokio::task::spawn_blocking(move || update_store.snapshot(&uuids, &path, index_handle))
-            .await??;
-
-        Ok(())
-    }
-
-    async fn handle_dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
-        let index_handle = self.index_handle.clone();
-        let update_store = self.store.clone();
-
-        tokio::task::spawn_blocking(move || -> Result<()> {
-            update_store.dump(&uuids, path.to_path_buf(), index_handle)?;
-            Ok(())
-        })
-        .await??;
-
-        Ok(())
-    }
-
-    async fn handle_get_info(&self) -> Result<UpdateStoreInfo> {
-        let update_store = self.store.clone();
-        let info = tokio::task::spawn_blocking(move || -> Result<UpdateStoreInfo> {
-            let info = update_store.get_info()?;
-            Ok(info)
-        })
-        .await??;
-
-        Ok(info)
-    }
-}

meilisearch-http/src/index_controller/update_actor/error.rs

@@ -1,61 +0,0 @@
-use std::error::Error;
-
-use meilisearch_error::{Code, ErrorCode};
-
-use crate::index_controller::index_actor::error::IndexActorError;
-
-pub type Result<T> = std::result::Result<T, UpdateActorError>;
-
-#[derive(Debug, thiserror::Error)]
-#[allow(clippy::large_enum_variant)]
-pub enum UpdateActorError {
-    #[error("Update {0} not found.")]
-    UnexistingUpdate(u64),
-    #[error("Internal error: {0}")]
-    Internal(Box<dyn Error + Send + Sync + 'static>),
-    #[error("{0}")]
-    IndexActor(#[from] IndexActorError),
-    #[error(
-        "update store was shut down due to a fatal error, please check your logs for more info."
-    )]
-    FatalUpdateStoreError,
-    #[error("{0}")]
-    InvalidPayload(Box<dyn Error + Send + Sync + 'static>),
-    #[error("{0}")]
-    PayloadError(#[from] actix_web::error::PayloadError),
-}
-
-impl<T> From<tokio::sync::mpsc::error::SendError<T>> for UpdateActorError {
-    fn from(_: tokio::sync::mpsc::error::SendError<T>) -> Self {
-        Self::FatalUpdateStoreError
-    }
-}
-
-impl From<tokio::sync::oneshot::error::RecvError> for UpdateActorError {
-    fn from(_: tokio::sync::oneshot::error::RecvError) -> Self {
-        Self::FatalUpdateStoreError
-    }
-}
-
-internal_error!(
-    UpdateActorError: heed::Error,
-    std::io::Error,
-    serde_json::Error,
-    tokio::task::JoinError
-);
-
-impl ErrorCode for UpdateActorError {
-    fn error_code(&self) -> Code {
-        match self {
-            UpdateActorError::UnexistingUpdate(_) => Code::NotFound,
-            UpdateActorError::Internal(_) => Code::Internal,
-            UpdateActorError::IndexActor(e) => e.error_code(),
-            UpdateActorError::FatalUpdateStoreError => Code::Internal,
-            UpdateActorError::InvalidPayload(_) => Code::BadRequest,
-            UpdateActorError::PayloadError(error) => match error {
-                actix_http::error::PayloadError::Overflow => Code::PayloadTooLarge,
-                _ => Code::Internal,
-            },
-        }
-    }
-}

meilisearch-http/src/index_controller/update_actor/handle_impl.rs

@@ -1,103 +0,0 @@
-use std::collections::HashSet;
-use std::path::{Path, PathBuf};
-
-use tokio::sync::{mpsc, oneshot};
-use uuid::Uuid;
-
-use crate::index_controller::{IndexActorHandle, UpdateStatus};
-
-use super::error::Result;
-use super::{PayloadData, UpdateActor, UpdateActorHandle, UpdateMeta, UpdateMsg, UpdateStoreInfo};
-
-#[derive(Clone)]
-pub struct UpdateActorHandleImpl<D> {
-    sender: mpsc::Sender<UpdateMsg<D>>,
-}
-
-impl<D> UpdateActorHandleImpl<D>
-where
-    D: AsRef<[u8]> + Sized + 'static + Sync + Send,
-{
-    pub fn new<I>(
-        index_handle: I,
-        path: impl AsRef<Path>,
-        update_store_size: usize,
-    ) -> anyhow::Result<Self>
-    where
-        I: IndexActorHandle + Clone + Send + Sync + 'static,
-    {
-        let path = path.as_ref().to_owned();
-        let (sender, receiver) = mpsc::channel(100);
-        let actor = UpdateActor::new(update_store_size, receiver, path, index_handle)?;
-
-        tokio::task::spawn(actor.run());
-
-        Ok(Self { sender })
-    }
-}
-
-#[async_trait::async_trait]
-impl<D> UpdateActorHandle for UpdateActorHandleImpl<D>
-where
-    D: AsRef<[u8]> + Sized + 'static + Sync + Send,
-{
-    type Data = D;
-
-    async fn get_all_updates_status(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::ListUpdates { uuid, ret };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-    async fn update_status(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::GetUpdate { uuid, id, ret };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-
-    async fn delete(&self, uuid: Uuid) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::Delete { uuid, ret };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-
-    async fn snapshot(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::Snapshot { uuids, path, ret };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-
-    async fn dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::Dump { uuids, path, ret };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-
-    async fn get_info(&self) -> Result<UpdateStoreInfo> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::GetInfo { ret };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-
-    async fn update(
-        &self,
-        meta: UpdateMeta,
-        data: mpsc::Receiver<PayloadData<Self::Data>>,
-        uuid: Uuid,
-    ) -> Result<UpdateStatus> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UpdateMsg::Update {
-            uuid,
-            data,
-            meta,
-            ret,
-        };
-        self.sender.send(msg).await?;
-        receiver.await?
-    }
-}

meilisearch-http/src/index_controller/update_actor/message.rs

@@ -1,43 +0,0 @@
-use std::collections::HashSet;
-use std::path::PathBuf;
-
-use tokio::sync::{mpsc, oneshot};
-use uuid::Uuid;
-
-use super::error::Result;
-use super::{PayloadData, UpdateMeta, UpdateStatus, UpdateStoreInfo};
-
-pub enum UpdateMsg<D> {
-    Update {
-        uuid: Uuid,
-        meta: UpdateMeta,
-        data: mpsc::Receiver<PayloadData<D>>,
-        ret: oneshot::Sender<Result<UpdateStatus>>,
-    },
-    ListUpdates {
-        uuid: Uuid,
-        ret: oneshot::Sender<Result<Vec<UpdateStatus>>>,
-    },
-    GetUpdate {
-        uuid: Uuid,
-        ret: oneshot::Sender<Result<UpdateStatus>>,
-        id: u64,
-    },
-    Delete {
-        uuid: Uuid,
-        ret: oneshot::Sender<Result<()>>,
-    },
-    Snapshot {
-        uuids: HashSet<Uuid>,
-        path: PathBuf,
-        ret: oneshot::Sender<Result<()>>,
-    },
-    Dump {
-        uuids: HashSet<Uuid>,
-        path: PathBuf,
-        ret: oneshot::Sender<Result<()>>,
-    },
-    GetInfo {
-        ret: oneshot::Sender<Result<UpdateStoreInfo>>,
-    },
-}

meilisearch-http/src/index_controller/update_actor/mod.rs

@@ -1,44 +0,0 @@
-use std::{collections::HashSet, path::PathBuf};
-
-use actix_http::error::PayloadError;
-use tokio::sync::mpsc;
-use uuid::Uuid;
-
-use crate::index_controller::{UpdateMeta, UpdateStatus};
-
-use actor::UpdateActor;
-use error::Result;
-use message::UpdateMsg;
-
-pub use handle_impl::UpdateActorHandleImpl;
-pub use store::{UpdateStore, UpdateStoreInfo};
-
-mod actor;
-pub mod error;
-mod handle_impl;
-mod message;
-pub mod store;
-
-type PayloadData<D> = std::result::Result<D, PayloadError>;
-
-#[cfg(test)]
-use mockall::automock;
-
-#[async_trait::async_trait]
-#[cfg_attr(test, automock(type Data=Vec<u8>;))]
-pub trait UpdateActorHandle {
-    type Data: AsRef<[u8]> + Sized + 'static + Sync + Send;
-
-    async fn get_all_updates_status(&self, uuid: Uuid) -> Result<Vec<UpdateStatus>>;
-    async fn update_status(&self, uuid: Uuid, id: u64) -> Result<UpdateStatus>;
-    async fn delete(&self, uuid: Uuid) -> Result<()>;
-    async fn snapshot(&self, uuid: HashSet<Uuid>, path: PathBuf) -> Result<()>;
-    async fn dump(&self, uuids: HashSet<Uuid>, path: PathBuf) -> Result<()>;
-    async fn get_info(&self) -> Result<UpdateStoreInfo>;
-    async fn update(
-        &self,
-        meta: UpdateMeta,
-        data: mpsc::Receiver<PayloadData<Self::Data>>,
-        uuid: Uuid,
-    ) -> Result<UpdateStatus>;
-}

meilisearch-http/src/index_controller/update_actor/store/dump.rs

@@ -1,184 +0,0 @@
-use std::{
-    collections::HashSet,
-    fs::{create_dir_all, File},
-    io::{BufRead, BufReader, Write},
-    path::{Path, PathBuf},
-};
-
-use heed::{EnvOpenOptions, RoTxn};
-use serde::{Deserialize, Serialize};
-use uuid::Uuid;
-
-use super::{Result, State, UpdateStore};
-use crate::index_controller::{
-    index_actor::IndexActorHandle, update_actor::store::update_uuid_to_file_path, Enqueued,
-    UpdateStatus,
-};
-
-#[derive(Serialize, Deserialize)]
-struct UpdateEntry {
-    uuid: Uuid,
-    update: UpdateStatus,
-}
-
-impl UpdateStore {
-    pub fn dump(
-        &self,
-        uuids: &HashSet<Uuid>,
-        path: PathBuf,
-        handle: impl IndexActorHandle,
-    ) -> Result<()> {
-        let state_lock = self.state.write();
-        state_lock.swap(State::Dumping);
-
-        // txn must *always* be acquired after state lock, or it will dead lock.
-        let txn = self.env.write_txn()?;
-
-        let dump_path = path.join("updates");
-        create_dir_all(&dump_path)?;
-
-        self.dump_updates(&txn, uuids, &dump_path)?;
-
-        let fut = dump_indexes(uuids, handle, &path);
-        tokio::runtime::Handle::current().block_on(fut)?;
-
-        state_lock.swap(State::Idle);
-
-        Ok(())
-    }
-
-    fn dump_updates(
-        &self,
-        txn: &RoTxn,
-        uuids: &HashSet<Uuid>,
-        path: impl AsRef<Path>,
-    ) -> Result<()> {
-        let dump_data_path = path.as_ref().join("data.jsonl");
-        let mut dump_data_file = File::create(dump_data_path)?;
-
-        let update_files_path = path.as_ref().join(super::UPDATE_DIR);
-        create_dir_all(&update_files_path)?;
-
-        self.dump_pending(txn, uuids, &mut dump_data_file, &path)?;
-        self.dump_completed(txn, uuids, &mut dump_data_file)?;
-
-        Ok(())
-    }
-
-    fn dump_pending(
-        &self,
-        txn: &RoTxn,
-        uuids: &HashSet<Uuid>,
-        mut file: &mut File,
-        dst_path: impl AsRef<Path>,
-    ) -> Result<()> {
-        let pendings = self.pending_queue.iter(txn)?.lazily_decode_data();
-
-        for pending in pendings {
-            let ((_, uuid, _), data) = pending?;
-            if uuids.contains(&uuid) {
-                let update = data.decode()?;
-
-                if let Some(ref update_uuid) = update.content {
-                    let src = super::update_uuid_to_file_path(&self.path, *update_uuid);
-                    let dst = super::update_uuid_to_file_path(&dst_path, *update_uuid);
-                    std::fs::copy(src, dst)?;
-                }
-
-                let update_json = UpdateEntry {
-                    uuid,
-                    update: update.into(),
-                };
-
-                serde_json::to_writer(&mut file, &update_json)?;
-                file.write_all(b"\n")?;
-            }
-        }
-
-        Ok(())
-    }
-
-    fn dump_completed(
-        &self,
-        txn: &RoTxn,
-        uuids: &HashSet<Uuid>,
-        mut file: &mut File,
-    ) -> Result<()> {
-        let updates = self.updates.iter(txn)?.lazily_decode_data();
-
-        for update in updates {
-            let ((uuid, _), data) = update?;
-            if uuids.contains(&uuid) {
-                let update = data.decode()?;
-
-                let update_json = UpdateEntry { uuid, update };
-
-                serde_json::to_writer(&mut file, &update_json)?;
-                file.write_all(b"\n")?;
-            }
-        }
-
-        Ok(())
-    }
-
-    pub fn load_dump(
-        src: impl AsRef<Path>,
-        dst: impl AsRef<Path>,
-        db_size: usize,
-    ) -> anyhow::Result<()> {
-        let dst_update_path = dst.as_ref().join("updates/");
-        create_dir_all(&dst_update_path)?;
-
-        let mut options = EnvOpenOptions::new();
-        options.map_size(db_size as usize);
-        let (store, _) = UpdateStore::new(options, &dst_update_path)?;
-
-        let src_update_path = src.as_ref().join("updates");
-        let update_data = File::open(&src_update_path.join("data.jsonl"))?;
-        let mut update_data = BufReader::new(update_data);
-
-        std::fs::create_dir_all(dst_update_path.join("update_files/"))?;
-
-        let mut wtxn = store.env.write_txn()?;
-        let mut line = String::new();
-        loop {
-            match update_data.read_line(&mut line) {
-                Ok(0) => break,
-                Ok(_) => {
-                    let UpdateEntry { uuid, update } = serde_json::from_str(&line)?;
-                    store.register_raw_updates(&mut wtxn, &update, uuid)?;
-
-                    // Copy ascociated update path if it exists
-                    if let UpdateStatus::Enqueued(Enqueued {
-                        content: Some(uuid),
-                        ..
-                    }) = update
-                    {
-                        let src = update_uuid_to_file_path(&src_update_path, uuid);
-                        let dst = update_uuid_to_file_path(&dst_update_path, uuid);
-                        std::fs::copy(src, dst)?;
-                    }
-                }
-                _ => break,
-            }
-
-            line.clear();
-        }
-
-        wtxn.commit()?;
-
-        Ok(())
-    }
-}
-
-async fn dump_indexes(
-    uuids: &HashSet<Uuid>,
-    handle: impl IndexActorHandle,
-    path: impl AsRef<Path>,
-) -> Result<()> {
-    for uuid in uuids {
-        handle.dump(*uuid, path.as_ref().to_owned()).await?;
-    }
-
-    Ok(())
-}

meilisearch-http/src/index_controller/uuid_resolver/actor.rs

@@ -1,98 +0,0 @@
-use std::{collections::HashSet, path::PathBuf};
-
-use log::{trace, warn};
-use tokio::sync::mpsc;
-use uuid::Uuid;
-
-use super::{error::UuidResolverError, Result, UuidResolveMsg, UuidStore};
-
-pub struct UuidResolverActor<S> {
-    inbox: mpsc::Receiver<UuidResolveMsg>,
-    store: S,
-}
-
-impl<S: UuidStore> UuidResolverActor<S> {
-    pub fn new(inbox: mpsc::Receiver<UuidResolveMsg>, store: S) -> Self {
-        Self { inbox, store }
-    }
-
-    pub async fn run(mut self) {
-        use UuidResolveMsg::*;
-
-        trace!("uuid resolver started");
-
-        loop {
-            match self.inbox.recv().await {
-                Some(Get { uid: name, ret }) => {
-                    let _ = ret.send(self.handle_get(name).await);
-                }
-                Some(Delete { uid: name, ret }) => {
-                    let _ = ret.send(self.handle_delete(name).await);
-                }
-                Some(List { ret }) => {
-                    let _ = ret.send(self.handle_list().await);
-                }
-                Some(Insert { ret, uuid, name }) => {
-                    let _ = ret.send(self.handle_insert(name, uuid).await);
-                }
-                Some(SnapshotRequest { path, ret }) => {
-                    let _ = ret.send(self.handle_snapshot(path).await);
-                }
-                Some(GetSize { ret }) => {
-                    let _ = ret.send(self.handle_get_size().await);
-                }
-                Some(DumpRequest { path, ret }) => {
-                    let _ = ret.send(self.handle_dump(path).await);
-                }
-                // all senders have been dropped, need to quit.
-                None => break,
-            }
-        }
-
-        warn!("exiting uuid resolver loop");
-    }
-
-    async fn handle_get(&self, uid: String) -> Result<Uuid> {
-        self.store
-            .get_uuid(uid.clone())
-            .await?
-            .ok_or(UuidResolverError::UnexistingIndex(uid))
-    }
-
-    async fn handle_delete(&self, uid: String) -> Result<Uuid> {
-        self.store
-            .delete(uid.clone())
-            .await?
-            .ok_or(UuidResolverError::UnexistingIndex(uid))
-    }
-
-    async fn handle_list(&self) -> Result<Vec<(String, Uuid)>> {
-        let result = self.store.list().await?;
-        Ok(result)
-    }
-
-    async fn handle_snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
-        self.store.snapshot(path).await
-    }
-
-    async fn handle_dump(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
-        self.store.dump(path).await
-    }
-
-    async fn handle_insert(&self, uid: String, uuid: Uuid) -> Result<()> {
-        if !is_index_uid_valid(&uid) {
-            return Err(UuidResolverError::BadlyFormatted(uid));
-        }
-        self.store.insert(uid, uuid).await?;
-        Ok(())
-    }
-
-    async fn handle_get_size(&self) -> Result<u64> {
-        self.store.get_size().await
-    }
-}
-
-fn is_index_uid_valid(uid: &str) -> bool {
-    uid.chars()
-        .all(|x| x.is_ascii_alphanumeric() || x == '-' || x == '_')
-}

meilisearch-http/src/index_controller/uuid_resolver/error.rs

@@ -1,34 +0,0 @@
-use meilisearch_error::{Code, ErrorCode};
-
-pub type Result<T> = std::result::Result<T, UuidResolverError>;
-
-#[derive(Debug, thiserror::Error)]
-pub enum UuidResolverError {
-    #[error("Index already exists.")]
-    NameAlreadyExist,
-    #[error("Index \"{0}\" not found.")]
-    UnexistingIndex(String),
-    #[error("Index must have a valid uid; Index uid can be of type integer or string only composed of alphanumeric characters, hyphens (-) and underscores (_).")]
-    BadlyFormatted(String),
-    #[error("Internal error: {0}")]
-    Internal(Box<dyn std::error::Error + Sync + Send + 'static>),
-}
-
-internal_error!(
-    UuidResolverError: heed::Error,
-    uuid::Error,
-    std::io::Error,
-    tokio::task::JoinError,
-    serde_json::Error
-);
-
-impl ErrorCode for UuidResolverError {
-    fn error_code(&self) -> Code {
-        match self {
-            UuidResolverError::NameAlreadyExist => Code::IndexAlreadyExists,
-            UuidResolverError::UnexistingIndex(_) => Code::IndexNotFound,
-            UuidResolverError::BadlyFormatted(_) => Code::InvalidIndexUid,
-            UuidResolverError::Internal(_) => Code::Internal,
-        }
-    }
-}

meilisearch-http/src/index_controller/uuid_resolver/handle_impl.rs

@@ -1,87 +0,0 @@
-use std::collections::HashSet;
-use std::path::{Path, PathBuf};
-
-use tokio::sync::{mpsc, oneshot};
-use uuid::Uuid;
-
-use super::{HeedUuidStore, Result, UuidResolveMsg, UuidResolverActor, UuidResolverHandle};
-
-#[derive(Clone)]
-pub struct UuidResolverHandleImpl {
-    sender: mpsc::Sender<UuidResolveMsg>,
-}
-
-impl UuidResolverHandleImpl {
-    pub fn new(path: impl AsRef<Path>) -> Result<Self> {
-        let (sender, reveiver) = mpsc::channel(100);
-        let store = HeedUuidStore::new(path)?;
-        let actor = UuidResolverActor::new(reveiver, store);
-        tokio::spawn(actor.run());
-        Ok(Self { sender })
-    }
-}
-
-#[async_trait::async_trait]
-impl UuidResolverHandle for UuidResolverHandleImpl {
-    async fn get(&self, name: String) -> Result<Uuid> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::Get { uid: name, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-
-    async fn delete(&self, name: String) -> Result<Uuid> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::Delete { uid: name, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-
-    async fn list(&self) -> Result<Vec<(String, Uuid)>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::List { ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-
-    async fn insert(&self, name: String, uuid: Uuid) -> Result<()> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::Insert { ret, name, uuid };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-
-    async fn snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::SnapshotRequest { path, ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-
-    async fn get_size(&self) -> Result<u64> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::GetSize { ret };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-    async fn dump(&self, path: PathBuf) -> Result<HashSet<Uuid>> {
-        let (ret, receiver) = oneshot::channel();
-        let msg = UuidResolveMsg::DumpRequest { ret, path };
-        let _ = self.sender.send(msg).await;
-        Ok(receiver
-            .await
-            .expect("Uuid resolver actor has been killed")?)
-    }
-}

meilisearch-http/src/index_controller/uuid_resolver/mod.rs

@@ -1,35 +0,0 @@
-mod actor;
-pub mod error;
-mod handle_impl;
-mod message;
-pub mod store;
-
-use std::collections::HashSet;
-use std::path::PathBuf;
-
-use uuid::Uuid;
-
-use actor::UuidResolverActor;
-use error::Result;
-use message::UuidResolveMsg;
-use store::UuidStore;
-
-#[cfg(test)]
-use mockall::automock;
-
-pub use handle_impl::UuidResolverHandleImpl;
-pub use store::HeedUuidStore;
-
-const UUID_STORE_SIZE: usize = 1_073_741_824; //1GiB
-
-#[async_trait::async_trait]
-#[cfg_attr(test, automock)]
-pub trait UuidResolverHandle {
-    async fn get(&self, name: String) -> Result<Uuid>;
-    async fn insert(&self, name: String, uuid: Uuid) -> Result<()>;
-    async fn delete(&self, name: String) -> Result<Uuid>;
-    async fn list(&self) -> Result<Vec<(String, Uuid)>>;
-    async fn snapshot(&self, path: PathBuf) -> Result<HashSet<Uuid>>;
-    async fn get_size(&self) -> Result<u64>;
-    async fn dump(&self, path: PathBuf) -> Result<HashSet<Uuid>>;
-}

meilisearch-http/src/lib.rs

@@ -1,87 +1,123 @@
-//! # MeiliSearch
-//! Hello there, future contributors. If you are here and see this code, it's probably because you want to add a super new fancy feature in MeiliSearch or fix a bug and first of all, thank you for that!
-//!
-//! To help you in this task, we'll try to do a little overview of the project.
-//! ## Milli
-//! [Milli](https://github.com/meilisearch/milli) is the core library of MeiliSearch. It's where we actually index documents and perform searches. Its purpose is to do these two tasks as fast as possible. You can give an update to milli, and it'll uses as many cores as provided to perform it as fast as possible. Nothing more. You can perform searches at the same time (search only uses one core).
-//! As you can see, we're missing quite a lot of features here; milli does not handle multiples indexes, it can't queue updates, it doesn't provide any web / API frontend, it doesn't implement dumps or snapshots, etc...
-//!
-//! ## `Index` module
-//! The [index] module is what encapsulates one milli index. It abstracts over its transaction and isolates a task that can be run into a thread. This is the unit of interaction with milli.
-//! If you add a feature to milli, you'll probably need to add it in this module too before exposing it to the rest of meilisearch.
-//!
-//! ## `IndexController` module
-//! To handle multiple indexes, we created an [index_controller]. It's in charge of creating new indexes, keeping references to all its indexes, forward asynchronous updates to its indexes, and provide an API to search in its indexes synchronously.
-//! To achieves this goal, we use an [actor model](https://en.wikipedia.org/wiki/Actor_model).
-//!
-//! ### The actor model
-//! Every actor is composed of at least three files:
-//! - `mod.rs` declare and import all the files used by the actor. We also describe the interface (= all the methods) used to interact with the actor. If you are not modifying anything inside of an actor, this is usually all you need to see.
-//! - `handle_impl.rs` implements the interface described in the `mod.rs`; in reality, there is no code logic in this file. Every method is only wrapping its parameters in a structure that is sent to the actor. This is useful for test and futureproofing.
-//! - `message.rs` contains an enum that describes all the interactions you can have with the actor.
-//! - `actor.rs` is used to create and execute the actor. It's where we'll write the loop looking for new messages and actually perform the tasks.
-//!
-//! MeiliSearch currently uses four actors:
-//! - [`uuid_resolver`](index_controller/uuid_resolver/index.html) hold the association between the user-provided indexes name and the internal [`uuid`](https://en.wikipedia.org/wiki/Universally_unique_identifier) representation we use.
-//! - [`index_actor`](index_controller::index_actor) is our representation of multiples indexes. Any request made to MeiliSearch that needs to talk to milli will pass through this actor.
-//! - [`update_actor`](index_controller/update_actor/index.html) is in charge of indexes updates. Since updates can take a long time to receive and  process, we need to:
-//!   1. Store them as fast as possible so we can continue to receive other updates even if nothing has been processed
-//!   2. Feed the `index_actor` with a new update every time it finished its current job.
-//! - [`dump_actor`](index_controller/dump_actor/index.html) this actor handle the  [dumps](https://docs.meilisearch.com/reference/api/dump.html). It needs to contact all the others actors and create a dump of everything that was currently happening.
-//!
-//! ## Data module
-//! The [data] module provide a unified interface to communicate with the index controller and other services (snapshot, dumps, ...), initialize the MeiliSearch instance
-//!
-//! ## HTTP server
-//! To handle the web and API part, we are using [actix-web](https://docs.rs/actix-web/); you can find all routes in the [routes] module.
-//! Currently, the configuration of actix-web is made in the [lib.rs](crate).
-//! Most of the routes use [extractors] to handle the authentication.
-
 #![allow(rustdoc::private_intra_doc_links)]
-pub mod data;
 #[macro_use]
 pub mod error;
 #[macro_use]
 pub mod extractors;
+pub mod analytics;
 pub mod helpers;
-mod index;
-mod index_controller;
 pub mod option;
 pub mod routes;
+use std::sync::Arc;
+use std::time::Duration;
 
-#[cfg(all(not(debug_assertions), feature = "analytics"))]
-pub mod analytics;
-
+use crate::error::MeilisearchHttpError;
 use crate::extractors::authentication::AuthConfig;
-
-pub use self::data::Data;
+use actix_web::error::JsonPayloadError;
+use analytics::Analytics;
+use error::PayloadError;
+use http::header::CONTENT_TYPE;
 pub use option::Opt;
 
-use actix_web::web;
+use actix_web::{web, HttpRequest};
 
 use extractors::authentication::policies::*;
 use extractors::payload::PayloadConfig;
+use meilisearch_lib::MeiliSearch;
+use sha2::Digest;
 
-pub fn configure_data(config: &mut web::ServiceConfig, data: Data) {
-    let http_payload_size_limit = data.http_payload_size_limit();
+#[derive(Clone)]
+pub struct ApiKeys {
+    pub public: Option<String>,
+    pub private: Option<String>,
+    pub master: Option<String>,
+}
+
+impl ApiKeys {
+    pub fn generate_missing_api_keys(&mut self) {
+        if let Some(master_key) = &self.master {
+            if self.private.is_none() {
+                let key = format!("{}-private", master_key);
+                let sha = sha2::Sha256::digest(key.as_bytes());
+                self.private = Some(format!("{:x}", sha));
+            }
+            if self.public.is_none() {
+                let key = format!("{}-public", master_key);
+                let sha = sha2::Sha256::digest(key.as_bytes());
+                self.public = Some(format!("{:x}", sha));
+            }
+        }
+    }
+}
+
+pub fn setup_meilisearch(opt: &Opt) -> anyhow::Result<MeiliSearch> {
+    let mut meilisearch = MeiliSearch::builder();
+    meilisearch
+        .set_max_index_size(opt.max_index_size.get_bytes() as usize)
+        .set_max_update_store_size(opt.max_udb_size.get_bytes() as usize)
+        .set_ignore_missing_snapshot(opt.ignore_missing_snapshot)
+        .set_ignore_snapshot_if_db_exists(opt.ignore_snapshot_if_db_exists)
+        .set_dump_dst(opt.dumps_dir.clone())
+        .set_snapshot_interval(Duration::from_secs(opt.snapshot_interval_sec))
+        .set_snapshot_dir(opt.snapshot_dir.clone());
+
+    if let Some(ref path) = opt.import_snapshot {
+        meilisearch.set_import_snapshot(path.clone());
+    }
+
+    if let Some(ref path) = opt.import_dump {
+        meilisearch.set_dump_src(path.clone());
+    }
+
+    if opt.schedule_snapshot {
+        meilisearch.set_schedule_snapshot();
+    }
+
+    meilisearch.build(opt.db_path.clone(), opt.indexer_options.clone())
+}
+
+pub fn configure_data(
+    config: &mut web::ServiceConfig,
+    data: MeiliSearch,
+    opt: &Opt,
+    analytics: Arc<dyn Analytics>,
+) {
+    let http_payload_size_limit = opt.http_payload_size_limit.get_bytes() as usize;
     config
-        .data(data.clone())
         .app_data(data)
+        .app_data(web::Data::from(analytics))
         .app_data(
             web::JsonConfig::default()
-                .limit(http_payload_size_limit)
-                .content_type(|_mime| true) // Accept all mime types
-                .error_handler(|err, _req| error::payload_error_handler(err).into()),
+                .content_type(|mime| mime == mime::APPLICATION_JSON)
+                .error_handler(|err, req: &HttpRequest| match err {
+                    JsonPayloadError::ContentType => match req.headers().get(CONTENT_TYPE) {
+                        Some(content_type) => MeilisearchHttpError::InvalidContentType(
+                            content_type.to_str().unwrap_or("unknown").to_string(),
+                            vec![mime::APPLICATION_JSON.to_string()],
+                        )
+                        .into(),
+                        None => MeilisearchHttpError::MissingContentType(vec![
+                            mime::APPLICATION_JSON.to_string(),
+                        ])
+                        .into(),
+                    },
+                    err => PayloadError::from(err).into(),
+                }),
         )
         .app_data(PayloadConfig::new(http_payload_size_limit))
         .app_data(
-            web::QueryConfig::default()
-                .error_handler(|err, _req| error::payload_error_handler(err).into()),
+            web::QueryConfig::default().error_handler(|err, _req| PayloadError::from(err).into()),
         );
 }
 
-pub fn configure_auth(config: &mut web::ServiceConfig, data: &Data) {
-    let keys = data.api_keys();
+pub fn configure_auth(config: &mut web::ServiceConfig, opts: &Opt) {
+    let mut keys = ApiKeys {
+        master: opts.master_key.clone(),
+        private: None,
+        public: None,
+    };
+
+    keys.generate_missing_api_keys();
+
     let auth_config = if let Some(ref master_key) = keys.master {
         let private_key = keys.private.as_ref().unwrap();
         let public_key = keys.public.as_ref().unwrap();
@@ -97,7 +133,7 @@ pub fn configure_auth(config: &mut web::ServiceConfig, data: &Data) {
         AuthConfig::NoAuth
     };
 
-    config.app_data(auth_config);
+    config.app_data(auth_config).app_data(keys);
 }
 
 #[cfg(feature = "mini-dashboard")]
@@ -111,7 +147,6 @@ pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) {
 
     if enable_frontend {
         let generated = generated::generate();
-        let mut scope = web::scope("/");
         // Generate routes for mini-dashboard assets
         for (path, resource) in generated.into_iter() {
             let Resource {
@@ -123,12 +158,11 @@ pub fn dashboard(config: &mut web::ServiceConfig, enable_frontend: bool) {
                     web::get().to(move || HttpResponse::Ok().content_type(mime_type).body(data)),
                 ));
             } else {
-                scope = scope.service(web::resource(path).route(
+                config.service(web::resource(path).route(
                     web::get().to(move || HttpResponse::Ok().content_type(mime_type).body(data)),
                 ));
             }
         }
-        config.service(scope);
     } else {
         config.service(web::resource("/").route(web::get().to(routes::running)));
     }
@@ -141,17 +175,18 @@ pub fn dashboard(config: &mut web::ServiceConfig, _enable_frontend: bool) {
 
 #[macro_export]
 macro_rules! create_app {
-    ($data:expr, $enable_frontend:expr) => {{
+    ($data:expr, $enable_frontend:expr, $opt:expr, $analytics:expr) => {{
         use actix_cors::Cors;
         use actix_web::middleware::TrailingSlash;
         use actix_web::App;
         use actix_web::{middleware, web};
+        use meilisearch_http::error::{MeilisearchHttpError, ResponseError};
         use meilisearch_http::routes;
         use meilisearch_http::{configure_auth, configure_data, dashboard};
 
         App::new()
-            .configure(|s| configure_data(s, $data.clone()))
-            .configure(|s| configure_auth(s, &$data))
+            .configure(|s| configure_data(s, $data.clone(), &$opt, $analytics))
+            .configure(|s| configure_auth(s, &$opt))
             .configure(routes::configure)
             .configure(|s| dashboard(s, $enable_frontend))
             .wrap(

meilisearch-http/src/main.rs

@@ -1,21 +1,19 @@
 use std::env;
+use std::sync::Arc;
 
 use actix_web::HttpServer;
-use main_error::MainError;
-use meilisearch_http::{create_app, Data, Opt};
-use structopt::StructOpt;
-
-#[cfg(all(not(debug_assertions), feature = "analytics"))]
 use meilisearch_http::analytics;
+use meilisearch_http::analytics::Analytics;
+use meilisearch_http::{create_app, setup_meilisearch, Opt};
+use meilisearch_lib::MeiliSearch;
+use structopt::StructOpt;
 
 #[cfg(target_os = "linux")]
 #[global_allocator]
-static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc;
-
-#[actix_web::main]
-async fn main() -> Result<(), MainError> {
-    let opt = Opt::from_args();
+static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;
 
+/// does all the setup before meilisearch is launched
+fn setup(opt: &Opt) -> anyhow::Result<()> {
     let mut log_builder = env_logger::Builder::new();
     log_builder.parse_filters(&opt.log_level);
     if opt.log_level == "info" {
@@ -25,40 +23,56 @@ async fn main() -> Result<(), MainError> {
 
     log_builder.init();
 
+    Ok(())
+}
+
+#[actix_web::main]
+async fn main() -> anyhow::Result<()> {
+    let opt = Opt::from_args();
+
+    setup(&opt)?;
+
     match opt.env.as_ref() {
         "production" => {
             if opt.master_key.is_none() {
-                return Err(
+                anyhow::bail!(
                     "In production mode, the environment variable MEILI_MASTER_KEY is mandatory"
-                        .into(),
-                );
+                )
             }
         }
         "development" => (),
         _ => unreachable!(),
     }
 
-    let data = Data::new(opt.clone())?;
+    let meilisearch = setup_meilisearch(&opt)?;
 
     #[cfg(all(not(debug_assertions), feature = "analytics"))]
-    if !opt.no_analytics {
-        let analytics_data = data.clone();
-        let analytics_opt = opt.clone();
-        tokio::task::spawn(analytics::analytics_sender(analytics_data, analytics_opt));
-    }
+    let (analytics, user) = if !opt.no_analytics {
+        analytics::SegmentAnalytics::new(&opt, &meilisearch).await
+    } else {
+        analytics::MockAnalytics::new(&opt)
+    };
+    #[cfg(any(debug_assertions, not(feature = "analytics")))]
+    let (analytics, user) = analytics::MockAnalytics::new(&opt);
 
-    print_launch_resume(&opt, &data);
+    print_launch_resume(&opt, &user);
 
-    run_http(data, opt).await?;
+    run_http(meilisearch, opt, analytics).await?;
 
     Ok(())
 }
 
-async fn run_http(data: Data, opt: Opt) -> Result<(), Box<dyn std::error::Error>> {
+async fn run_http(
+    data: MeiliSearch,
+    opt: Opt,
+    analytics: Arc<dyn Analytics>,
+) -> anyhow::Result<()> {
     let _enable_dashboard = &opt.env == "development";
-    let http_server = HttpServer::new(move || create_app!(data, _enable_dashboard))
-        // Disable signals allows the server to terminate immediately when a user enter CTRL-C
-        .disable_signals();
+    let opt_clone = opt.clone();
+    let http_server =
+        HttpServer::new(move || create_app!(data, _enable_dashboard, opt_clone, analytics.clone()))
+            // Disable signals allows the server to terminate immediately when a user enter CTRL-C
+            .disable_signals();
 
     if let Some(config) = opt.get_ssl_config()? {
         http_server
@@ -66,12 +80,12 @@ async fn run_http(data: Data, opt: Opt) -> Result<(), Box<dyn std::error::Error>
             .run()
             .await?;
     } else {
-        http_server.bind(opt.http_addr)?.run().await?;
+        http_server.bind(&opt.http_addr)?.run().await?;
     }
     Ok(())
 }
 
-pub fn print_launch_resume(opt: &Opt, data: &Data) {
+pub fn print_launch_resume(opt: &Opt, user: &str) {
     let commit_sha = option_env!("VERGEN_GIT_SHA").unwrap_or("unknown");
     let commit_date = option_env!("VERGEN_GIT_COMMIT_TIMESTAMP").unwrap_or("unknown");
 
@@ -109,14 +123,18 @@ Thank you for using MeiliSearch!
 
 We collect anonymized analytics to improve our product and your experience. To learn more, including how to turn off analytics, visit our dedicated documentation page: https://docs.meilisearch.com/learn/what_is_meilisearch/telemetry.html
 
-Anonymous telemetry:   \"Enabled\""
+Anonymous telemetry:\t\"Enabled\""
             );
         }
     }
 
+    if !user.is_empty() {
+        eprintln!("Instance UID:\t\t\"{}\"", user);
+    }
+
     eprintln!();
 
-    if data.api_keys().master.is_some() {
+    if opt.master_key.is_some() {
         eprintln!("A Master Key has been set. Requests to MeiliSearch won't be authorized unless you provide an authentication key.");
     } else {
         eprintln!("No master key found; The server will accept unidentified requests. \

meilisearch-http/src/option.rs

@@ -1,71 +1,16 @@
-use byte_unit::ByteError;
-use std::fmt;
+use std::fs;
 use std::io::{BufReader, Read};
-use std::ops::Deref;
 use std::path::PathBuf;
-use std::str::FromStr;
 use std::sync::Arc;
-use std::{error, fs};
 
 use byte_unit::Byte;
-use milli::CompressionType;
+use meilisearch_lib::options::IndexerOpts;
 use rustls::internal::pemfile::{certs, pkcs8_private_keys, rsa_private_keys};
 use rustls::{
     AllowAnyAnonymousOrAuthenticatedClient, AllowAnyAuthenticatedClient, NoClientAuth,
     RootCertStore,
 };
 use structopt::StructOpt;
-use sysinfo::{RefreshKind, System, SystemExt};
-
-#[derive(Debug, Clone, StructOpt)]
-pub struct IndexerOpts {
-    /// The amount of documents to skip before printing
-    /// a log regarding the indexing advancement.
-    #[structopt(long, default_value = "100000")] // 100k
-    pub log_every_n: usize,
-
-    /// Grenad max number of chunks in bytes.
-    #[structopt(long)]
-    pub max_nb_chunks: Option<usize>,
-
-    /// The maximum amount of memory the indexer will use. It defaults to 2/3
-    /// of the available memory. It is recommended to use something like 80%-90%
-    /// of the available memory, no more.
-    ///
-    /// In case the engine is unable to retrieve the available memory the engine will
-    /// try to use the memory it needs but without real limit, this can lead to
-    /// Out-Of-Memory issues and it is recommended to specify the amount of memory to use.
-    #[structopt(long, default_value)]
-    pub max_memory: MaxMemory,
-
-    /// The name of the compression algorithm to use when compressing intermediate
-    /// Grenad chunks while indexing documents.
-    ///
-    /// Choosing a fast algorithm will make the indexing faster but may consume more memory.
-    #[structopt(long, default_value = "snappy", possible_values = &["snappy", "zlib", "lz4", "lz4hc", "zstd"])]
-    pub chunk_compression_type: CompressionType,
-
-    /// The level of compression of the chosen algorithm.
-    #[structopt(long, requires = "chunk-compression-type")]
-    pub chunk_compression_level: Option<u32>,
-
-    /// Number of parallel jobs for indexing, defaults to # of CPUs.
-    #[structopt(long)]
-    pub indexing_jobs: Option<usize>,
-}
-
-impl Default for IndexerOpts {
-    fn default() -> Self {
-        Self {
-            log_every_n: 100_000,
-            max_nb_chunks: None,
-            max_memory: MaxMemory::default(),
-            chunk_compression_type: CompressionType::None,
-            chunk_compression_level: None,
-            indexing_jobs: None,
-        }
-    }
-}
 
 const POSSIBLE_ENV: [&str; 2] = ["development", "production"];
 
@@ -184,7 +129,7 @@ pub struct Opt {
 }
 
 impl Opt {
-    pub fn get_ssl_config(&self) -> Result<Option<rustls::ServerConfig>, Box<dyn error::Error>> {
+    pub fn get_ssl_config(&self) -> anyhow::Result<Option<rustls::ServerConfig>> {
         if let (Some(cert_path), Some(key_path)) = (&self.ssl_cert_path, &self.ssl_key_path) {
             let client_auth = match &self.ssl_auth_path {
                 Some(auth_path) => {
@@ -210,7 +155,7 @@ impl Opt {
             let ocsp = load_ocsp(&self.ssl_ocsp_path)?;
             config
                 .set_single_cert_with_ocsp_and_sct(certs, privkey, ocsp, vec![])
-                .map_err(|_| "bad certificates/private key")?;
+                .map_err(|_| anyhow::anyhow!("bad certificates/private key"))?;
 
             if self.ssl_resumption {
                 config.set_persistence(rustls::ServerSessionMemoryCache::new(256));
@@ -227,82 +172,31 @@ impl Opt {
     }
 }
 
-/// A type used to detect the max memory available and use 2/3 of it.
-#[derive(Debug, Clone, Copy)]
-pub struct MaxMemory(Option<Byte>);
-
-impl FromStr for MaxMemory {
-    type Err = ByteError;
-
-    fn from_str(s: &str) -> Result<MaxMemory, ByteError> {
-        Byte::from_str(s).map(Some).map(MaxMemory)
-    }
-}
-
-impl Default for MaxMemory {
-    fn default() -> MaxMemory {
-        MaxMemory(
-            total_memory_bytes()
-                .map(|bytes| bytes * 2 / 3)
-                .map(Byte::from_bytes),
-        )
-    }
-}
-
-impl fmt::Display for MaxMemory {
-    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
-        match self.0 {
-            Some(memory) => write!(f, "{}", memory.get_appropriate_unit(true)),
-            None => f.write_str("unknown"),
-        }
-    }
-}
-
-impl Deref for MaxMemory {
-    type Target = Option<Byte>;
-
-    fn deref(&self) -> &Self::Target {
-        &self.0
-    }
-}
-
-impl MaxMemory {
-    pub fn unlimited() -> Self {
-        Self(None)
-    }
-}
-
-/// Returns the total amount of bytes available or `None` if this system isn't supported.
-fn total_memory_bytes() -> Option<u64> {
-    if System::IS_SUPPORTED {
-        let memory_kind = RefreshKind::new().with_memory();
-        let mut system = System::new_with_specifics(memory_kind);
-        system.refresh_memory();
-        Some(system.total_memory() * 1024) // KiB into bytes
-    } else {
-        None
-    }
-}
-
-fn load_certs(filename: PathBuf) -> Result<Vec<rustls::Certificate>, Box<dyn error::Error>> {
-    let certfile = fs::File::open(filename).map_err(|_| "cannot open certificate file")?;
+fn load_certs(filename: PathBuf) -> anyhow::Result<Vec<rustls::Certificate>> {
+    let certfile =
+        fs::File::open(filename).map_err(|_| anyhow::anyhow!("cannot open certificate file"))?;
     let mut reader = BufReader::new(certfile);
-    Ok(certs(&mut reader).map_err(|_| "cannot read certificate file")?)
+    certs(&mut reader).map_err(|_| anyhow::anyhow!("cannot read certificate file"))
 }
 
-fn load_private_key(filename: PathBuf) -> Result<rustls::PrivateKey, Box<dyn error::Error>> {
+fn load_private_key(filename: PathBuf) -> anyhow::Result<rustls::PrivateKey> {
     let rsa_keys = {
-        let keyfile =
-            fs::File::open(filename.clone()).map_err(|_| "cannot open private key file")?;
+        let keyfile = fs::File::open(filename.clone())
+            .map_err(|_| anyhow::anyhow!("cannot open private key file"))?;
         let mut reader = BufReader::new(keyfile);
-        rsa_private_keys(&mut reader).map_err(|_| "file contains invalid rsa private key")?
+        rsa_private_keys(&mut reader)
+            .map_err(|_| anyhow::anyhow!("file contains invalid rsa private key"))?
     };
 
     let pkcs8_keys = {
-        let keyfile = fs::File::open(filename).map_err(|_| "cannot open private key file")?;
+        let keyfile = fs::File::open(filename)
+            .map_err(|_| anyhow::anyhow!("cannot open private key file"))?;
         let mut reader = BufReader::new(keyfile);
-        pkcs8_private_keys(&mut reader)
-            .map_err(|_| "file contains invalid pkcs8 private key (encrypted keys not supported)")?
+        pkcs8_private_keys(&mut reader).map_err(|_| {
+            anyhow::anyhow!(
+                "file contains invalid pkcs8 private key (encrypted keys not supported)"
+            )
+        })?
     };
 
     // prefer to load pkcs8 keys
@@ -314,14 +208,14 @@ fn load_private_key(filename: PathBuf) -> Result<rustls::PrivateKey, Box<dyn err
     }
 }
 
-fn load_ocsp(filename: &Option<PathBuf>) -> Result<Vec<u8>, Box<dyn error::Error>> {
+fn load_ocsp(filename: &Option<PathBuf>) -> anyhow::Result<Vec<u8>> {
     let mut ret = Vec::new();
 
     if let Some(ref name) = filename {
         fs::File::open(name)
-            .map_err(|_| "cannot open ocsp file")?
+            .map_err(|_| anyhow::anyhow!("cannot open ocsp file"))?
             .read_to_end(&mut ret)
-            .map_err(|_| "cannot read oscp file")?;
+            .map_err(|_| anyhow::anyhow!("cannot read oscp file"))?;
     }
 
     Ok(ret)

meilisearch-http/src/routes/dump.rs

@@ -1,18 +1,26 @@
-use actix_web::{web, HttpResponse};
+use actix_web::{web, HttpRequest, HttpResponse};
 use log::debug;
+use meilisearch_lib::MeiliSearch;
 use serde::{Deserialize, Serialize};
+use serde_json::json;
 
+use crate::analytics::Analytics;
 use crate::error::ResponseError;
 use crate::extractors::authentication::{policies::*, GuardedData};
-use crate::Data;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(web::resource("").route(web::post().to(create_dump)))
         .service(web::resource("/{dump_uid}/status").route(web::get().to(get_dump_status)));
 }
 
-pub async fn create_dump(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
-    let res = data.create_dump().await?;
+pub async fn create_dump(
+    meilisearch: GuardedData<Private, MeiliSearch>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
+) -> Result<HttpResponse, ResponseError> {
+    analytics.publish("Dump Created".to_string(), json!({}), Some(&req));
+
+    let res = meilisearch.create_dump().await?;
 
     debug!("returns: {:?}", res);
     Ok(HttpResponse::Accepted().json(res))
@@ -30,10 +38,10 @@ struct DumpParam {
 }
 
 async fn get_dump_status(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
     path: web::Path<DumpParam>,
 ) -> Result<HttpResponse, ResponseError> {
-    let res = data.dump_status(path.dump_uid.clone()).await?;
+    let res = meilisearch.dump_info(path.dump_uid.clone()).await?;
 
     debug!("returns: {:?}", res);
     Ok(HttpResponse::Ok().json(res))

meilisearch-http/src/routes/indexes/documents.rs

@@ -1,51 +1,34 @@
-use actix_web::{web, HttpResponse};
+use actix_web::error::PayloadError;
+use actix_web::web::Bytes;
+use actix_web::{web, HttpRequest, HttpResponse};
+use futures::{Stream, StreamExt};
 use log::debug;
-use milli::update::{IndexDocumentsMethod, UpdateFormat};
+use meilisearch_lib::index_controller::{DocumentAdditionFormat, Update};
+use meilisearch_lib::milli::update::IndexDocumentsMethod;
+use meilisearch_lib::MeiliSearch;
+use once_cell::sync::Lazy;
 use serde::Deserialize;
 use serde_json::Value;
+use tokio::sync::mpsc;
 
-use crate::error::ResponseError;
+use crate::analytics::Analytics;
+use crate::error::{MeilisearchHttpError, ResponseError};
 use crate::extractors::authentication::{policies::*, GuardedData};
 use crate::extractors::payload::Payload;
 use crate::routes::IndexParam;
-use crate::Data;
 
 const DEFAULT_RETRIEVE_DOCUMENTS_OFFSET: usize = 0;
 const DEFAULT_RETRIEVE_DOCUMENTS_LIMIT: usize = 20;
 
-/*
-macro_rules! guard_content_type {
-    ($fn_name:ident, $guard_value:literal) => {
-        fn $fn_name(head: &actix_web::dev::RequestHead) -> bool {
-            if let Some(content_type) = head.headers.get("Content-Type") {
-                content_type
-                    .to_str()
-                    .map(|v| v.contains($guard_value))
-                    .unwrap_or(false)
-            } else {
-                false
-            }
+/// This is required because Payload is not Sync nor Send
+fn payload_to_stream(mut payload: Payload) -> impl Stream<Item = Result<Bytes, PayloadError>> {
+    let (snd, recv) = mpsc::channel(1);
+    tokio::task::spawn_local(async move {
+        while let Some(data) = payload.next().await {
+            let _ = snd.send(data).await;
         }
-    };
-}
-
-guard_content_type!(guard_json, "application/json");
-*/
-
-fn guard_json(head: &actix_web::dev::RequestHead) -> bool {
-    if let Some(_content_type) = head.headers.get("Content-Type") {
-        // CURRENTLY AND FOR THIS RELEASE ONLY WE DECIDED TO INTERPRET ALL CONTENT-TYPES AS JSON
-        true
-        /*
-        content_type
-            .to_str()
-            .map(|v| v.contains("application/json"))
-            .unwrap_or(false)
-        */
-    } else {
-        // if no content-type is specified we still accept the data as json!
-        true
-    }
+    });
+    tokio_stream::wrappers::ReceiverStream::new(recv)
 }
 
 #[derive(Deserialize)]
@@ -58,8 +41,8 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(
         web::resource("")
             .route(web::get().to(get_all_documents))
-            .route(web::post().guard(guard_json).to(add_documents))
-            .route(web::put().guard(guard_json).to(update_documents))
+            .route(web::post().to(add_documents))
+            .route(web::put().to(update_documents))
             .route(web::delete().to(clear_all_documents)),
     )
     // this route needs to be before the /documents/{document_id} to match properly
@@ -72,24 +55,29 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
 }
 
 pub async fn get_document(
-    data: GuardedData<Public, Data>,
+    meilisearch: GuardedData<Public, MeiliSearch>,
     path: web::Path<DocumentParam>,
 ) -> Result<HttpResponse, ResponseError> {
     let index = path.index_uid.clone();
     let id = path.document_id.clone();
-    let document = data
-        .retrieve_document(index, id, None as Option<Vec<String>>)
+    let document = meilisearch
+        .document(index, id, None as Option<Vec<String>>)
         .await?;
     debug!("returns: {:?}", document);
     Ok(HttpResponse::Ok().json(document))
 }
 
 pub async fn delete_document(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
     path: web::Path<DocumentParam>,
 ) -> Result<HttpResponse, ResponseError> {
-    let update_status = data
-        .delete_documents(path.index_uid.clone(), vec![path.document_id.clone()])
+    let DocumentParam {
+        document_id,
+        index_uid,
+    } = path.into_inner();
+    let update = Update::DeleteDocuments(vec![document_id]);
+    let update_status = meilisearch
+        .register_update(index_uid, update, false)
         .await?;
     debug!("returns: {:?}", update_status);
     Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
@@ -104,7 +92,7 @@ pub struct BrowseQuery {
 }
 
 pub async fn get_all_documents(
-    data: GuardedData<Public, Data>,
+    meilisearch: GuardedData<Public, MeiliSearch>,
     path: web::Path<IndexParam>,
     params: web::Query<BrowseQuery>,
 ) -> Result<HttpResponse, ResponseError> {
@@ -120,8 +108,8 @@ pub async fn get_all_documents(
         Some(names)
     });
 
-    let documents = data
-        .retrieve_documents(
+    let documents = meilisearch
+        .documents(
             path.index_uid.clone(),
             params.offset.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_OFFSET),
             params.limit.unwrap_or(DEFAULT_RETRIEVE_DOCUMENTS_LIMIT),
@@ -135,57 +123,122 @@ pub async fn get_all_documents(
 #[derive(Deserialize, Debug)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct UpdateDocumentsQuery {
-    primary_key: Option<String>,
+    pub primary_key: Option<String>,
+}
+
+pub async fn add_documents(
+    meilisearch: GuardedData<Private, MeiliSearch>,
+    path: web::Path<IndexParam>,
+    params: web::Query<UpdateDocumentsQuery>,
+    body: Payload,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
+) -> Result<HttpResponse, ResponseError> {
+    debug!("called with params: {:?}", params);
+    let content_type = req
+        .headers()
+        .get("Content-type")
+        .map(|s| s.to_str().unwrap_or("unkown"));
+    let params = params.into_inner();
+
+    analytics.add_documents(
+        &params,
+        meilisearch.get_index(path.index_uid.clone()).await.is_err(),
+        &req,
+    );
+
+    document_addition(
+        content_type,
+        meilisearch,
+        path.index_uid.clone(),
+        params.primary_key,
+        body,
+        IndexDocumentsMethod::ReplaceDocuments,
+    )
+    .await
+}
+
+pub async fn update_documents(
+    meilisearch: GuardedData<Private, MeiliSearch>,
+    path: web::Path<IndexParam>,
+    params: web::Query<UpdateDocumentsQuery>,
+    body: Payload,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
+) -> Result<HttpResponse, ResponseError> {
+    debug!("called with params: {:?}", params);
+    let content_type = req
+        .headers()
+        .get("Content-type")
+        .map(|s| s.to_str().unwrap_or("unkown"));
+
+    analytics.update_documents(
+        &params,
+        meilisearch.get_index(path.index_uid.clone()).await.is_err(),
+        &req,
+    );
+
+    document_addition(
+        content_type,
+        meilisearch,
+        path.into_inner().index_uid,
+        params.into_inner().primary_key,
+        body,
+        IndexDocumentsMethod::UpdateDocuments,
+    )
+    .await
 }
 
 /// Route used when the payload type is "application/json"
 /// Used to add or replace documents
-pub async fn add_documents(
-    data: GuardedData<Private, Data>,
-    path: web::Path<IndexParam>,
-    params: web::Query<UpdateDocumentsQuery>,
+async fn document_addition(
+    content_type: Option<&str>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
+    index_uid: String,
+    primary_key: Option<String>,
     body: Payload,
+    method: IndexDocumentsMethod,
 ) -> Result<HttpResponse, ResponseError> {
-    debug!("called with params: {:?}", params);
-    let update_status = data
-        .add_documents(
-            path.into_inner().index_uid,
-            IndexDocumentsMethod::ReplaceDocuments,
-            UpdateFormat::Json,
-            body,
-            params.primary_key.clone(),
-        )
-        .await?;
+    static ACCEPTED_CONTENT_TYPE: Lazy<Vec<String>> = Lazy::new(|| {
+        vec![
+            "application/json".to_string(),
+            "application/x-ndjson".to_string(),
+            "text/csv".to_string(),
+        ]
+    });
+    let format = match content_type {
+        Some("application/json") => DocumentAdditionFormat::Json,
+        Some("application/x-ndjson") => DocumentAdditionFormat::Ndjson,
+        Some("text/csv") => DocumentAdditionFormat::Csv,
+        Some(other) => {
+            return Err(MeilisearchHttpError::InvalidContentType(
+                other.to_string(),
+                ACCEPTED_CONTENT_TYPE.clone(),
+            )
+            .into())
+        }
+        None => {
+            return Err(
+                MeilisearchHttpError::MissingContentType(ACCEPTED_CONTENT_TYPE.clone()).into(),
+            )
+        }
+    };
+
+    let update = Update::DocumentAddition {
+        payload: Box::new(payload_to_stream(body)),
+        primary_key,
+        method,
+        format,
+    };
+
+    let update_status = meilisearch.register_update(index_uid, update, true).await?;
 
     debug!("returns: {:?}", update_status);
     Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
 }
 
-/// Route used when the payload type is "application/json"
-/// Used to add or replace documents
-pub async fn update_documents(
-    data: GuardedData<Private, Data>,
-    path: web::Path<IndexParam>,
-    params: web::Query<UpdateDocumentsQuery>,
-    body: Payload,
-) -> Result<HttpResponse, ResponseError> {
-    debug!("called with params: {:?}", params);
-    let update = data
-        .add_documents(
-            path.into_inner().index_uid,
-            IndexDocumentsMethod::UpdateDocuments,
-            UpdateFormat::Json,
-            body,
-            params.primary_key.clone(),
-        )
-        .await?;
-
-    debug!("returns: {:?}", update);
-    Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update.id() })))
-}
-
 pub async fn delete_documents(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
     path: web::Path<IndexParam>,
     body: web::Json<Vec<Value>>,
 ) -> Result<HttpResponse, ResponseError> {
@@ -199,16 +252,22 @@ pub async fn delete_documents(
         })
         .collect();
 
-    let update_status = data.delete_documents(path.index_uid.clone(), ids).await?;
+    let update = Update::DeleteDocuments(ids);
+    let update_status = meilisearch
+        .register_update(path.into_inner().index_uid, update, false)
+        .await?;
     debug!("returns: {:?}", update_status);
     Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
 }
 
 pub async fn clear_all_documents(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
     path: web::Path<IndexParam>,
 ) -> Result<HttpResponse, ResponseError> {
-    let update_status = data.clear_documents(path.index_uid.clone()).await?;
+    let update = Update::ClearDocuments;
+    let update_status = meilisearch
+        .register_update(path.into_inner().index_uid, update, false)
+        .await?;
     debug!("returns: {:?}", update_status);
     Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
 }

meilisearch-http/src/routes/indexes/mod.rs

@@ -1,12 +1,15 @@
-use actix_web::{web, HttpResponse};
+use actix_web::{web, HttpRequest, HttpResponse};
 use chrono::{DateTime, Utc};
 use log::debug;
+use meilisearch_lib::index_controller::IndexSettings;
+use meilisearch_lib::MeiliSearch;
 use serde::{Deserialize, Serialize};
+use serde_json::json;
 
+use crate::analytics::Analytics;
 use crate::error::ResponseError;
 use crate::extractors::authentication::{policies::*, GuardedData};
 use crate::routes::IndexParam;
-use crate::Data;
 
 pub mod documents;
 pub mod search;
@@ -35,7 +38,9 @@ pub fn configure(cfg: &mut web::ServiceConfig) {
     );
 }
 
-pub async fn list_indexes(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
+pub async fn list_indexes(
+    data: GuardedData<Private, MeiliSearch>,
+) -> Result<HttpResponse, ResponseError> {
     let indexes = data.list_indexes().await?;
     debug!("returns: {:?}", indexes);
     Ok(HttpResponse::Ok().json(indexes))
@@ -49,11 +54,19 @@ pub struct IndexCreateRequest {
 }
 
 pub async fn create_index(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
     body: web::Json<IndexCreateRequest>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     let body = body.into_inner();
-    let meta = data.create_index(body.uid, body.primary_key).await?;
+
+    analytics.publish(
+        "Index Created".to_string(),
+        json!({ "primary_key": body.primary_key}),
+        Some(&req),
+    );
+    let meta = meilisearch.create_index(body.uid, body.primary_key).await?;
     Ok(HttpResponse::Created().json(meta))
 }
 
@@ -75,41 +88,52 @@ pub struct UpdateIndexResponse {
 }
 
 pub async fn get_index(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
     path: web::Path<IndexParam>,
 ) -> Result<HttpResponse, ResponseError> {
-    let meta = data.index(path.index_uid.clone()).await?;
+    let meta = meilisearch.get_index(path.index_uid.clone()).await?;
     debug!("returns: {:?}", meta);
     Ok(HttpResponse::Ok().json(meta))
 }
 
 pub async fn update_index(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
     path: web::Path<IndexParam>,
     body: web::Json<UpdateIndexRequest>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!("called with params: {:?}", body);
     let body = body.into_inner();
-    let meta = data
-        .update_index(path.into_inner().index_uid, body.primary_key, body.uid)
+    analytics.publish(
+        "Index Updated".to_string(),
+        json!({ "primary_key": body.primary_key}),
+        Some(&req),
+    );
+    let settings = IndexSettings {
+        uid: body.uid,
+        primary_key: body.primary_key,
+    };
+    let meta = meilisearch
+        .update_index(path.into_inner().index_uid, settings)
         .await?;
     debug!("returns: {:?}", meta);
     Ok(HttpResponse::Ok().json(meta))
 }
 
 pub async fn delete_index(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
     path: web::Path<IndexParam>,
 ) -> Result<HttpResponse, ResponseError> {
-    data.delete_index(path.index_uid.clone()).await?;
+    meilisearch.delete_index(path.index_uid.clone()).await?;
     Ok(HttpResponse::NoContent().finish())
 }
 
 pub async fn get_index_stats(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
     path: web::Path<IndexParam>,
 ) -> Result<HttpResponse, ResponseError> {
-    let response = data.get_index_stats(path.index_uid.clone()).await?;
+    let response = meilisearch.get_index_stats(path.index_uid.clone()).await?;
 
     debug!("returns: {:?}", response);
     Ok(HttpResponse::Ok().json(response))

meilisearch-http/src/routes/indexes/search.rs

@@ -1,13 +1,14 @@
-use actix_web::{web, HttpResponse};
+use actix_web::{web, HttpRequest, HttpResponse};
 use log::debug;
+use meilisearch_lib::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT};
+use meilisearch_lib::MeiliSearch;
 use serde::Deserialize;
 use serde_json::Value;
 
+use crate::analytics::{Analytics, SearchAggregator};
 use crate::error::ResponseError;
 use crate::extractors::authentication::{policies::*, GuardedData};
-use crate::index::{default_crop_length, SearchQuery, DEFAULT_SEARCH_LIMIT};
 use crate::routes::IndexParam;
-use crate::Data;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(
@@ -61,9 +62,7 @@ impl From<SearchQueryGet> for SearchQuery {
             None => None,
         };
 
-        let sort = other
-            .sort
-            .map(|attrs| attrs.split(',').map(String::from).collect());
+        let sort = other.sort.map(|attr| fix_sort_query_parameters(&attr));
 
         Self {
             q: other.q,
@@ -81,14 +80,51 @@ impl From<SearchQueryGet> for SearchQuery {
     }
 }
 
+// TODO: TAMO: split on :asc, and :desc, instead of doing some weird things
+
+/// Transform the sort query parameter into something that matches the post expected format.
+fn fix_sort_query_parameters(sort_query: &str) -> Vec<String> {
+    let mut sort_parameters = Vec::new();
+    let mut merge = false;
+    for current_sort in sort_query.trim_matches('"').split(',').map(|s| s.trim()) {
+        if current_sort.starts_with("_geoPoint(") {
+            sort_parameters.push(current_sort.to_string());
+            merge = true;
+        } else if merge && !sort_parameters.is_empty() {
+            sort_parameters
+                .last_mut()
+                .unwrap()
+                .push_str(&format!(",{}", current_sort));
+            if current_sort.ends_with("):desc") || current_sort.ends_with("):asc") {
+                merge = false;
+            }
+        } else {
+            sort_parameters.push(current_sort.to_string());
+            merge = false;
+        }
+    }
+    sort_parameters
+}
+
 pub async fn search_with_url_query(
-    data: GuardedData<Public, Data>,
+    meilisearch: GuardedData<Public, MeiliSearch>,
     path: web::Path<IndexParam>,
     params: web::Query<SearchQueryGet>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
     debug!("called with params: {:?}", params);
-    let query = params.into_inner().into();
-    let search_result = data.search(path.into_inner().index_uid, query).await?;
+    let query: SearchQuery = params.into_inner().into();
+
+    let mut aggregate = SearchAggregator::from_query(&query, &req);
+
+    let search_result = meilisearch.search(path.into_inner().index_uid, query).await;
+    if let Ok(ref search_result) = search_result {
+        aggregate.succeed(search_result);
+    }
+    analytics.get_search(aggregate);
+
+    let search_result = search_result?;
 
     // Tests that the nb_hits is always set to false
     #[cfg(test)]
@@ -99,14 +135,24 @@ pub async fn search_with_url_query(
 }
 
 pub async fn search_with_post(
-    data: GuardedData<Public, Data>,
+    meilisearch: GuardedData<Public, MeiliSearch>,
     path: web::Path<IndexParam>,
     params: web::Json<SearchQuery>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    debug!("search called with params: {:?}", params);
-    let search_result = data
-        .search(path.into_inner().index_uid, params.into_inner())
-        .await?;
+    let query = params.into_inner();
+    debug!("search called with params: {:?}", query);
+
+    let mut aggregate = SearchAggregator::from_query(&query, &req);
+
+    let search_result = meilisearch.search(path.into_inner().index_uid, query).await;
+    if let Ok(ref search_result) = search_result {
+        aggregate.succeed(search_result);
+    }
+    analytics.post_search(aggregate);
+
+    let search_result = search_result?;
 
     // Tests that the nb_hits is always set to false
     #[cfg(test)]
@@ -115,3 +161,42 @@ pub async fn search_with_post(
     debug!("returns: {:?}", search_result);
     Ok(HttpResponse::Ok().json(search_result))
 }
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    #[test]
+    fn test_fix_sort_query_parameters() {
+        let sort = fix_sort_query_parameters("_geoPoint(12, 13):asc");
+        assert_eq!(sort, vec!["_geoPoint(12,13):asc".to_string()]);
+        let sort = fix_sort_query_parameters("doggo:asc,_geoPoint(12.45,13.56):desc");
+        assert_eq!(
+            sort,
+            vec![
+                "doggo:asc".to_string(),
+                "_geoPoint(12.45,13.56):desc".to_string(),
+            ]
+        );
+        let sort = fix_sort_query_parameters(
+            "doggo:asc , _geoPoint(12.45, 13.56, 2590352):desc , catto:desc",
+        );
+        assert_eq!(
+            sort,
+            vec![
+                "doggo:asc".to_string(),
+                "_geoPoint(12.45,13.56,2590352):desc".to_string(),
+                "catto:desc".to_string(),
+            ]
+        );
+        let sort = fix_sort_query_parameters("doggo:asc , _geoPoint(1, 2), catto:desc");
+        // This is ugly but eh, I don't want to write a full parser just for this unused route
+        assert_eq!(
+            sort,
+            vec![
+                "doggo:asc".to_string(),
+                "_geoPoint(1,2),catto:desc".to_string(),
+            ]
+        );
+    }
+}

meilisearch-http/src/routes/indexes/settings.rs

@@ -1,62 +1,73 @@
-use actix_web::{web, HttpResponse};
 use log::debug;
 
+use actix_web::{web, HttpRequest, HttpResponse};
+use meilisearch_lib::index::{Settings, Unchecked};
+use meilisearch_lib::index_controller::Update;
+use meilisearch_lib::MeiliSearch;
+use serde_json::json;
+
+use crate::analytics::Analytics;
+use crate::error::ResponseError;
 use crate::extractors::authentication::{policies::*, GuardedData};
-use crate::index::Settings;
-use crate::Data;
-use crate::{error::ResponseError, index::Unchecked};
 
 #[macro_export]
 macro_rules! make_setting_route {
-    ($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => {
+    ($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal, $analytics_var:ident, $analytics:expr) => {
         pub mod $attr {
             use log::debug;
-            use actix_web::{web, HttpResponse, Resource};
+            use actix_web::{web, HttpResponse, HttpRequest, Resource};
 
-            use milli::update::Setting;
+            use meilisearch_lib::milli::update::Setting;
+            use meilisearch_lib::{MeiliSearch, index::Settings, index_controller::Update};
 
-            use crate::data;
+            use crate::analytics::Analytics;
             use crate::error::ResponseError;
-            use crate::index::Settings;
             use crate::extractors::authentication::{GuardedData, policies::*};
 
             pub async fn delete(
-                data: GuardedData<Private, data::Data>,
+                meilisearch: GuardedData<Private, MeiliSearch>,
                 index_uid: web::Path<String>,
             ) -> Result<HttpResponse, ResponseError> {
-                use crate::index::Settings;
                 let settings = Settings {
                     $attr: Setting::Reset,
                     ..Default::default()
                 };
-                let update_status = data.update_settings(index_uid.into_inner(), settings, false).await?;
+                let update = Update::Settings(settings);
+                let update_status = meilisearch.register_update(index_uid.into_inner(), update, false).await?;
                 debug!("returns: {:?}", update_status);
                 Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
             }
 
             pub async fn update(
-                data: GuardedData<Private, data::Data>,
+                meilisearch: GuardedData<Private, MeiliSearch>,
                 index_uid: actix_web::web::Path<String>,
                 body: actix_web::web::Json<Option<$type>>,
+                req: HttpRequest,
+                $analytics_var: web::Data< dyn Analytics>,
             ) -> std::result::Result<HttpResponse, ResponseError> {
+                let body = body.into_inner();
+
+                $analytics(&body, &req);
+
                 let settings = Settings {
-                    $attr: match body.into_inner() {
+                    $attr: match body {
                         Some(inner_body) => Setting::Set(inner_body),
                         None => Setting::Reset
                     },
                     ..Default::default()
                 };
 
-                let update_status = data.update_settings(index_uid.into_inner(), settings, true).await?;
+                let update = Update::Settings(settings);
+                let update_status = meilisearch.register_update(index_uid.into_inner(), update, true).await?;
                 debug!("returns: {:?}", update_status);
                 Ok(HttpResponse::Accepted().json(serde_json::json!({ "updateId": update_status.id() })))
             }
 
             pub async fn get(
-                data: GuardedData<Private, data::Data>,
+                meilisearch: GuardedData<Private, MeiliSearch>,
                 index_uid: actix_web::web::Path<String>,
             ) -> std::result::Result<HttpResponse, ResponseError> {
-                let settings = data.settings(index_uid.into_inner()).await?;
+                let settings = meilisearch.settings(index_uid.into_inner()).await?;
                 debug!("returns: {:?}", settings);
                 let mut json = serde_json::json!(&settings);
                 let val = json[$camelcase_attr].take();
@@ -71,20 +82,53 @@ macro_rules! make_setting_route {
             }
         }
     };
+    ($route:literal, $type:ty, $attr:ident, $camelcase_attr:literal) => {
+        make_setting_route!($route, $type, $attr, $camelcase_attr, _analytics, |_, _| {});
+    };
 }
 
 make_setting_route!(
     "/filterable-attributes",
     std::collections::BTreeSet<String>,
     filterable_attributes,
-    "filterableAttributes"
+    "filterableAttributes",
+    analytics,
+    |setting: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "FilterableAttributes Updated".to_string(),
+            json!({
+                "filterable_attributes": {
+                    "total": setting.as_ref().map(|filter| filter.len()).unwrap_or(0),
+                    "has_geo": setting.as_ref().map(|filter| filter.contains("_geo")).unwrap_or(false),
+                }
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
     "/sortable-attributes",
     std::collections::BTreeSet<String>,
     sortable_attributes,
-    "sortableAttributes"
+    "sortableAttributes",
+    analytics,
+    |setting: &Option<std::collections::BTreeSet<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "SortableAttributes Updated".to_string(),
+            json!({
+                "sortable_attributes": {
+                    "total": setting.as_ref().map(|sort| sort.len()).unwrap_or(0),
+                    "has_geo": setting.as_ref().map(|sort| sort.contains("_geo")).unwrap_or(false),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -98,7 +142,21 @@ make_setting_route!(
     "/searchable-attributes",
     Vec<String>,
     searchable_attributes,
-    "searchableAttributes"
+    "searchableAttributes",
+    analytics,
+    |setting: &Option<Vec<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "SearchableAttributes Updated".to_string(),
+            json!({
+                "searchable_attributes": {
+                    "total": setting.as_ref().map(|sort| sort.len()).unwrap_or(0),
+                },
+            }),
+            Some(req),
+        );
+    }
 );
 
 make_setting_route!(
@@ -122,7 +180,26 @@ make_setting_route!(
     "distinctAttribute"
 );
 
-make_setting_route!("/ranking-rules", Vec<String>, ranking_rules, "rankingRules");
+make_setting_route!(
+    "/ranking-rules",
+    Vec<String>,
+    ranking_rules,
+    "rankingRules",
+    analytics,
+    |setting: &Option<Vec<String>>, req: &HttpRequest| {
+        use serde_json::json;
+
+        analytics.publish(
+            "RankingRules Updated".to_string(),
+            json!({
+                "ranking_rules": {
+                    "sort_position": setting.as_ref().map(|sort| sort.iter().position(|s| s == "sort")),
+                }
+            }),
+            Some(req),
+        );
+    }
+);
 
 macro_rules! generate_configure {
     ($($mod:ident),*) => {
@@ -149,13 +226,35 @@ generate_configure!(
 );
 
 pub async fn update_all(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
     index_uid: web::Path<String>,
     body: web::Json<Settings<Unchecked>>,
+    req: HttpRequest,
+    analytics: web::Data<dyn Analytics>,
 ) -> Result<HttpResponse, ResponseError> {
-    let settings = body.into_inner().check();
-    let update_result = data
-        .update_settings(index_uid.into_inner(), settings, true)
+    let settings = body.into_inner();
+
+    analytics.publish(
+        "Settings Updated".to_string(),
+        json!({
+           "ranking_rules": {
+                "sort_position": settings.ranking_rules.as_ref().set().map(|sort| sort.iter().position(|s| s == "sort")),
+            },
+           "sortable_attributes": {
+                "total": settings.sortable_attributes.as_ref().set().map(|sort| sort.len()).unwrap_or(0),
+                "has_geo": settings.sortable_attributes.as_ref().set().map(|sort| sort.iter().any(|s| s == "_geo")).unwrap_or(false),
+            },
+           "filterable_attributes": {
+                "total": settings.filterable_attributes.as_ref().set().map(|filter| filter.len()).unwrap_or(0),
+                "has_geo": settings.filterable_attributes.as_ref().set().map(|filter| filter.iter().any(|s| s == "_geo")).unwrap_or(false),
+            },
+        }),
+        Some(&req),
+    );
+
+    let update = Update::Settings(settings);
+    let update_result = meilisearch
+        .register_update(index_uid.into_inner(), update, true)
         .await?;
     let json = serde_json::json!({ "updateId": update_result.id() });
     debug!("returns: {:?}", json);
@@ -163,7 +262,7 @@ pub async fn update_all(
 }
 
 pub async fn get_all(
-    data: GuardedData<Private, Data>,
+    data: GuardedData<Private, MeiliSearch>,
     index_uid: web::Path<String>,
 ) -> Result<HttpResponse, ResponseError> {
     let settings = data.settings(index_uid.into_inner()).await?;
@@ -172,12 +271,14 @@ pub async fn get_all(
 }
 
 pub async fn delete_all(
-    data: GuardedData<Private, Data>,
+    data: GuardedData<Private, MeiliSearch>,
     index_uid: web::Path<String>,
 ) -> Result<HttpResponse, ResponseError> {
     let settings = Settings::cleared();
+
+    let update = Update::Settings(settings.into_unchecked());
     let update_result = data
-        .update_settings(index_uid.into_inner(), settings, false)
+        .register_update(index_uid.into_inner(), update, false)
         .await?;
     let json = serde_json::json!({ "updateId": update_result.id() });
     debug!("returns: {:?}", json);

meilisearch-http/src/routes/indexes/updates.rs

@@ -1,25 +1,18 @@
 use actix_web::{web, HttpResponse};
 use chrono::{DateTime, Utc};
 use log::debug;
+use meilisearch_lib::MeiliSearch;
 use serde::{Deserialize, Serialize};
 
 use crate::error::ResponseError;
 use crate::extractors::authentication::{policies::*, GuardedData};
 use crate::routes::{IndexParam, UpdateStatusResponse};
-use crate::Data;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(web::resource("").route(web::get().to(get_all_updates_status)))
         .service(web::resource("{update_id}").route(web::get().to(get_update_status)));
 }
 
-#[derive(Debug, Deserialize)]
-#[serde(rename_all = "camelCase", deny_unknown_fields)]
-struct UpdateIndexRequest {
-    uid: Option<String>,
-    primary_key: Option<String>,
-}
-
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
 pub struct UpdateIndexResponse {
@@ -37,12 +30,12 @@ pub struct UpdateParam {
 }
 
 pub async fn get_update_status(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
     path: web::Path<UpdateParam>,
 ) -> Result<HttpResponse, ResponseError> {
     let params = path.into_inner();
-    let meta = data
-        .get_update_status(params.index_uid, params.update_id)
+    let meta = meilisearch
+        .update_status(params.index_uid, params.update_id)
         .await?;
     let meta = UpdateStatusResponse::from(meta);
     debug!("returns: {:?}", meta);
@@ -50,10 +43,12 @@ pub async fn get_update_status(
 }
 
 pub async fn get_all_updates_status(
-    data: GuardedData<Private, Data>,
+    meilisearch: GuardedData<Private, MeiliSearch>,
     path: web::Path<IndexParam>,
 ) -> Result<HttpResponse, ResponseError> {
-    let metas = data.get_updates_status(path.into_inner().index_uid).await?;
+    let metas = meilisearch
+        .all_update_status(path.into_inner().index_uid)
+        .await?;
     let metas = metas
         .into_iter()
         .map(UpdateStatusResponse::from)

meilisearch-http/src/routes/mod.rs

@@ -3,16 +3,18 @@ use std::time::Duration;
 use actix_web::{web, HttpResponse};
 use chrono::{DateTime, Utc};
 use log::debug;
+use meilisearch_lib::index_controller::updates::status::{UpdateResult, UpdateStatus};
 use serde::{Deserialize, Serialize};
 
+use meilisearch_lib::index::{Settings, Unchecked};
+use meilisearch_lib::{MeiliSearch, Update};
+
 use crate::error::ResponseError;
 use crate::extractors::authentication::{policies::*, GuardedData};
-use crate::index::{Settings, Unchecked};
-use crate::index_controller::{UpdateMeta, UpdateResult, UpdateStatus};
-use crate::Data;
+use crate::ApiKeys;
 
 mod dump;
-mod indexes;
+pub mod indexes;
 
 pub fn configure(cfg: &mut web::ServiceConfig) {
     cfg.service(web::resource("/health").route(web::get().to(get_health)))
@@ -48,9 +50,9 @@ pub enum UpdateType {
 
 impl From<&UpdateStatus> for UpdateType {
     fn from(other: &UpdateStatus) -> Self {
-        use milli::update::IndexDocumentsMethod::*;
+        use meilisearch_lib::milli::update::IndexDocumentsMethod::*;
         match other.meta() {
-            UpdateMeta::DocumentsAddition { method, .. } => {
+            Update::DocumentAddition { method, .. } => {
                 let number = match other {
                     UpdateStatus::Processed(processed) => match processed.success {
                         UpdateResult::DocumentsAddition(ref addition) => {
@@ -67,13 +69,13 @@ impl From<&UpdateStatus> for UpdateType {
                     _ => unreachable!(),
                 }
             }
-            UpdateMeta::ClearDocuments => UpdateType::ClearAll,
-            UpdateMeta::DeleteDocuments { ids } => UpdateType::DocumentsDeletion {
-                number: Some(ids.len()),
-            },
-            UpdateMeta::Settings(settings) => UpdateType::Settings {
+            Update::Settings(settings) => UpdateType::Settings {
                 settings: settings.clone(),
             },
+            Update::ClearDocuments => UpdateType::ClearAll,
+            Update::DeleteDocuments(ids) => UpdateType::DocumentsDeletion {
+                number: Some(ids.len()),
+            },
         }
     }
 }
@@ -95,8 +97,7 @@ pub struct FailedUpdateResult {
     pub update_id: u64,
     #[serde(rename = "type")]
     pub update_type: UpdateType,
-    #[serde(flatten)]
-    pub response: ResponseError,
+    pub error: ResponseError,
     pub duration: f64, // in seconds
     pub enqueued_at: DateTime<Utc>,
     pub processed_at: DateTime<Utc>,
@@ -186,15 +187,17 @@ impl From<UpdateStatus> for UpdateStatusResponse {
                 let duration = Duration::from_millis(duration as u64).as_secs_f64();
 
                 let update_id = failed.id();
-                let response = failed.error;
+                let processed_at = failed.failed_at;
+                let enqueued_at = failed.from.from.enqueued_at;
+                let error = failed.into();
 
                 let content = FailedUpdateResult {
                     update_id,
                     update_type,
-                    response,
+                    error,
                     duration,
-                    enqueued_at: failed.from.from.enqueued_at,
-                    processed_at: failed.failed_at,
+                    enqueued_at,
+                    processed_at,
                 };
                 UpdateStatusResponse::Failed { content }
             }
@@ -229,8 +232,10 @@ pub async fn running() -> HttpResponse {
     HttpResponse::Ok().json(serde_json::json!({ "status": "MeiliSearch is running" }))
 }
 
-async fn get_stats(data: GuardedData<Private, Data>) -> Result<HttpResponse, ResponseError> {
-    let response = data.get_all_stats().await?;
+async fn get_stats(
+    meilisearch: GuardedData<Private, MeiliSearch>,
+) -> Result<HttpResponse, ResponseError> {
+    let response = meilisearch.get_all_stats().await?;
 
     debug!("returns: {:?}", response);
     Ok(HttpResponse::Ok().json(response))
@@ -244,7 +249,7 @@ struct VersionResponse {
     pkg_version: String,
 }
 
-async fn get_version(_data: GuardedData<Private, Data>) -> HttpResponse {
+async fn get_version(_meilisearch: GuardedData<Private, MeiliSearch>) -> HttpResponse {
     let commit_sha = option_env!("VERGEN_GIT_SHA").unwrap_or("unknown");
     let commit_date = option_env!("VERGEN_GIT_COMMIT_TIMESTAMP").unwrap_or("unknown");
 
@@ -261,8 +266,8 @@ struct KeysResponse {
     public: Option<String>,
 }
 
-pub async fn list_keys(data: GuardedData<Admin, Data>) -> HttpResponse {
-    let api_keys = data.api_keys.clone();
+pub async fn list_keys(meilisearch: GuardedData<Admin, ApiKeys>) -> HttpResponse {
+    let api_keys = (*meilisearch).clone();
     HttpResponse::Ok().json(&KeysResponse {
         private: api_keys.private,
         public: api_keys.public,
@@ -276,17 +281,16 @@ pub async fn get_health() -> Result<HttpResponse, ResponseError> {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::data::Data;
     use crate::extractors::authentication::GuardedData;
 
     /// A type implemented for a route that uses a authentication policy `Policy`.
     ///
     /// This trait is used for regression testing of route authenticaton policies.
-    trait Is<Policy, T> {}
+    trait Is<Policy, Data, T> {}
 
     macro_rules! impl_is_policy {
         ($($param:ident)*) => {
-            impl<Policy, Func, $($param,)* Res> Is<Policy, (($($param,)*), Res)> for Func
+            impl<Policy, Func, Data, $($param,)* Res> Is<Policy, Data, (($($param,)*), Res)> for Func
                 where Func: Fn(GuardedData<Policy, Data>, $($param,)*) -> Res {}
 
         };
@@ -297,6 +301,7 @@ mod test {
     impl_is_policy! {A B}
     impl_is_policy! {A B C}
     impl_is_policy! {A B C D}
+    impl_is_policy! {A B C D E}
 
     /// Emits a compile error if a route doesn't have the correct authentication policy.
     ///
@@ -306,7 +311,7 @@ mod test {
         ($($policy:ident => { $($route:expr,)*})*) => {
             #[test]
             fn test_auth() {
-                $($(let _: &dyn Is<$policy, _> = &$route;)*)*
+                $($(let _: &dyn Is<$policy, _, _> = &$route;)*)*
             }
         };
     }

meilisearch-http/tests/common/index.rs

@@ -7,6 +7,7 @@ use actix_web::http::StatusCode;
 use paste::paste;
 use serde_json::{json, Value};
 use tokio::time::sleep;
+use urlencoding::encode;
 
 use super::service::Service;
 
@@ -14,12 +15,12 @@ macro_rules! make_settings_test_routes {
     ($($name:ident),+) => {
         $(paste! {
             pub async fn [<update_$name>](&self, value: Value) -> (Value, StatusCode) {
-                let url = format!("/indexes/{}/settings/{}", self.uid, stringify!($name).replace("_", "-"));
+                let url = format!("/indexes/{}/settings/{}", encode(self.uid.as_ref()).to_string(), stringify!($name).replace("_", "-"));
                 self.service.post(url, value).await
             }
 
             pub async fn [<get_$name>](&self) -> (Value, StatusCode) {
-                let url = format!("/indexes/{}/settings/{}", self.uid, stringify!($name).replace("_", "-"));
+                let url = format!("/indexes/{}/settings/{}", encode(self.uid.as_ref()).to_string(), stringify!($name).replace("_", "-"));
                 self.service.get(url).await
             }
         })*
@@ -34,12 +35,15 @@ pub struct Index<'a> {
 #[allow(dead_code)]
 impl Index<'_> {
     pub async fn get(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}", self.uid);
+        let url = format!("/indexes/{}", encode(self.uid.as_ref()).to_string());
         self.service.get(url).await
     }
 
     pub async fn load_test_set(&self) -> u64 {
-        let url = format!("/indexes/{}/documents", self.uid);
+        let url = format!(
+            "/indexes/{}/documents",
+            encode(self.uid.as_ref()).to_string()
+        );
         let (response, code) = self
             .service
             .post_str(url, include_str!("../assets/test_set.json"))
@@ -62,13 +66,13 @@ impl Index<'_> {
         let body = json!({
             "primaryKey": primary_key,
         });
-        let url = format!("/indexes/{}", self.uid);
+        let url = format!("/indexes/{}", encode(self.uid.as_ref()).to_string());
 
         self.service.put(url, body).await
     }
 
     pub async fn delete(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}", self.uid);
+        let url = format!("/indexes/{}", encode(self.uid.as_ref()).to_string());
         self.service.delete(url).await
     }
 
@@ -78,8 +82,15 @@ impl Index<'_> {
         primary_key: Option<&str>,
     ) -> (Value, StatusCode) {
         let url = match primary_key {
-            Some(key) => format!("/indexes/{}/documents?primaryKey={}", self.uid, key),
-            None => format!("/indexes/{}/documents", self.uid),
+            Some(key) => format!(
+                "/indexes/{}/documents?primaryKey={}",
+                encode(self.uid.as_ref()).to_string(),
+                key
+            ),
+            None => format!(
+                "/indexes/{}/documents",
+                encode(self.uid.as_ref()).to_string()
+            ),
         };
         self.service.post(url, documents).await
     }
@@ -90,15 +101,26 @@ impl Index<'_> {
         primary_key: Option<&str>,
     ) -> (Value, StatusCode) {
         let url = match primary_key {
-            Some(key) => format!("/indexes/{}/documents?primaryKey={}", self.uid, key),
-            None => format!("/indexes/{}/documents", self.uid),
+            Some(key) => format!(
+                "/indexes/{}/documents?primaryKey={}",
+                encode(self.uid.as_ref()).to_string(),
+                key
+            ),
+            None => format!(
+                "/indexes/{}/documents",
+                encode(self.uid.as_ref()).to_string()
+            ),
         };
         self.service.put(url, documents).await
     }
 
     pub async fn wait_update_id(&self, update_id: u64) -> Value {
         // try 10 times to get status, or panic to not wait forever
-        let url = format!("/indexes/{}/updates/{}", self.uid, update_id);
+        let url = format!(
+            "/indexes/{}/updates/{}",
+            encode(self.uid.as_ref()).to_string(),
+            update_id
+        );
         for _ in 0..10 {
             let (response, status_code) = self.service.get(&url).await;
             assert_eq!(status_code, 200, "response: {}", response);
@@ -113,12 +135,16 @@ impl Index<'_> {
     }
 
     pub async fn get_update(&self, update_id: u64) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/updates/{}", self.uid, update_id);
+        let url = format!(
+            "/indexes/{}/updates/{}",
+            encode(self.uid.as_ref()).to_string(),
+            update_id
+        );
         self.service.get(url).await
     }
 
     pub async fn list_updates(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/updates", self.uid);
+        let url = format!("/indexes/{}/updates", encode(self.uid.as_ref()).to_string());
         self.service.get(url).await
     }
 
@@ -127,12 +153,19 @@ impl Index<'_> {
         id: u64,
         _options: Option<GetDocumentOptions>,
     ) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents/{}", self.uid, id);
+        let url = format!(
+            "/indexes/{}/documents/{}",
+            encode(self.uid.as_ref()).to_string(),
+            id
+        );
         self.service.get(url).await
     }
 
     pub async fn get_all_documents(&self, options: GetAllDocumentsOptions) -> (Value, StatusCode) {
-        let mut url = format!("/indexes/{}/documents?", self.uid);
+        let mut url = format!(
+            "/indexes/{}/documents?",
+            encode(self.uid.as_ref()).to_string()
+        );
         if let Some(limit) = options.limit {
             url.push_str(&format!("limit={}&", limit));
         }
@@ -152,39 +185,58 @@ impl Index<'_> {
     }
 
     pub async fn delete_document(&self, id: u64) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents/{}", self.uid, id);
+        let url = format!(
+            "/indexes/{}/documents/{}",
+            encode(self.uid.as_ref()).to_string(),
+            id
+        );
         self.service.delete(url).await
     }
 
     pub async fn clear_all_documents(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents", self.uid);
+        let url = format!(
+            "/indexes/{}/documents",
+            encode(self.uid.as_ref()).to_string()
+        );
         self.service.delete(url).await
     }
 
     pub async fn delete_batch(&self, ids: Vec<u64>) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/documents/delete-batch", self.uid);
+        let url = format!(
+            "/indexes/{}/documents/delete-batch",
+            encode(self.uid.as_ref()).to_string()
+        );
         self.service
             .post(url, serde_json::to_value(&ids).unwrap())
             .await
     }
 
     pub async fn settings(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", self.uid);
+        let url = format!(
+            "/indexes/{}/settings",
+            encode(self.uid.as_ref()).to_string()
+        );
         self.service.get(url).await
     }
 
     pub async fn update_settings(&self, settings: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", self.uid);
+        let url = format!(
+            "/indexes/{}/settings",
+            encode(self.uid.as_ref()).to_string()
+        );
         self.service.post(url, settings).await
     }
 
     pub async fn delete_settings(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/settings", self.uid);
+        let url = format!(
+            "/indexes/{}/settings",
+            encode(self.uid.as_ref()).to_string()
+        );
         self.service.delete(url).await
     }
 
     pub async fn stats(&self) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/stats", self.uid);
+        let url = format!("/indexes/{}/stats", encode(self.uid.as_ref()).to_string());
         self.service.get(url).await
     }
 
@@ -209,13 +261,17 @@ impl Index<'_> {
     }
 
     pub async fn search_post(&self, query: Value) -> (Value, StatusCode) {
-        let url = format!("/indexes/{}/search", self.uid);
+        let url = format!("/indexes/{}/search", encode(self.uid.as_ref()).to_string());
         self.service.post(url, query).await
     }
 
     pub async fn search_get(&self, query: Value) -> (Value, StatusCode) {
         let params = serde_url_params::to_string(&query).unwrap();
-        let url = format!("/indexes/{}/search?{}", self.uid, params);
+        let url = format!(
+            "/indexes/{}/search?{}",
+            encode(self.uid.as_ref()).to_string(),
+            params
+        );
         self.service.get(url).await
     }
 

meilisearch-http/tests/common/server.rs

@@ -2,12 +2,13 @@ use std::path::Path;
 
 use actix_web::http::StatusCode;
 use byte_unit::{Byte, ByteUnit};
+use meilisearch_http::setup_meilisearch;
+use meilisearch_lib::options::{IndexerOpts, MaxMemory};
+use once_cell::sync::Lazy;
 use serde_json::Value;
-use tempdir::TempDir;
-use urlencoding::encode;
+use tempfile::TempDir;
 
-use meilisearch_http::data::Data;
-use meilisearch_http::option::{IndexerOpts, MaxMemory, Opt};
+use meilisearch_http::option::Opt;
 
 use super::index::Index;
 use super::service::Service;
@@ -15,17 +16,28 @@ use super::service::Service;
 pub struct Server {
     pub service: Service,
     // hold ownership to the tempdir while we use the server instance.
-    _dir: Option<tempdir::TempDir>,
+    _dir: Option<TempDir>,
 }
 
+static TEST_TEMP_DIR: Lazy<TempDir> = Lazy::new(|| TempDir::new().unwrap());
+
 impl Server {
     pub async fn new() -> Self {
-        let dir = TempDir::new("meilisearch").unwrap();
+        let dir = TempDir::new().unwrap();
 
-        let opt = default_settings(dir.path());
+        if cfg!(windows) {
+            std::env::set_var("TMP", TEST_TEMP_DIR.path());
+        } else {
+            std::env::set_var("TMPDIR", TEST_TEMP_DIR.path());
+        }
 
-        let data = Data::new(opt).unwrap();
-        let service = Service(data);
+        let options = default_settings(dir.path());
+
+        let meilisearch = setup_meilisearch(&options).unwrap();
+        let service = Service {
+            meilisearch,
+            options,
+        };
 
         Server {
             service,
@@ -33,9 +45,12 @@ impl Server {
         }
     }
 
-    pub async fn new_with_options(opt: Opt) -> Self {
-        let data = Data::new(opt).unwrap();
-        let service = Service(data);
+    pub async fn new_with_options(options: Opt) -> Self {
+        let meilisearch = setup_meilisearch(&options).unwrap();
+        let service = Service {
+            meilisearch,
+            options,
+        };
 
         Server {
             service,
@@ -46,7 +61,7 @@ impl Server {
     /// Returns a view to an index. There is no guarantee that the index exists.
     pub fn index(&self, uid: impl AsRef<str>) -> Index<'_> {
         Index {
-            uid: encode(uid.as_ref()),
+            uid: uid.as_ref().to_string(),
             service: &self.service,
         }
     }

meilisearch-http/tests/common/service.rs

@@ -1,14 +1,23 @@
 use actix_web::{http::StatusCode, test};
+use meilisearch_lib::MeiliSearch;
 use serde_json::Value;
 
-use meilisearch_http::create_app;
-use meilisearch_http::data::Data;
+use meilisearch_http::{analytics, create_app, Opt};
 
-pub struct Service(pub Data);
+pub struct Service {
+    pub meilisearch: MeiliSearch,
+    pub options: Opt,
+}
 
 impl Service {
     pub async fn post(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(&self.0, true)).await;
+        let app = test::init_service(create_app!(
+            &self.meilisearch,
+            true,
+            &self.options,
+            analytics::MockAnalytics::new(&self.options).0
+        ))
+        .await;
 
         let req = test::TestRequest::post()
             .uri(url.as_ref())
@@ -28,7 +37,13 @@ impl Service {
         url: impl AsRef<str>,
         body: impl AsRef<str>,
     ) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(&self.0, true)).await;
+        let app = test::init_service(create_app!(
+            &self.meilisearch,
+            true,
+            &self.options,
+            analytics::MockAnalytics::new(&self.options).0
+        ))
+        .await;
 
         let req = test::TestRequest::post()
             .uri(url.as_ref())
@@ -44,7 +59,13 @@ impl Service {
     }
 
     pub async fn get(&self, url: impl AsRef<str>) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(&self.0, true)).await;
+        let app = test::init_service(create_app!(
+            &self.meilisearch,
+            true,
+            &self.options,
+            analytics::MockAnalytics::new(&self.options).0
+        ))
+        .await;
 
         let req = test::TestRequest::get().uri(url.as_ref()).to_request();
         let res = test::call_service(&app, req).await;
@@ -56,7 +77,13 @@ impl Service {
     }
 
     pub async fn put(&self, url: impl AsRef<str>, body: Value) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(&self.0, true)).await;
+        let app = test::init_service(create_app!(
+            &self.meilisearch,
+            true,
+            &self.options,
+            analytics::MockAnalytics::new(&self.options).0
+        ))
+        .await;
 
         let req = test::TestRequest::put()
             .uri(url.as_ref())
@@ -71,7 +98,13 @@ impl Service {
     }
 
     pub async fn delete(&self, url: impl AsRef<str>) -> (Value, StatusCode) {
-        let app = test::init_service(create_app!(&self.0, true)).await;
+        let app = test::init_service(create_app!(
+            &self.meilisearch,
+            true,
+            &self.options,
+            analytics::MockAnalytics::new(&self.options).0
+        ))
+        .await;
 
         let req = test::TestRequest::delete().uri(url.as_ref()).to_request();
         let res = test::call_service(&app, req).await;

meilisearch-http/tests/content_type.rs

@@ -0,0 +1,112 @@
+#![allow(dead_code)]
+
+mod common;
+
+use crate::common::Server;
+use actix_web::test;
+use meilisearch_http::{analytics, create_app};
+use serde_json::{json, Value};
+
+#[actix_rt::test]
+async fn error_json_bad_content_type() {
+    let routes = [
+        // all the POST routes except the dumps that can be created without any body or content-type
+        // and the search that is not a strict json
+        "/indexes",
+        "/indexes/doggo/documents/delete-batch",
+        "/indexes/doggo/search",
+        "/indexes/doggo/settings",
+        "/indexes/doggo/settings/displayed-attributes",
+        "/indexes/doggo/settings/distinct-attribute",
+        "/indexes/doggo/settings/filterable-attributes",
+        "/indexes/doggo/settings/ranking-rules",
+        "/indexes/doggo/settings/searchable-attributes",
+        "/indexes/doggo/settings/sortable-attributes",
+        "/indexes/doggo/settings/stop-words",
+        "/indexes/doggo/settings/synonyms",
+    ];
+    let bad_content_types = [
+        "application/csv",
+        "application/x-ndjson",
+        "application/x-www-form-urlencoded",
+        "text/plain",
+        "json",
+        "application",
+        "json/application",
+    ];
+
+    let document = "{}";
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        true,
+        &server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+    for route in routes {
+        // Good content-type, we probably have an error since we didn't send anything in the json
+        // so we only ensure we didn't get a bad media type error.
+        let req = test::TestRequest::post()
+            .uri(route)
+            .set_payload(document)
+            .insert_header(("content-type", "application/json"))
+            .to_request();
+        let res = test::call_service(&app, req).await;
+        let status_code = res.status();
+        assert_ne!(status_code, 415,
+        "calling the route `{}` with a content-type of json isn't supposed to throw a bad media type error", route);
+
+        // No content-type.
+        let req = test::TestRequest::post()
+            .uri(route)
+            .set_payload(document)
+            .to_request();
+        let res = test::call_service(&app, req).await;
+        let status_code = res.status();
+        let body = test::read_body(res).await;
+        let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+        assert_eq!(status_code, 415, "calling the route `{}` without content-type is supposed to throw a bad media type error", route);
+        assert_eq!(
+            response,
+            json!({
+                    "message": r#"A Content-Type header is missing. Accepted values for the Content-Type header are: `application/json`"#,
+                    "code": "missing_content_type",
+                    "type": "invalid_request",
+                    "link": "https://docs.meilisearch.com/errors#missing_content_type",
+            }),
+            "when calling the route `{}` with no content-type",
+            route,
+        );
+
+        for bad_content_type in bad_content_types {
+            // Always bad content-type
+            let req = test::TestRequest::post()
+                .uri(route)
+                .set_payload(document.to_string())
+                .insert_header(("content-type", bad_content_type))
+                .to_request();
+            let res = test::call_service(&app, req).await;
+            let status_code = res.status();
+            let body = test::read_body(res).await;
+            let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+            assert_eq!(status_code, 415);
+            let expected_error_message = format!(
+                r#"The Content-Type `{}` is invalid. Accepted values for the Content-Type header are: `application/json`"#,
+                bad_content_type
+            );
+            assert_eq!(
+                response,
+                json!({
+                        "message": expected_error_message,
+                        "code": "invalid_content_type",
+                        "type": "invalid_request",
+                        "link": "https://docs.meilisearch.com/errors#invalid_content_type",
+                }),
+                "when calling the route `{}` with a content-type of `{}`",
+                route,
+                bad_content_type,
+            );
+        }
+    }
+}

meilisearch-http/tests/dashboard/mod.rs

@@ -0,0 +1,24 @@
+use crate::common::Server;
+
+#[actix_rt::test]
+async fn dashboard_assets_load() {
+    let server = Server::new().await;
+
+    mod generated {
+        include!(concat!(env!("OUT_DIR"), "/generated.rs"));
+    }
+
+    let generated = generated::generate();
+
+    for (path, _) in generated.into_iter() {
+        let path = if path == "index.html" {
+            // "index.html" redirects to "/"
+            "/".to_owned()
+        } else {
+            "/".to_owned() + path
+        };
+
+        let (_, status_code) = server.service.get(&path).await;
+        assert_eq!(status_code, 200);
+    }
+}

meilisearch-http/tests/documents/add_documents.rs

@@ -1,7 +1,7 @@
 use crate::common::{GetAllDocumentsOptions, Server};
 use actix_web::test;
 use chrono::DateTime;
-use meilisearch_http::create_app;
+use meilisearch_http::{analytics, create_app};
 use serde_json::{json, Value};
 
 /// This is the basic usage of our API and every other tests uses the content-type application/json
@@ -16,7 +16,14 @@ async fn add_documents_test_json_content_types() {
 
     // this is a what is expected and should work
     let server = Server::new().await;
-    let app = test::init_service(create_app!(&server.service.0, true)).await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        true,
+        &server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+    // post
     let req = test::TestRequest::post()
         .uri("/indexes/dog/documents")
         .set_payload(document.to_string())
@@ -28,21 +35,9 @@ async fn add_documents_test_json_content_types() {
     let response: Value = serde_json::from_slice(&body).unwrap_or_default();
     assert_eq!(status_code, 202);
     assert_eq!(response, json!({ "updateId": 0 }));
-}
 
-/// no content type is still supposed to be accepted as json
-#[actix_rt::test]
-async fn add_documents_test_no_content_types() {
-    let document = json!([
-        {
-            "id": 1,
-            "content": "Montagne des Pyrénées",
-        }
-    ]);
-
-    let server = Server::new().await;
-    let app = test::init_service(create_app!(&server.service.0, true)).await;
-    let req = test::TestRequest::post()
+    // put
+    let req = test::TestRequest::put()
         .uri("/indexes/dog/documents")
         .set_payload(document.to_string())
         .insert_header(("content-type", "application/json"))
@@ -52,13 +47,12 @@ async fn add_documents_test_no_content_types() {
     let body = test::read_body(res).await;
     let response: Value = serde_json::from_slice(&body).unwrap_or_default();
     assert_eq!(status_code, 202);
-    assert_eq!(response, json!({ "updateId": 0 }));
+    assert_eq!(response, json!({ "updateId": 1 }));
 }
 
 /// any other content-type is must be refused
 #[actix_rt::test]
-#[ignore]
-async fn add_documents_test_bad_content_types() {
+async fn error_add_documents_test_bad_content_types() {
     let document = json!([
         {
             "id": 1,
@@ -67,7 +61,14 @@ async fn add_documents_test_bad_content_types() {
     ]);
 
     let server = Server::new().await;
-    let app = test::init_service(create_app!(&server.service.0, true)).await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        true,
+        &server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+    // post
     let req = test::TestRequest::post()
         .uri("/indexes/dog/documents")
         .set_payload(document.to_string())
@@ -76,8 +77,451 @@ async fn add_documents_test_bad_content_types() {
     let res = test::call_service(&app, req).await;
     let status_code = res.status();
     let body = test::read_body(res).await;
-    assert_eq!(status_code, 405);
-    assert!(body.is_empty());
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 415);
+    assert_eq!(
+        response["message"],
+        json!(
+            r#"The Content-Type `text/plain` is invalid. Accepted values for the Content-Type header are: `application/json`, `application/x-ndjson`, `text/csv`"#
+        )
+    );
+    assert_eq!(response["code"], "invalid_content_type");
+    assert_eq!(response["type"], "invalid_request");
+    assert_eq!(
+        response["link"],
+        "https://docs.meilisearch.com/errors#invalid_content_type"
+    );
+
+    // put
+    let req = test::TestRequest::put()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "text/plain"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 415);
+    assert_eq!(
+        response["message"],
+        json!(
+            r#"The Content-Type `text/plain` is invalid. Accepted values for the Content-Type header are: `application/json`, `application/x-ndjson`, `text/csv`"#
+        )
+    );
+    assert_eq!(response["code"], "invalid_content_type");
+    assert_eq!(response["type"], "invalid_request");
+    assert_eq!(
+        response["link"],
+        "https://docs.meilisearch.com/errors#invalid_content_type"
+    );
+}
+
+/// missing content-type must be refused
+#[actix_rt::test]
+async fn error_add_documents_test_no_content_type() {
+    let document = json!([
+        {
+            "id": 1,
+            "content": "Leonberg",
+        }
+    ]);
+
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        true,
+        &server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+    // post
+    let req = test::TestRequest::post()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 415);
+    assert_eq!(
+        response["message"],
+        json!(
+            r#"A Content-Type header is missing. Accepted values for the Content-Type header are: `application/json`, `application/x-ndjson`, `text/csv`"#
+        )
+    );
+    assert_eq!(response["code"], "missing_content_type");
+    assert_eq!(response["type"], "invalid_request");
+    assert_eq!(
+        response["link"],
+        "https://docs.meilisearch.com/errors#missing_content_type"
+    );
+
+    // put
+    let req = test::TestRequest::put()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 415);
+    assert_eq!(
+        response["message"],
+        json!(
+            r#"A Content-Type header is missing. Accepted values for the Content-Type header are: `application/json`, `application/x-ndjson`, `text/csv`"#
+        )
+    );
+    assert_eq!(response["code"], "missing_content_type");
+    assert_eq!(response["type"], "invalid_request");
+    assert_eq!(
+        response["link"],
+        "https://docs.meilisearch.com/errors#missing_content_type"
+    );
+}
+
+#[actix_rt::test]
+async fn error_add_malformed_csv_documents() {
+    let document = "id, content\n1234, hello, world\n12, hello world";
+
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        true,
+        &server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+    // post
+    let req = test::TestRequest::post()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "text/csv"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 400);
+    assert_eq!(
+        response["message"],
+        json!(
+            r#"The `csv` payload provided is malformed. `CSV error: record 1 (line: 2, byte: 12): found record with 3 fields, but the previous record has 2 fields`."#
+        )
+    );
+    assert_eq!(response["code"], json!("malformed_payload"));
+    assert_eq!(response["type"], json!("invalid_request"));
+    assert_eq!(
+        response["link"],
+        json!("https://docs.meilisearch.com/errors#malformed_payload")
+    );
+
+    // put
+    let req = test::TestRequest::put()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "text/csv"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 400);
+    assert_eq!(
+        response["message"],
+        json!(
+            r#"The `csv` payload provided is malformed. `CSV error: record 1 (line: 2, byte: 12): found record with 3 fields, but the previous record has 2 fields`."#
+        )
+    );
+    assert_eq!(response["code"], json!("malformed_payload"));
+    assert_eq!(response["type"], json!("invalid_request"));
+    assert_eq!(
+        response["link"],
+        json!("https://docs.meilisearch.com/errors#malformed_payload")
+    );
+}
+
+#[actix_rt::test]
+async fn error_add_malformed_json_documents() {
+    let document = r#"[{"id": 1}, {id: 2}]"#;
+
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        true,
+        &server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+    // post
+    let req = test::TestRequest::post()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "application/json"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 400);
+    assert_eq!(
+        response["message"],
+        json!(
+            r#"The `json` payload provided is malformed. `Couldn't serialize document value: key must be a string at line 1 column 14`."#
+        )
+    );
+    assert_eq!(response["code"], json!("malformed_payload"));
+    assert_eq!(response["type"], json!("invalid_request"));
+    assert_eq!(
+        response["link"],
+        json!("https://docs.meilisearch.com/errors#malformed_payload")
+    );
+
+    // put
+    let req = test::TestRequest::put()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "application/json"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 400);
+    assert_eq!(
+        response["message"],
+        json!(
+            r#"The `json` payload provided is malformed. `Couldn't serialize document value: key must be a string at line 1 column 14`."#
+        )
+    );
+    assert_eq!(response["code"], json!("malformed_payload"));
+    assert_eq!(response["type"], json!("invalid_request"));
+    assert_eq!(
+        response["link"],
+        json!("https://docs.meilisearch.com/errors#malformed_payload")
+    );
+}
+
+#[actix_rt::test]
+async fn error_add_malformed_ndjson_documents() {
+    let document = "{\"id\": 1}\n{id: 2}";
+
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        true,
+        &server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+    // post
+    let req = test::TestRequest::post()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "application/x-ndjson"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 400);
+    assert_eq!(
+        response["message"],
+        json!(
+            r#"The `ndjson` payload provided is malformed. `Couldn't serialize document value: key must be a string at line 1 column 2`."#
+        )
+    );
+    assert_eq!(response["code"], json!("malformed_payload"));
+    assert_eq!(response["type"], json!("invalid_request"));
+    assert_eq!(
+        response["link"],
+        json!("https://docs.meilisearch.com/errors#malformed_payload")
+    );
+
+    // put
+    let req = test::TestRequest::put()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "application/x-ndjson"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 400);
+    assert_eq!(
+        response["message"],
+        json!(
+            r#"The `ndjson` payload provided is malformed. `Couldn't serialize document value: key must be a string at line 1 column 2`."#
+        )
+    );
+    assert_eq!(response["code"], json!("malformed_payload"));
+    assert_eq!(response["type"], json!("invalid_request"));
+    assert_eq!(
+        response["link"],
+        json!("https://docs.meilisearch.com/errors#malformed_payload")
+    );
+}
+
+#[actix_rt::test]
+async fn error_add_missing_payload_csv_documents() {
+    let document = "";
+
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        true,
+        &server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+    // post
+    let req = test::TestRequest::post()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "text/csv"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 400);
+    assert_eq!(response["message"], json!(r#"A csv payload is missing."#));
+    assert_eq!(response["code"], json!("missing_payload"));
+    assert_eq!(response["type"], json!("invalid_request"));
+    assert_eq!(
+        response["link"],
+        json!("https://docs.meilisearch.com/errors#missing_payload")
+    );
+
+    // put
+    let req = test::TestRequest::put()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "text/csv"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 400);
+    assert_eq!(response["message"], json!(r#"A csv payload is missing."#));
+    assert_eq!(response["code"], json!("missing_payload"));
+    assert_eq!(response["type"], json!("invalid_request"));
+    assert_eq!(
+        response["link"],
+        json!("https://docs.meilisearch.com/errors#missing_payload")
+    );
+}
+
+#[actix_rt::test]
+async fn error_add_missing_payload_json_documents() {
+    let document = "";
+
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        true,
+        &server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+    // post
+    let req = test::TestRequest::post()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "application/json"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 400);
+    assert_eq!(response["message"], json!(r#"A json payload is missing."#));
+    assert_eq!(response["code"], json!("missing_payload"));
+    assert_eq!(response["type"], json!("invalid_request"));
+    assert_eq!(
+        response["link"],
+        json!("https://docs.meilisearch.com/errors#missing_payload")
+    );
+
+    // put
+    let req = test::TestRequest::put()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "application/json"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 400);
+    assert_eq!(response["message"], json!(r#"A json payload is missing."#));
+    assert_eq!(response["code"], json!("missing_payload"));
+    assert_eq!(response["type"], json!("invalid_request"));
+    assert_eq!(
+        response["link"],
+        json!("https://docs.meilisearch.com/errors#missing_payload")
+    );
+}
+
+#[actix_rt::test]
+async fn error_add_missing_payload_ndjson_documents() {
+    let document = "";
+
+    let server = Server::new().await;
+    let app = test::init_service(create_app!(
+        &server.service.meilisearch,
+        true,
+        &server.service.options,
+        analytics::MockAnalytics::new(&server.service.options).0
+    ))
+    .await;
+    // post
+    let req = test::TestRequest::post()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "application/x-ndjson"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 400);
+    assert_eq!(
+        response["message"],
+        json!(r#"A ndjson payload is missing."#)
+    );
+    assert_eq!(response["code"], json!("missing_payload"));
+    assert_eq!(response["type"], json!("invalid_request"));
+    assert_eq!(
+        response["link"],
+        json!("https://docs.meilisearch.com/errors#missing_payload")
+    );
+
+    // put
+    let req = test::TestRequest::put()
+        .uri("/indexes/dog/documents")
+        .set_payload(document.to_string())
+        .insert_header(("content-type", "application/x-ndjson"))
+        .to_request();
+    let res = test::call_service(&app, req).await;
+    let status_code = res.status();
+    let body = test::read_body(res).await;
+    let response: Value = serde_json::from_slice(&body).unwrap_or_default();
+    assert_eq!(status_code, 400);
+    assert_eq!(
+        response["message"],
+        json!(r#"A ndjson payload is missing."#)
+    );
+    assert_eq!(response["code"], json!("missing_payload"));
+    assert_eq!(response["type"], json!("invalid_request"));
+    assert_eq!(
+        response["link"],
+        json!("https://docs.meilisearch.com/errors#missing_payload")
+    );
 }
 
 #[actix_rt::test]
@@ -126,18 +570,36 @@ async fn add_documents_no_index_creation() {
 }
 
 #[actix_rt::test]
-async fn document_add_create_index_bad_uid() {
+async fn error_document_add_create_index_bad_uid() {
     let server = Server::new().await;
     let index = server.index("883  fj!");
-    let (_response, code) = index.add_documents(json!([]), None).await;
+    let (response, code) = index.add_documents(json!([]), None).await;
+
+    let expected_response = json!({
+        "message": "`883  fj!` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).",
+        "code": "invalid_index_uid",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_index_uid"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 400);
 }
 
 #[actix_rt::test]
-async fn document_update_create_index_bad_uid() {
+async fn error_document_update_create_index_bad_uid() {
     let server = Server::new().await;
     let index = server.index("883  fj!");
-    let (_response, code) = index.update_documents(json!([]), None).await;
+    let (response, code) = index.update_documents(json!([]), None).await;
+
+    let expected_response = json!({
+        "message": "`883  fj!` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).",
+        "code": "invalid_index_uid",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_index_uid"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 400);
 }
 
@@ -197,60 +659,6 @@ async fn document_update_with_primary_key() {
     assert_eq!(response["primaryKey"], "primary");
 }
 
-#[actix_rt::test]
-async fn add_documents_with_primary_key_and_primary_key_already_exists() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.create(Some("primary")).await;
-    let documents = json!([
-        {
-            "id": 1,
-            "content": "foo",
-        }
-    ]);
-
-    let (_response, code) = index.add_documents(documents, Some("id")).await;
-    assert_eq!(code, 202);
-
-    index.wait_update_id(0).await;
-
-    let (response, code) = index.get_update(0).await;
-    assert_eq!(code, 200);
-    assert_eq!(response["status"], "failed");
-
-    let (response, code) = index.get().await;
-    assert_eq!(code, 200);
-    assert_eq!(response["primaryKey"], "primary");
-}
-
-#[actix_rt::test]
-async fn update_documents_with_primary_key_and_primary_key_already_exists() {
-    let server = Server::new().await;
-    let index = server.index("test");
-
-    index.create(Some("primary")).await;
-    let documents = json!([
-        {
-            "id": 1,
-            "content": "foo",
-        }
-    ]);
-
-    let (_response, code) = index.update_documents(documents, Some("id")).await;
-    assert_eq!(code, 202);
-
-    index.wait_update_id(0).await;
-    let (response, code) = index.get_update(0).await;
-    assert_eq!(code, 200);
-    // Documents without a primary key are not accepted.
-    assert_eq!(response["status"], "failed");
-
-    let (response, code) = index.get().await;
-    assert_eq!(code, 200);
-    assert_eq!(response["primaryKey"], "primary");
-}
-
 #[actix_rt::test]
 async fn replace_document() {
     let server = Server::new().await;
@@ -289,25 +697,21 @@ async fn replace_document() {
     assert_eq!(response.to_string(), r##"{"doc_id":1,"other":"bar"}"##);
 }
 
-// test broken, see issue milli#92
 #[actix_rt::test]
-#[ignore]
-async fn add_no_documents() {
+async fn error_add_no_documents() {
     let server = Server::new().await;
     let index = server.index("test");
-    let (_response, code) = index.add_documents(json!([]), None).await;
-    assert_eq!(code, 200);
+    let (response, code) = index.add_documents(json!([]), None).await;
 
-    index.wait_update_id(0).await;
-    let (response, code) = index.get_update(0).await;
-    assert_eq!(code, 200);
-    assert_eq!(response["status"], "processed");
-    assert_eq!(response["updateId"], 0);
-    assert_eq!(response["success"]["DocumentsAddition"]["nb_documents"], 0);
+    let expected_response = json!({
+        "message": "The `json` payload must contain at least one document.",
+        "code": "malformed_payload",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#malformed_payload"
+    });
 
-    let (response, code) = index.get().await;
-    assert_eq!(code, 200);
-    assert_eq!(response["primaryKey"], Value::Null);
+    assert_eq!(response, expected_response);
+    assert_eq!(code, 400);
 }
 
 #[actix_rt::test]
@@ -393,7 +797,7 @@ async fn update_larger_dataset() {
 }
 
 #[actix_rt::test]
-async fn add_documents_bad_primary_key() {
+async fn error_add_documents_bad_document_id() {
     let server = Server::new().await;
     let index = server.index("test");
     index.create(Some("docid")).await;
@@ -407,11 +811,20 @@ async fn add_documents_bad_primary_key() {
     index.wait_update_id(0).await;
     let (response, code) = index.get_update(0).await;
     assert_eq!(code, 200);
-    assert_eq!(response["status"], "failed");
+    assert_eq!(response["status"], json!("failed"));
+
+    let expected_error = json!({
+        "message": "Document identifier `foo & bar` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_).",
+        "code": "invalid_document_id",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_document_id"
+    });
+
+    assert_eq!(response["error"], expected_error);
 }
 
 #[actix_rt::test]
-async fn update_documents_bad_primary_key() {
+async fn error_update_documents_bad_document_id() {
     let server = Server::new().await;
     let index = server.index("test");
     index.create(Some("docid")).await;
@@ -425,5 +838,190 @@ async fn update_documents_bad_primary_key() {
     index.wait_update_id(0).await;
     let (response, code) = index.get_update(0).await;
     assert_eq!(code, 200);
-    assert_eq!(response["status"], "failed");
+    assert_eq!(response["status"], json!("failed"));
+
+    let expected_error = json!({
+        "message": "Document identifier `foo & bar` is invalid. A document identifier can be of type integer or string, only composed of alphanumeric characters (a-z A-Z 0-9), hyphens (-) and underscores (_).",
+        "code": "invalid_document_id",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_document_id"
+    });
+
+    assert_eq!(response["error"], expected_error);
+}
+
+#[actix_rt::test]
+async fn error_add_documents_missing_document_id() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(Some("docid")).await;
+    let documents = json!([
+        {
+            "id": "11",
+            "content": "foobar"
+        }
+    ]);
+    index.add_documents(documents, None).await;
+    index.wait_update_id(0).await;
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "failed");
+
+    let expected_error = json!({
+        "message": r#"Document doesn't have a `docid` attribute: `{"id":"11","content":"foobar"}`."#,
+        "code": "missing_document_id",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#missing_document_id"
+    });
+
+    assert_eq!(response["error"], expected_error);
+}
+
+#[actix_rt::test]
+async fn error_update_documents_missing_document_id() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(Some("docid")).await;
+    let documents = json!([
+        {
+            "id": "11",
+            "content": "foobar"
+        }
+    ]);
+    index.update_documents(documents, None).await;
+    index.wait_update_id(0).await;
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "failed");
+
+    let expected_error = json!({
+        "message": r#"Document doesn't have a `docid` attribute: `{"id":"11","content":"foobar"}`."#,
+        "code": "missing_document_id",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#missing_document_id"
+    });
+
+    assert_eq!(response["error"], expected_error);
+}
+
+#[actix_rt::test]
+#[ignore] // // TODO: Fix in an other PR: this does not provoke any error.
+async fn error_document_field_limit_reached() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index.create(Some("id")).await;
+
+    let mut big_object = std::collections::HashMap::new();
+    big_object.insert("id".to_owned(), "wow");
+    for i in 0..65535 {
+        let key = i.to_string();
+        big_object.insert(key, "I am a text!");
+    }
+
+    let documents = json!([big_object]);
+
+    let (_response, code) = index.update_documents(documents, Some("id")).await;
+    assert_eq!(code, 202);
+
+    index.wait_update_id(0).await;
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    // Documents without a primary key are not accepted.
+    assert_eq!(response["status"], "failed");
+
+    let expected_error = json!({
+        "message": "A document cannot contain more than 65,535 fields.",
+        "code": "document_fields_limit_reached",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#document_fields_limit_reached"
+    });
+
+    assert_eq!(response["error"], expected_error);
+}
+
+#[actix_rt::test]
+async fn error_add_documents_invalid_geo_field() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(Some("id")).await;
+    index
+        .update_settings(json!({"sortableAttributes": ["_geo"]}))
+        .await;
+
+    let documents = json!([
+        {
+            "id": "11",
+            "_geo": "foobar"
+        }
+    ]);
+
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+    let (response, code) = index.get_update(1).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "failed");
+
+    let expected_error = json!({
+        "message": r#"The document with the id: `11` contains an invalid _geo field: `foobar`."#,
+        "code": "invalid_geo_field",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_geo_field"
+    });
+
+    assert_eq!(response["error"], expected_error);
+}
+
+#[actix_rt::test]
+async fn error_add_documents_payload_size() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(Some("id")).await;
+    let document = json!(
+        {
+            "id": "11",
+            "content": "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec metus erat, consequat in blandit venenatis, ultricies eu ipsum. Etiam luctus elit et mollis ultrices. Nam turpis risus, dictum non eros in, eleifend feugiat elit. Morbi non dolor pulvinar, sagittis mi sed, ultricies lorem. Nulla ultricies sem metus. Donec at suscipit quam, sed elementum mi. Suspendisse potenti. Fusce pharetra turpis tortor, sed eleifend odio dapibus ut. Nulla facilisi. Suspendisse elementum, dui eget aliquet dignissim, ex tellus aliquam nisl, at eleifend nisl metus tempus diam. Mauris fermentum sollicitudin efficitur. Donec dignissim est vitae elit finibus faucibus"
+        }
+    );
+    let documents: Vec<_> = (0..16000).into_iter().map(|_| document.clone()).collect();
+    let documents = json!(documents);
+    let (response, code) = index.add_documents(documents, None).await;
+
+    let expected_response = json!({
+        "message": "The provided payload reached the size limit.",
+        "code": "payload_too_large",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#payload_too_large"
+    });
+
+    assert_eq!(response, expected_response);
+    assert_eq!(code, 413);
+}
+
+#[actix_rt::test]
+async fn error_primary_key_inference() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let documents = json!([
+        {
+            "title": "11",
+            "desc": "foobar"
+        }
+    ]);
+
+    index.add_documents(documents, None).await;
+    index.wait_update_id(0).await;
+    let (response, code) = index.get_update(0).await;
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "failed");
+
+    let expected_error = json!({
+        "message": r#"The primary key inference process failed because the engine did not find any fields containing `id` substring in their name. If your document identifier does not contain any `id` substring, you can set the primary key of the index."#,
+        "code": "primary_key_inference_failed",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#primary_key_inference_failed"
+    });
+
+    assert_eq!(response["error"], expected_error);
 }

meilisearch-http/tests/documents/delete_documents.rs

@@ -83,10 +83,17 @@ async fn clear_all_documents_empty_index() {
 }
 
 #[actix_rt::test]
-async fn delete_batch_unexisting_index() {
+async fn error_delete_batch_unexisting_index() {
     let server = Server::new().await;
     let (response, code) = server.index("test").delete_batch(vec![]).await;
-    assert_eq!(code, 404, "{}", response);
+    let expected_response = json!({
+        "message": "Index `test` not found.",
+        "code": "index_not_found",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#index_not_found"
+    });
+    assert_eq!(code, 404);
+    assert_eq!(response, expected_response);
 }
 
 #[actix_rt::test]

meilisearch-http/tests/documents/get_documents.rs

@@ -13,11 +13,20 @@ async fn get_unexisting_index_single_document() {
 }
 
 #[actix_rt::test]
-async fn get_unexisting_document() {
+async fn error_get_unexisting_document() {
     let server = Server::new().await;
     let index = server.index("test");
     index.create(None).await;
-    let (_response, code) = index.get_document(1, None).await;
+    let (response, code) = index.get_document(1, None).await;
+
+    let expected_response = json!({
+        "message": "Document `1` not found.",
+        "code": "document_not_found",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#document_not_found"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 404);
 }
 
@@ -47,12 +56,21 @@ async fn get_document() {
 }
 
 #[actix_rt::test]
-async fn get_unexisting_index_all_documents() {
+async fn error_get_unexisting_index_all_documents() {
     let server = Server::new().await;
-    let (_response, code) = server
+    let (response, code) = server
         .index("test")
         .get_all_documents(GetAllDocumentsOptions::default())
         .await;
+
+    let expected_response = json!({
+        "message": "Index `test` not found.",
+        "code": "index_not_found",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#index_not_found"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 404);
 }
 

meilisearch-http/tests/index/create_index.rs

@@ -1,5 +1,5 @@
 use crate::common::Server;
-use serde_json::Value;
+use serde_json::{json, Value};
 
 #[actix_rt::test]
 async fn create_index_no_primary_key() {
@@ -33,26 +33,20 @@ async fn create_index_with_primary_key() {
     assert_eq!(response.as_object().unwrap().len(), 5);
 }
 
-// TODO: partial test since we are testing error, amd error is not yet fully implemented in
-// transplant
 #[actix_rt::test]
-async fn create_existing_index() {
+async fn create_index_with_invalid_primary_key() {
+    let document = json!([ { "id": 2, "title": "Pride and Prejudice" } ]);
+
     let server = Server::new().await;
-    let index = server.index("test");
-    let (_, code) = index.create(Some("primary")).await;
+    let index = server.index("movies");
+    let (_response, code) = index.add_documents(document, Some("title")).await;
+    assert_eq!(code, 202);
 
-    assert_eq!(code, 201);
+    index.wait_update_id(0).await;
 
-    let (_response, code) = index.create(Some("primary")).await;
-    assert_eq!(code, 400);
-}
-
-#[actix_rt::test]
-async fn create_with_invalid_index_uid() {
-    let server = Server::new().await;
-    let index = server.index("test test#!");
-    let (_, code) = index.create(None).await;
-    assert_eq!(code, 400);
+    let (response, code) = index.get().await;
+    assert_eq!(code, 200);
+    assert_eq!(response["primaryKey"], Value::Null);
 }
 
 #[actix_rt::test]
@@ -72,3 +66,41 @@ async fn test_create_multiple_indexes() {
     assert_eq!(index3.get().await.1, 200);
     assert_eq!(index4.get().await.1, 404);
 }
+
+#[actix_rt::test]
+async fn error_create_existing_index() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    let (_, code) = index.create(Some("primary")).await;
+
+    assert_eq!(code, 201);
+
+    let (response, code) = index.create(Some("primary")).await;
+
+    let expected_response = json!({
+        "message": "Index `test` already exists.",
+        "code": "index_already_exists",
+        "type": "invalid_request",
+        "link":"https://docs.meilisearch.com/errors#index_already_exists"
+    });
+
+    assert_eq!(response, expected_response);
+    assert_eq!(code, 409);
+}
+
+#[actix_rt::test]
+async fn error_create_with_invalid_index_uid() {
+    let server = Server::new().await;
+    let index = server.index("test test#!");
+    let (response, code) = index.create(None).await;
+
+    let expected_response = json!({
+        "message": "`test test#!` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).",
+        "code": "invalid_index_uid",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_index_uid"
+    });
+
+    assert_eq!(response, expected_response);
+    assert_eq!(code, 400);
+}

meilisearch-http/tests/index/delete_index.rs

@@ -18,11 +18,19 @@ async fn create_and_delete_index() {
 }
 
 #[actix_rt::test]
-async fn delete_unexisting_index() {
+async fn error_delete_unexisting_index() {
     let server = Server::new().await;
     let index = server.index("test");
-    let (_response, code) = index.delete().await;
+    let (response, code) = index.delete().await;
 
+    let expected_response = json!({
+        "message": "Index `test` not found.",
+        "code": "index_not_found",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#index_not_found"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 404);
 }
 

meilisearch-http/tests/index/get_index.rs

@@ -1,4 +1,5 @@
 use crate::common::Server;
+use serde_json::json;
 use serde_json::Value;
 
 #[actix_rt::test]
@@ -21,15 +22,21 @@ async fn create_and_get_index() {
     assert_eq!(response.as_object().unwrap().len(), 5);
 }
 
-// TODO: partial test since we are testing error, and error is not yet fully implemented in
-// transplant
 #[actix_rt::test]
-async fn get_unexisting_index() {
+async fn error_get_unexisting_index() {
     let server = Server::new().await;
     let index = server.index("test");
 
-    let (_response, code) = index.get().await;
+    let (response, code) = index.get().await;
 
+    let expected_response = json!({
+        "message": "Index `test` not found.",
+        "code": "index_not_found",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#index_not_found"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 404);
 }
 

meilisearch-http/tests/index/stats.rs

@@ -46,3 +46,19 @@ async fn stats() {
     assert_eq!(response["fieldDistribution"]["name"], 1);
     assert_eq!(response["fieldDistribution"]["age"], 1);
 }
+
+#[actix_rt::test]
+async fn error_get_stats_unexisting_index() {
+    let server = Server::new().await;
+    let (response, code) = server.index("test").stats().await;
+
+    let expected_response = json!({
+        "message": "Index `test` not found.",
+        "code": "index_not_found",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#index_not_found"
+    });
+
+    assert_eq!(response, expected_response);
+    assert_eq!(code, 404);
+}

meilisearch-http/tests/index/update_index.rs

@@ -1,5 +1,6 @@
 use crate::common::Server;
 use chrono::DateTime;
+use serde_json::json;
 
 #[actix_rt::test]
 async fn update_primary_key() {
@@ -39,26 +40,48 @@ async fn update_nothing() {
     assert_eq!(response, update);
 }
 
-// TODO: partial test since we are testing error, amd error is not yet fully implemented in
-// transplant
 #[actix_rt::test]
-async fn update_existing_primary_key() {
+async fn error_update_existing_primary_key() {
     let server = Server::new().await;
     let index = server.index("test");
-    let (_response, code) = index.create(Some("primary")).await;
+    let (_response, code) = index.create(Some("id")).await;
 
     assert_eq!(code, 201);
 
-    let (_update, code) = index.update(Some("primary2")).await;
+    let documents = json!([
+        {
+            "id": "11",
+            "content": "foobar"
+        }
+    ]);
+    index.add_documents(documents, None).await;
+    index.wait_update_id(0).await;
 
+    let (response, code) = index.update(Some("primary")).await;
+
+    let expected_response = json!({
+        "message": "Index already has a primary key: `id`.",
+        "code": "index_primary_key_already_exists",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#index_primary_key_already_exists"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 400);
 }
 
-// TODO: partial test since we are testing error, amd error is not yet fully implemented in
-// transplant
 #[actix_rt::test]
-async fn test_unexisting_index() {
+async fn error_update_unexisting_index() {
     let server = Server::new().await;
-    let (_response, code) = server.index("test").update(None).await;
+    let (response, code) = server.index("test").update(None).await;
+
+    let expected_response = json!({
+        "message": "Index `test` not found.",
+        "code": "index_not_found",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#index_not_found"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 404);
 }

meilisearch-http/tests/integration.rs

@@ -1,4 +1,5 @@
 mod common;
+mod dashboard;
 mod documents;
 mod index;
 mod search;

meilisearch-http/tests/search/errors.rs

@@ -1,15 +1,24 @@
 use crate::common::Server;
 use serde_json::json;
 
+use super::DOCUMENTS;
+
 #[actix_rt::test]
 async fn search_unexisting_index() {
     let server = Server::new().await;
     let index = server.index("test");
 
+    let expected_response = json!({
+        "message": "Index `test` not found.",
+        "code": "index_not_found",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#index_not_found"
+    });
+
     index
         .search(json!({"q": "hello"}), |response, code| {
-            assert_eq!(code, 404, "{}", response);
-            assert_eq!(response["errorCode"], "index_not_found");
+            assert_eq!(code, 404);
+            assert_eq!(response, expected_response);
         })
         .await;
 }
@@ -22,7 +31,421 @@ async fn search_unexisting_parameter() {
     index
         .search(json!({"marin": "hello"}), |response, code| {
             assert_eq!(code, 400, "{}", response);
-            assert_eq!(response["errorCode"], "bad_request");
+            assert_eq!(response["code"], "bad_request");
         })
         .await;
 }
+
+#[actix_rt::test]
+async fn filter_invalid_syntax_object() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"filterableAttributes": ["title"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "Invalid syntax for the filter parameter: ` --> 1:7\n  |\n1 | title & Glass\n  |       ^---\n  |\n  = expected word`.",
+        "code": "invalid_filter",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_filter"
+    });
+    index
+        .search(json!({"filter": "title & Glass"}), |response, code| {
+            assert_eq!(response, expected_response);
+            assert_eq!(code, 400);
+        })
+        .await;
+}
+
+#[actix_rt::test]
+async fn filter_invalid_syntax_array() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"filterableAttributes": ["title"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "Invalid syntax for the filter parameter: ` --> 1:7\n  |\n1 | title & Glass\n  |       ^---\n  |\n  = expected word`.",
+        "code": "invalid_filter",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_filter"
+    });
+    index
+        .search(json!({"filter": [["title & Glass"]]}), |response, code| {
+            assert_eq!(response, expected_response);
+            assert_eq!(code, 400);
+        })
+        .await;
+}
+
+#[actix_rt::test]
+async fn filter_invalid_syntax_string() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"filterableAttributes": ["title"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "Invalid syntax for the filter parameter: ` --> 1:15\n  |\n1 | title = Glass XOR title = Glass\n  |               ^---\n  |\n  = expected EOI, and, or or`.",
+        "code": "invalid_filter",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_filter"
+    });
+    index
+        .search(
+            json!({"filter": "title = Glass XOR title = Glass"}),
+            |response, code| {
+                assert_eq!(response, expected_response);
+                assert_eq!(code, 400);
+            },
+        )
+        .await;
+}
+
+#[actix_rt::test]
+async fn filter_invalid_attribute_array() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"filterableAttributes": ["title"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "Attribute `many` is not filterable. Available filterable attributes are: `title`.",
+        "code": "invalid_filter",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_filter"
+    });
+    index
+        .search(json!({"filter": [["many = Glass"]]}), |response, code| {
+            assert_eq!(response, expected_response);
+            assert_eq!(code, 400);
+        })
+        .await;
+}
+
+#[actix_rt::test]
+async fn filter_invalid_attribute_string() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"filterableAttributes": ["title"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "Attribute `many` is not filterable. Available filterable attributes are: `title`.",
+        "code": "invalid_filter",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_filter"
+    });
+    index
+        .search(json!({"filter": "many = Glass"}), |response, code| {
+            assert_eq!(response, expected_response);
+            assert_eq!(code, 400);
+        })
+        .await;
+}
+
+#[actix_rt::test]
+async fn filter_reserved_geo_attribute_array() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"filterableAttributes": ["title"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "`_geo` is a reserved keyword and thus can't be used as a filter expression. Use the _geoRadius(latitude, longitude, distance) built-in rule to filter on _geo field coordinates.",
+        "code": "invalid_filter",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_filter"
+    });
+    index
+        .search(json!({"filter": [["_geo = Glass"]]}), |response, code| {
+            assert_eq!(response, expected_response);
+            assert_eq!(code, 400);
+        })
+        .await;
+}
+
+#[actix_rt::test]
+async fn filter_reserved_geo_attribute_string() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"filterableAttributes": ["title"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "`_geo` is a reserved keyword and thus can't be used as a filter expression. Use the _geoRadius(latitude, longitude, distance) built-in rule to filter on _geo field coordinates.",
+        "code": "invalid_filter",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_filter"
+    });
+    index
+        .search(json!({"filter": "_geo = Glass"}), |response, code| {
+            assert_eq!(response, expected_response);
+            assert_eq!(code, 400);
+        })
+        .await;
+}
+
+#[actix_rt::test]
+async fn filter_reserved_attribute_array() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"filterableAttributes": ["title"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "`_geoDistance` is a reserved keyword and thus can't be used as a filter expression.",
+        "code": "invalid_filter",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_filter"
+    });
+    index
+        .search(
+            json!({"filter": [["_geoDistance = Glass"]]}),
+            |response, code| {
+                assert_eq!(response, expected_response);
+                assert_eq!(code, 400);
+            },
+        )
+        .await;
+}
+
+#[actix_rt::test]
+async fn filter_reserved_attribute_string() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"filterableAttributes": ["title"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "`_geoDistance` is a reserved keyword and thus can't be used as a filter expression.",
+        "code": "invalid_filter",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_filter"
+    });
+    index
+        .search(
+            json!({"filter": "_geoDistance = Glass"}),
+            |response, code| {
+                assert_eq!(response, expected_response);
+                assert_eq!(code, 400);
+            },
+        )
+        .await;
+}
+
+#[actix_rt::test]
+async fn sort_geo_reserved_attribute() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"sortableAttributes": ["id"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "`_geo` is a reserved keyword and thus can't be used as a sort expression. Use the _geoPoint(latitude, longitude) built-in rule to sort on _geo field coordinates.",
+        "code": "invalid_sort",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_sort"
+    });
+    index
+        .search(
+            json!({
+                "sort": ["_geo:asc"]
+            }),
+            |response, code| {
+                assert_eq!(response, expected_response);
+                assert_eq!(code, 400);
+            },
+        )
+        .await;
+}
+
+#[actix_rt::test]
+async fn sort_reserved_attribute() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"sortableAttributes": ["id"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "`_geoDistance` is a reserved keyword and thus can't be used as a sort expression.",
+        "code": "invalid_sort",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_sort"
+    });
+    index
+        .search(
+            json!({
+                "sort": ["_geoDistance:asc"]
+            }),
+            |response, code| {
+                assert_eq!(response, expected_response);
+                assert_eq!(code, 400);
+            },
+        )
+        .await;
+}
+
+#[actix_rt::test]
+async fn sort_unsortable_attribute() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"sortableAttributes": ["id"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "Attribute `title` is not sortable. Available sortable attributes are: `id`.",
+        "code": "invalid_sort",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_sort"
+    });
+    index
+        .search(
+            json!({
+                "sort": ["title:asc"]
+            }),
+            |response, code| {
+                assert_eq!(response, expected_response);
+                assert_eq!(code, 400);
+            },
+        )
+        .await;
+}
+
+#[actix_rt::test]
+async fn sort_invalid_syntax() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(json!({"sortableAttributes": ["id"]}))
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "Invalid syntax for the sort parameter: expected expression ending by `:asc` or `:desc`, found `title`.",
+        "code": "invalid_sort",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_sort"
+    });
+    index
+        .search(
+            json!({
+                "sort": ["title"]
+            }),
+            |response, code| {
+                assert_eq!(response, expected_response);
+                assert_eq!(code, 400);
+            },
+        )
+        .await;
+}
+
+#[actix_rt::test]
+async fn sort_unset_ranking_rule() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    index
+        .update_settings(
+            json!({"sortableAttributes": ["title"], "rankingRules": ["proximity", "exactness"]}),
+        )
+        .await;
+
+    let documents = DOCUMENTS.clone();
+    index.add_documents(documents, None).await;
+    index.wait_update_id(1).await;
+
+    let expected_response = json!({
+        "message": "The sort ranking rule must be specified in the ranking rules settings to use the sort parameter at search time.",
+        "code": "invalid_sort",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_sort"
+    });
+    index
+        .search(
+            json!({
+                "sort": ["title:asc"]
+            }),
+            |response, code| {
+                assert_eq!(response, expected_response);
+                assert_eq!(code, 400);
+            },
+        )
+        .await;
+}

meilisearch-http/tests/settings/get_settings.rs

@@ -30,8 +30,8 @@ static DEFAULT_SETTINGS_VALUES: Lazy<HashMap<&'static str, Value>> = Lazy::new(|
 #[actix_rt::test]
 async fn get_settings_unexisting_index() {
     let server = Server::new().await;
-    let (_response, code) = server.index("test").settings().await;
-    assert_eq!(code, 404)
+    let (response, code) = server.index("test").settings().await;
+    assert_eq!(code, 404, "{}", response)
 }
 
 #[actix_rt::test]
@@ -63,7 +63,7 @@ async fn get_settings() {
 }
 
 #[actix_rt::test]
-async fn update_settings_unknown_field() {
+async fn error_update_settings_unknown_field() {
     let server = Server::new().await;
     let index = server.index("test");
     let (_response, code) = index.update_settings(json!({"foo": 12})).await;
@@ -95,10 +95,19 @@ async fn test_partial_update() {
 }
 
 #[actix_rt::test]
-async fn delete_settings_unexisting_index() {
+async fn error_delete_settings_unexisting_index() {
     let server = Server::new().await;
     let index = server.index("test");
-    let (_response, code) = index.delete_settings().await;
+    let (response, code) = index.delete_settings().await;
+
+    let expected_response = json!({
+        "message": "Index `test` not found.",
+        "code": "index_not_found",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#index_not_found"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 404);
 }
 
@@ -164,10 +173,19 @@ async fn update_setting_unexisting_index() {
 }
 
 #[actix_rt::test]
-async fn update_setting_unexisting_index_invalid_uid() {
+async fn error_update_setting_unexisting_index_invalid_uid() {
     let server = Server::new().await;
     let index = server.index("test##!  ");
-    let (_response, code) = index.update_settings(json!({})).await;
+    let (response, code) = index.update_settings(json!({})).await;
+
+    let expected_response = json!({
+        "message": "`test##!  ` is not a valid index uid. Index uid can be an integer or a string containing only alphanumeric characters, hyphens (-) and underscores (_).",
+        "code": "invalid_index_uid",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_index_uid"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 400);
 }
 
@@ -246,3 +264,49 @@ test_setting_routes!(
     ranking_rules,
     synonyms
 );
+
+#[actix_rt::test]
+async fn error_set_invalid_ranking_rules() {
+    let server = Server::new().await;
+    let index = server.index("test");
+    index.create(None).await;
+
+    let (_response, _code) = index
+        .update_settings(json!({ "rankingRules": [ "manyTheFish"]}))
+        .await;
+    index.wait_update_id(0).await;
+    let (response, code) = index.get_update(0).await;
+
+    assert_eq!(code, 200);
+    assert_eq!(response["status"], "failed");
+
+    let expected_error = json!({
+        "message": r#"`manyTheFish` ranking rule is invalid. Valid ranking rules are Words, Typo, Sort, Proximity, Attribute, Exactness and custom ranking rules."#,
+        "code": "invalid_ranking_rule",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#invalid_ranking_rule"
+    });
+
+    assert_eq!(response["error"], expected_error);
+}
+
+#[actix_rt::test]
+async fn set_and_reset_distinct_attribute_with_dedicated_route() {
+    let server = Server::new().await;
+    let index = server.index("test");
+
+    let (_response, _code) = index.update_distinct_attribute(json!("test")).await;
+    index.wait_update_id(0).await;
+
+    let (response, _) = index.get_distinct_attribute().await;
+
+    assert_eq!(response, "test");
+
+    index.update_distinct_attribute(json!(null)).await;
+
+    index.wait_update_id(1).await;
+
+    let (response, _) = index.get_distinct_attribute().await;
+
+    assert_eq!(response, json!(null));
+}

meilisearch-http/tests/snapshot/mod.rs

@@ -9,8 +9,8 @@ use meilisearch_http::Opt;
 
 #[actix_rt::test]
 async fn perform_snapshot() {
-    let temp = tempfile::tempdir_in(".").unwrap();
-    let snapshot_dir = tempfile::tempdir_in(".").unwrap();
+    let temp = tempfile::tempdir().unwrap();
+    let snapshot_dir = tempfile::tempdir().unwrap();
 
     let options = Opt {
         snapshot_dir: snapshot_dir.path().to_owned(),
@@ -29,7 +29,7 @@ async fn perform_snapshot() {
 
     sleep(Duration::from_secs(2)).await;
 
-    let temp = tempfile::tempdir_in(".").unwrap();
+    let temp = tempfile::tempdir().unwrap();
 
     let snapshot_path = snapshot_dir
         .path()

meilisearch-http/tests/updates/mod.rs

@@ -1,18 +1,37 @@
 use crate::common::Server;
+use serde_json::json;
 
 #[actix_rt::test]
-async fn get_update_unexisting_index() {
+async fn error_get_update_unexisting_index() {
     let server = Server::new().await;
-    let (_response, code) = server.index("test").get_update(0).await;
+    let (response, code) = server.index("test").get_update(0).await;
+
+    let expected_response = json!({
+        "message": "Index `test` not found.",
+        "code": "index_not_found",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#index_not_found"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 404);
 }
 
 #[actix_rt::test]
-async fn get_unexisting_update_status() {
+async fn error_get_unexisting_update_status() {
     let server = Server::new().await;
     let index = server.index("test");
     index.create(None).await;
-    let (_response, code) = index.get_update(0).await;
+    let (response, code) = index.get_update(0).await;
+
+    let expected_response = json!({
+        "message": "Task `0` not found.",
+        "code": "task_not_found",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#task_not_found"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 404);
 }
 
@@ -36,9 +55,18 @@ async fn get_update_status() {
 }
 
 #[actix_rt::test]
-async fn list_updates_unexisting_index() {
+async fn error_list_updates_unexisting_index() {
     let server = Server::new().await;
-    let (_response, code) = server.index("test").list_updates().await;
+    let (response, code) = server.index("test").list_updates().await;
+
+    let expected_response = json!({
+        "message": "Index `test` not found.",
+        "code": "index_not_found",
+        "type": "invalid_request",
+        "link": "https://docs.meilisearch.com/errors#index_not_found"
+    });
+
+    assert_eq!(response, expected_response);
     assert_eq!(code, 404);
 }
 

meilisearch-lib/Cargo.toml

@@ -0,0 +1,63 @@
+[package]
+name = "meilisearch-lib"
+version = "0.24.0"
+edition = "2018"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+actix-web = { version = "4.0.0-beta.9", features = ["rustls"] }
+actix-web-static-files = { git = "https://github.com/MarinPostma/actix-web-static-files.git", rev = "39d8006", optional = true }
+anyhow = { version = "1.0.43", features = ["backtrace"] }
+async-stream = "0.3.2"
+async-trait = "0.1.51"
+arc-swap = "1.3.2"
+byte-unit = { version = "4.0.12", default-features = false, features = ["std"] }
+bytes = "1.1.0"
+chrono = { version = "0.4.19", features = ["serde"] }
+csv = "1.1.6"
+crossbeam-channel = "0.5.1"
+either = "1.6.1"
+flate2 = "1.0.21"
+fst = "0.4.7"
+futures = "0.3.17"
+futures-util = "0.3.17"
+heed = { git = "https://github.com/Kerollmops/heed", tag = "v0.12.1" }
+http = "0.2.4"
+indexmap = { version = "1.7.0", features = ["serde-1"] }
+itertools = "0.10.1"
+lazy_static = "1.4.0"
+log = "0.4.14"
+meilisearch-error = { path = "../meilisearch-error" }
+meilisearch-tokenizer = { git = "https://github.com/meilisearch/tokenizer.git", tag = "v0.2.5" }
+milli = { git = "https://github.com/meilisearch/milli.git", tag = "v0.20.2" }
+mime = "0.3.16"
+num_cpus = "1.13.0"
+once_cell = "1.8.0"
+parking_lot = "0.11.2"
+rand = "0.8.4"
+rayon = "1.5.1"
+regex = "1.5.4"
+rustls = "0.19.1"
+serde = { version = "1.0.130", features = ["derive"] }
+serde_json = { version = "1.0.67", features = ["preserve_order"] }
+siphasher = "0.3.7"
+slice-group-by = "0.2.6"
+structopt = "0.3.23"
+tar = "0.4.37"
+tempfile = "3.2.0"
+thiserror = "1.0.28"
+tokio = { version = "1.11.0", features = ["full"] }
+uuid = { version = "0.8.2", features = ["serde"] }
+walkdir = "2.3.2"
+obkv = "0.2.0"
+pin-project = "1.0.8"
+whoami = { version = "1.1.3", optional = true }
+reqwest = { version = "0.11.4", features = ["json", "rustls-tls"], default-features = false, optional = true }
+sysinfo = "0.20.2"
+derivative = "2.2.0"
+
+[dev-dependencies]
+actix-rt = "2.2.0"
+mockall = "0.10.2"
+paste = "1.0.5"

meilisearch-lib/src/analytics.rs

@@ -0,0 +1,8 @@
+use std::{fs, path::Path};
+
+/// Copy the `instance-uid` contained in one db to another. Ignore all errors.
+pub fn copy_user_id(src: &Path, dst: &Path) {
+    if let Ok(user_id) = fs::read_to_string(src.join("instance-uid")) {
+        let _ = fs::write(dst.join("instance-uid"), &user_id);
+    }
+}

meilisearch-lib/src/document_formats.rs

@@ -0,0 +1,114 @@
+use std::fmt;
+use std::io::{self, BufRead, BufReader, BufWriter, Cursor, Read, Seek, Write};
+
+use meilisearch_error::{Code, ErrorCode};
+use milli::documents::DocumentBatchBuilder;
+
+type Result<T> = std::result::Result<T, DocumentFormatError>;
+
+#[derive(Debug)]
+pub enum PayloadType {
+    Ndjson,
+    Json,
+    Csv,
+}
+
+impl fmt::Display for PayloadType {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            PayloadType::Ndjson => write!(f, "ndjson"),
+            PayloadType::Json => write!(f, "json"),
+            PayloadType::Csv => write!(f, "csv"),
+        }
+    }
+}
+
+#[derive(thiserror::Error, Debug)]
+pub enum DocumentFormatError {
+    #[error("An internal error has occurred. `{0}`.")]
+    Internal(Box<dyn std::error::Error + Send + Sync + 'static>),
+    #[error("The `{1}` payload provided is malformed. `{0}`.")]
+    MalformedPayload(
+        Box<dyn std::error::Error + Send + Sync + 'static>,
+        PayloadType,
+    ),
+    #[error("The `{0}` payload must contain at least one document.")]
+    EmptyPayload(PayloadType),
+}
+
+impl From<(PayloadType, milli::documents::Error)> for DocumentFormatError {
+    fn from((ty, error): (PayloadType, milli::documents::Error)) -> Self {
+        match error {
+            milli::documents::Error::Io(e) => Self::Internal(Box::new(e)),
+            e => Self::MalformedPayload(Box::new(e), ty),
+        }
+    }
+}
+
+impl ErrorCode for DocumentFormatError {
+    fn error_code(&self) -> Code {
+        match self {
+            DocumentFormatError::Internal(_) => Code::Internal,
+            DocumentFormatError::MalformedPayload(_, _) => Code::MalformedPayload,
+            DocumentFormatError::EmptyPayload(_) => Code::MalformedPayload,
+        }
+    }
+}
+
+internal_error!(DocumentFormatError: io::Error);
+
+/// reads csv from input and write an obkv batch to writer.
+pub fn read_csv(input: impl Read, writer: impl Write + Seek) -> Result<()> {
+    let writer = BufWriter::new(writer);
+    let builder =
+        DocumentBatchBuilder::from_csv(input, writer).map_err(|e| (PayloadType::Csv, e))?;
+
+    if builder.len() == 0 {
+        return Err(DocumentFormatError::EmptyPayload(PayloadType::Csv));
+    }
+
+    builder.finish().map_err(|e| (PayloadType::Csv, e))?;
+
+    Ok(())
+}
+
+/// reads jsonl from input and write an obkv batch to writer.
+pub fn read_ndjson(input: impl Read, writer: impl Write + Seek) -> Result<()> {
+    let mut reader = BufReader::new(input);
+    let writer = BufWriter::new(writer);
+
+    let mut builder = DocumentBatchBuilder::new(writer).map_err(|e| (PayloadType::Ndjson, e))?;
+    let mut buf = String::new();
+
+    while reader.read_line(&mut buf)? > 0 {
+        builder
+            .extend_from_json(Cursor::new(&buf.as_bytes()))
+            .map_err(|e| (PayloadType::Ndjson, e))?;
+        buf.clear();
+    }
+
+    if builder.len() == 0 {
+        return Err(DocumentFormatError::EmptyPayload(PayloadType::Ndjson));
+    }
+
+    builder.finish().map_err(|e| (PayloadType::Ndjson, e))?;
+
+    Ok(())
+}
+
+/// reads json from input and write an obkv batch to writer.
+pub fn read_json(input: impl Read, writer: impl Write + Seek) -> Result<()> {
+    let writer = BufWriter::new(writer);
+    let mut builder = DocumentBatchBuilder::new(writer).map_err(|e| (PayloadType::Json, e))?;
+    builder
+        .extend_from_json(input)
+        .map_err(|e| (PayloadType::Json, e))?;
+
+    if builder.len() == 0 {
+        return Err(DocumentFormatError::EmptyPayload(PayloadType::Json));
+    }
+
+    builder.finish().map_err(|e| (PayloadType::Json, e))?;
+
+    Ok(())
+}

meilisearch-lib/src/error.rs

@@ -0,0 +1,60 @@
+use std::error::Error;
+use std::fmt;
+
+use meilisearch_error::{Code, ErrorCode};
+use milli::UserError;
+
+macro_rules! internal_error {
+    ($target:ty : $($other:path), *) => {
+        $(
+            impl From<$other> for $target {
+                fn from(other: $other) -> Self {
+                    Self::Internal(Box::new(other))
+                }
+            }
+        )*
+    }
+}
+
+#[derive(Debug)]
+pub struct MilliError<'a>(pub &'a milli::Error);
+
+impl Error for MilliError<'_> {}
+
+impl fmt::Display for MilliError<'_> {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        self.0.fmt(f)
+    }
+}
+
+impl ErrorCode for MilliError<'_> {
+    fn error_code(&self) -> Code {
+        match self.0 {
+            milli::Error::InternalError(_) => Code::Internal,
+            milli::Error::IoError(_) => Code::Internal,
+            milli::Error::UserError(ref error) => {
+                match error {
+                    // TODO: wait for spec for new error codes.
+                    UserError::SerdeJson(_)
+                    | UserError::DocumentLimitReached
+                    | UserError::UnknownInternalDocumentId { .. } => Code::Internal,
+                    UserError::InvalidStoreFile => Code::InvalidStore,
+                    UserError::NoSpaceLeftOnDevice => Code::NoSpaceLeftOnDevice,
+                    UserError::MaxDatabaseSizeReached => Code::DatabaseSizeLimitReached,
+                    UserError::AttributeLimitReached => Code::MaxFieldsLimitExceeded,
+                    UserError::InvalidFilter(_) => Code::Filter,
+                    UserError::MissingDocumentId { .. } => Code::MissingDocumentId,
+                    UserError::InvalidDocumentId { .. } => Code::InvalidDocumentId,
+                    UserError::MissingPrimaryKey => Code::MissingPrimaryKey,
+                    UserError::PrimaryKeyCannotBeChanged(_) => Code::PrimaryKeyAlreadyPresent,
+                    UserError::SortRankingRuleMissing => Code::Sort,
+                    UserError::InvalidFacetsDistribution { .. } => Code::BadRequest,
+                    UserError::InvalidSortableAttribute { .. } => Code::Sort,
+                    UserError::CriterionError(_) => Code::InvalidRankingRule,
+                    UserError::InvalidGeoField { .. } => Code::InvalidGeoField,
+                    UserError::SortError(_) => Code::Sort,
+                }
+            }
+        }
+    }
+}

meilisearch-lib/src/index/dump.rs

@@ -0,0 +1,152 @@
+use std::fs::{create_dir_all, File};
+use std::io::{BufReader, Seek, SeekFrom, Write};
+use std::path::Path;
+
+use anyhow::Context;
+use heed::{EnvOpenOptions, RoTxn};
+use indexmap::IndexMap;
+use milli::documents::DocumentBatchReader;
+use serde::{Deserialize, Serialize};
+
+use crate::document_formats::read_ndjson;
+use crate::index::update_handler::UpdateHandler;
+use crate::index::updates::apply_settings_to_builder;
+
+use super::error::Result;
+use super::{index::Index, Settings, Unchecked};
+
+#[derive(Serialize, Deserialize)]
+struct DumpMeta {
+    settings: Settings<Unchecked>,
+    primary_key: Option<String>,
+}
+
+const META_FILE_NAME: &str = "meta.json";
+const DATA_FILE_NAME: &str = "documents.jsonl";
+
+impl Index {
+    pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
+        // acquire write txn make sure any ongoing write is finished before we start.
+        let txn = self.env.write_txn()?;
+        let path = path
+            .as_ref()
+            .join(format!("indexes/{}", self.uuid.to_string()));
+
+        create_dir_all(&path)?;
+
+        self.dump_documents(&txn, &path)?;
+        self.dump_meta(&txn, &path)?;
+
+        Ok(())
+    }
+
+    fn dump_documents(&self, txn: &RoTxn, path: impl AsRef<Path>) -> Result<()> {
+        let document_file_path = path.as_ref().join(DATA_FILE_NAME);
+        let mut document_file = File::create(&document_file_path)?;
+
+        let documents = self.all_documents(txn)?;
+        let fields_ids_map = self.fields_ids_map(txn)?;
+
+        // dump documents
+        let mut json_map = IndexMap::new();
+        for document in documents {
+            let (_, reader) = document?;
+
+            for (fid, bytes) in reader.iter() {
+                if let Some(name) = fields_ids_map.name(fid) {
+                    json_map.insert(name, serde_json::from_slice::<serde_json::Value>(bytes)?);
+                }
+            }
+
+            serde_json::to_writer(&mut document_file, &json_map)?;
+            document_file.write_all(b"\n")?;
+
+            json_map.clear();
+        }
+
+        Ok(())
+    }
+
+    fn dump_meta(&self, txn: &RoTxn, path: impl AsRef<Path>) -> Result<()> {
+        let meta_file_path = path.as_ref().join(META_FILE_NAME);
+        let mut meta_file = File::create(&meta_file_path)?;
+
+        let settings = self.settings_txn(txn)?.into_unchecked();
+        let primary_key = self.primary_key(txn)?.map(String::from);
+        let meta = DumpMeta {
+            settings,
+            primary_key,
+        };
+
+        serde_json::to_writer(&mut meta_file, &meta)?;
+
+        Ok(())
+    }
+
+    pub fn load_dump(
+        src: impl AsRef<Path>,
+        dst: impl AsRef<Path>,
+        size: usize,
+        update_handler: &UpdateHandler,
+    ) -> anyhow::Result<()> {
+        let dir_name = src
+            .as_ref()
+            .file_name()
+            .with_context(|| format!("invalid dump index: {}", src.as_ref().display()))?;
+
+        let dst_dir_path = dst.as_ref().join("indexes").join(dir_name);
+        create_dir_all(&dst_dir_path)?;
+
+        let meta_path = src.as_ref().join(META_FILE_NAME);
+        let meta_file = File::open(meta_path)?;
+        let DumpMeta {
+            settings,
+            primary_key,
+        } = serde_json::from_reader(meta_file)?;
+        let settings = settings.check();
+
+        let mut options = EnvOpenOptions::new();
+        options.map_size(size);
+        let index = milli::Index::new(options, &dst_dir_path)?;
+
+        let mut txn = index.write_txn()?;
+
+        // Apply settings first
+        let builder = update_handler.update_builder(0);
+        let mut builder = builder.settings(&mut txn, &index);
+
+        if let Some(primary_key) = primary_key {
+            builder.set_primary_key(primary_key);
+        }
+
+        apply_settings_to_builder(&settings, &mut builder);
+
+        builder.execute(|_, _| ())?;
+
+        let document_file_path = src.as_ref().join(DATA_FILE_NAME);
+        let reader = BufReader::new(File::open(&document_file_path)?);
+
+        let mut tmp_doc_file = tempfile::tempfile()?;
+
+        read_ndjson(reader, &mut tmp_doc_file)?;
+
+        tmp_doc_file.seek(SeekFrom::Start(0))?;
+
+        let documents_reader = DocumentBatchReader::from_reader(tmp_doc_file)?;
+
+        //If the document file is empty, we don't perform the document addition, to prevent
+        //a primary key error to be thrown.
+        if !documents_reader.is_empty() {
+            let builder = update_handler
+                .update_builder(0)
+                .index_documents(&mut txn, &index);
+            builder.execute(documents_reader, |_, _| ())?;
+        }
+
+        txn.commit()?;
+
+        index.prepare_for_closing().wait();
+
+        Ok(())
+    }
+}

meilisearch-lib/src/index/error.rs

@@ -9,9 +9,9 @@ pub type Result<T> = std::result::Result<T, IndexError>;
 
 #[derive(Debug, thiserror::Error)]
 pub enum IndexError {
-    #[error("Internal error: {0}")]
+    #[error("An internal error has occurred. `{0}`.")]
     Internal(Box<dyn Error + Send + Sync + 'static>),
-    #[error("Document with id {0} not found.")]
+    #[error("Document `{0}` not found.")]
     DocumentNotFound(String),
     #[error("{0}")]
     Facet(#[from] FacetError),
@@ -39,14 +39,14 @@ impl ErrorCode for IndexError {
 
 #[derive(Debug, thiserror::Error)]
 pub enum FacetError {
-    #[error("Invalid facet expression, expected {}, found: {1}", .0.join(", "))]
+    #[error("Invalid syntax for the filter parameter: `expected {}, found: {1}`.", .0.join(", "))]
     InvalidExpression(&'static [&'static str], Value),
 }
 
 impl ErrorCode for FacetError {
     fn error_code(&self) -> Code {
         match self {
-            FacetError::InvalidExpression(_, _) => Code::Facet,
+            FacetError::InvalidExpression(_, _) => Code::Filter,
         }
     }
 }

meilisearch-lib/src/index/index.rs

@@ -5,53 +5,130 @@ use std::ops::Deref;
 use std::path::Path;
 use std::sync::Arc;
 
+use chrono::{DateTime, Utc};
 use heed::{EnvOpenOptions, RoTxn};
 use milli::update::Setting;
-use milli::{obkv_to_json, FieldId};
+use milli::{obkv_to_json, FieldDistribution, FieldId};
+use serde::{Deserialize, Serialize};
 use serde_json::{Map, Value};
+use uuid::Uuid;
 
-use error::Result;
-pub use search::{default_crop_length, SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT};
-pub use updates::{Checked, Facets, Settings, Unchecked};
+use crate::index_controller::update_file_store::UpdateFileStore;
+use crate::EnvSizer;
 
-use crate::helpers::EnvSizer;
-
-use self::error::IndexError;
-
-pub mod error;
-pub mod update_handler;
-
-mod dump;
-mod search;
-mod updates;
+use super::error::IndexError;
+use super::error::Result;
+use super::update_handler::UpdateHandler;
+use super::{Checked, Settings};
 
 pub type Document = Map<String, Value>;
 
-#[derive(Clone)]
-pub struct Index(pub Arc<milli::Index>);
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct IndexMeta {
+    created_at: DateTime<Utc>,
+    pub updated_at: DateTime<Utc>,
+    pub primary_key: Option<String>,
+}
+
+impl IndexMeta {
+    pub fn new(index: &Index) -> Result<Self> {
+        let txn = index.read_txn()?;
+        Self::new_txn(index, &txn)
+    }
+
+    pub fn new_txn(index: &Index, txn: &heed::RoTxn) -> Result<Self> {
+        let created_at = index.created_at(txn)?;
+        let updated_at = index.updated_at(txn)?;
+        let primary_key = index.primary_key(txn)?.map(String::from);
+        Ok(Self {
+            created_at,
+            updated_at,
+            primary_key,
+        })
+    }
+}
+
+#[derive(Serialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct IndexStats {
+    #[serde(skip)]
+    pub size: u64,
+    pub number_of_documents: u64,
+    /// Whether the current index is performing an update. It is initially `None` when the
+    /// index returns it, since it is the `UpdateStore` that knows what index is currently indexing. It is
+    /// later set to either true or false, we we retrieve the information from the `UpdateStore`
+    pub is_indexing: Option<bool>,
+    pub field_distribution: FieldDistribution,
+}
+
+#[derive(Clone, derivative::Derivative)]
+#[derivative(Debug)]
+pub struct Index {
+    pub uuid: Uuid,
+    #[derivative(Debug = "ignore")]
+    pub inner: Arc<milli::Index>,
+    #[derivative(Debug = "ignore")]
+    pub update_file_store: Arc<UpdateFileStore>,
+    #[derivative(Debug = "ignore")]
+    pub update_handler: Arc<UpdateHandler>,
+}
 
 impl Deref for Index {
     type Target = milli::Index;
 
     fn deref(&self) -> &Self::Target {
-        self.0.as_ref()
+        self.inner.as_ref()
     }
 }
 
 impl Index {
-    pub fn open(path: impl AsRef<Path>, size: usize) -> Result<Self> {
+    pub fn open(
+        path: impl AsRef<Path>,
+        size: usize,
+        update_file_store: Arc<UpdateFileStore>,
+        uuid: Uuid,
+        update_handler: Arc<UpdateHandler>,
+    ) -> Result<Self> {
         create_dir_all(&path)?;
         let mut options = EnvOpenOptions::new();
         options.map_size(size);
-        let index = milli::Index::new(options, &path)?;
-        Ok(Index(Arc::new(index)))
+        let inner = Arc::new(milli::Index::new(options, &path)?);
+        Ok(Index {
+            inner,
+            update_file_store,
+            uuid,
+            update_handler,
+        })
     }
 
+    pub fn inner(&self) -> &milli::Index {
+        &self.inner
+    }
+
+    pub fn stats(&self) -> Result<IndexStats> {
+        let rtxn = self.read_txn()?;
+
+        Ok(IndexStats {
+            size: self.size(),
+            number_of_documents: self.number_of_documents(&rtxn)?,
+            is_indexing: None,
+            field_distribution: self.field_distribution(&rtxn)?,
+        })
+    }
+
+    pub fn meta(&self) -> Result<IndexMeta> {
+        IndexMeta::new(self)
+    }
     pub fn settings(&self) -> Result<Settings<Checked>> {
         let txn = self.read_txn()?;
         self.settings_txn(&txn)
     }
 
+    pub fn uuid(&self) -> Uuid {
+        self.uuid
+    }
+
     pub fn settings_txn(&self, txn: &RoTxn) -> Result<Settings<Checked>> {
         let displayed_attributes = self
             .displayed_fields(txn)?
@@ -195,4 +272,15 @@ impl Index {
         displayed_fields_ids.retain(|fid| attributes_to_retrieve_ids.contains(fid));
         Ok(displayed_fields_ids)
     }
+
+    pub fn snapshot(&self, path: impl AsRef<Path>) -> Result<()> {
+        let mut dst = path.as_ref().join(format!("indexes/{}/", self.uuid));
+        create_dir_all(&dst)?;
+        dst.push("data.mdb");
+        let _txn = self.write_txn()?;
+        self.inner
+            .env
+            .copy_to_path(dst, heed::CompactionOption::Enabled)?;
+        Ok(())
+    }
 }

meilisearch-lib/src/index/mod.rs

@@ -0,0 +1,365 @@
+pub use search::{default_crop_length, SearchQuery, SearchResult, DEFAULT_SEARCH_LIMIT};
+pub use updates::{apply_settings_to_builder, Checked, Facets, Settings, Unchecked};
+
+mod dump;
+pub mod error;
+mod search;
+pub mod update_handler;
+mod updates;
+
+#[allow(clippy::module_inception)]
+mod index;
+
+pub use index::{Document, IndexMeta, IndexStats};
+
+#[cfg(not(test))]
+pub use index::Index;
+
+#[cfg(test)]
+pub use test::MockIndex as Index;
+
+/// The index::test module provides means of mocking an index instance. I can be used throughout the
+/// code for unit testing, in places where an index would normally be used.
+#[cfg(test)]
+pub mod test {
+    use std::any::Any;
+    use std::collections::HashMap;
+    use std::panic::{RefUnwindSafe, UnwindSafe};
+    use std::path::Path;
+    use std::path::PathBuf;
+    use std::sync::atomic::{AtomicBool, Ordering};
+    use std::sync::{Arc, Mutex};
+
+    use serde_json::{Map, Value};
+    use uuid::Uuid;
+
+    use crate::index_controller::update_file_store::UpdateFileStore;
+    use crate::index_controller::updates::status::{Failed, Processed, Processing};
+
+    use super::error::Result;
+    use super::index::Index;
+    use super::update_handler::UpdateHandler;
+    use super::{Checked, IndexMeta, IndexStats, SearchQuery, SearchResult, Settings};
+
+    pub struct Stub<A, R> {
+        name: String,
+        times: Mutex<Option<usize>>,
+        stub: Box<dyn Fn(A) -> R + Sync + Send>,
+        invalidated: AtomicBool,
+    }
+
+    impl<A, R> Drop for Stub<A, R> {
+        fn drop(&mut self) {
+            if !self.invalidated.load(Ordering::Relaxed) {
+                let lock = self.times.lock().unwrap();
+                if let Some(n) = *lock {
+                    assert_eq!(n, 0, "{} not called enough times", self.name);
+                }
+            }
+        }
+    }
+
+    impl<A, R> Stub<A, R> {
+        fn invalidate(&self) {
+            self.invalidated.store(true, Ordering::Relaxed);
+        }
+    }
+
+    impl<A: UnwindSafe, R> Stub<A, R> {
+        fn call(&self, args: A) -> R {
+            let mut lock = self.times.lock().unwrap();
+            match *lock {
+                Some(0) => panic!("{} called to many times", self.name),
+                Some(ref mut times) => {
+                    *times -= 1;
+                }
+                None => (),
+            }
+
+            // Since we add assertions in the drop implementation for Stub, a panic can occur in a
+            // panic, causing a hard abort of the program. To handle that, we catch the panic, and
+            // set the stub as invalidated so the assertions aren't run during the drop.
+            impl<'a, A, R> RefUnwindSafe for StubHolder<'a, A, R> {}
+            struct StubHolder<'a, A, R>(&'a (dyn Fn(A) -> R + Sync + Send));
+
+            let stub = StubHolder(self.stub.as_ref());
+
+            match std::panic::catch_unwind(|| (stub.0)(args)) {
+                Ok(r) => r,
+                Err(panic) => {
+                    self.invalidate();
+                    std::panic::resume_unwind(panic);
+                }
+            }
+        }
+    }
+
+    #[derive(Debug, Default)]
+    struct StubStore {
+        inner: Arc<Mutex<HashMap<String, Box<dyn Any + Sync + Send>>>>,
+    }
+
+    impl StubStore {
+        pub fn insert<A: 'static, R: 'static>(&self, name: String, stub: Stub<A, R>) {
+            let mut lock = self.inner.lock().unwrap();
+            lock.insert(name, Box::new(stub));
+        }
+
+        pub fn get<A, B>(&self, name: &str) -> Option<&Stub<A, B>> {
+            let mut lock = self.inner.lock().unwrap();
+            match lock.get_mut(name) {
+                Some(s) => {
+                    let s = s.as_mut() as *mut dyn Any as *mut Stub<A, B>;
+                    Some(unsafe { &mut *s })
+                }
+                None => None,
+            }
+        }
+    }
+
+    pub struct StubBuilder<'a, A, R> {
+        name: String,
+        store: &'a StubStore,
+        times: Option<usize>,
+        _f: std::marker::PhantomData<fn(A) -> R>,
+    }
+
+    impl<'a, A: 'static, R: 'static> StubBuilder<'a, A, R> {
+        /// Asserts the stub has been called exactly `times` times.
+        #[must_use]
+        pub fn times(mut self, times: usize) -> Self {
+            self.times = Some(times);
+            self
+        }
+
+        /// Asserts the stub has been called exactly once.
+        #[must_use]
+        pub fn once(mut self) -> Self {
+            self.times = Some(1);
+            self
+        }
+
+        /// The function that will be called when the stub is called. This needs to be called to
+        /// actually build the stub and register it to the stub store.
+        pub fn then(self, f: impl Fn(A) -> R + Sync + Send + 'static) {
+            let times = Mutex::new(self.times);
+            let stub = Stub {
+                stub: Box::new(f),
+                times,
+                name: self.name.clone(),
+                invalidated: AtomicBool::new(false),
+            };
+
+            self.store.insert(self.name, stub);
+        }
+    }
+
+    /// Mocker allows to stub metod call on any struct. you can register stubs by calling
+    /// `Mocker::when` and retrieve it in the proxy implementation when with `Mocker::get`.
+    #[derive(Debug, Default)]
+    pub struct Mocker {
+        store: StubStore,
+    }
+
+    impl Mocker {
+        pub fn when<A, R>(&self, name: &str) -> StubBuilder<A, R> {
+            StubBuilder {
+                name: name.to_string(),
+                store: &self.store,
+                times: None,
+                _f: std::marker::PhantomData,
+            }
+        }
+
+        pub fn get<A, R>(&self, name: &str) -> &Stub<A, R> {
+            match self.store.get(name) {
+                Some(stub) => stub,
+                None => {
+                    // panic here causes the stubs to get dropped, and panic in turn. To prevent
+                    // that, we forget them, and let them be cleaned by the os later. This is not
+                    // optimal, but is still better than nested panicks.
+                    let mut stubs = self.store.inner.lock().unwrap();
+                    let stubs = std::mem::take(&mut *stubs);
+                    std::mem::forget(stubs);
+                    panic!("unexpected call to {}", name)
+                }
+            }
+        }
+    }
+
+    #[derive(Debug, Clone)]
+    pub enum MockIndex {
+        Vrai(Index),
+        Faux(Arc<Mocker>),
+    }
+
+    impl MockIndex {
+        pub fn faux(faux: Mocker) -> Self {
+            Self::Faux(Arc::new(faux))
+        }
+
+        pub fn open(
+            path: impl AsRef<Path>,
+            size: usize,
+            update_file_store: Arc<UpdateFileStore>,
+            uuid: Uuid,
+            update_handler: Arc<UpdateHandler>,
+        ) -> Result<Self> {
+            let index = Index::open(path, size, update_file_store, uuid, update_handler)?;
+            Ok(Self::Vrai(index))
+        }
+
+        pub fn load_dump(
+            src: impl AsRef<Path>,
+            dst: impl AsRef<Path>,
+            size: usize,
+            update_handler: &UpdateHandler,
+        ) -> anyhow::Result<()> {
+            Index::load_dump(src, dst, size, update_handler)?;
+            Ok(())
+        }
+
+        pub fn handle_update(&self, update: Processing) -> std::result::Result<Processed, Failed> {
+            match self {
+                MockIndex::Vrai(index) => index.handle_update(update),
+                MockIndex::Faux(faux) => faux.get("handle_update").call(update),
+            }
+        }
+
+        pub fn uuid(&self) -> Uuid {
+            match self {
+                MockIndex::Vrai(index) => index.uuid(),
+                MockIndex::Faux(faux) => faux.get("uuid").call(()),
+            }
+        }
+
+        pub fn stats(&self) -> Result<IndexStats> {
+            match self {
+                MockIndex::Vrai(index) => index.stats(),
+                MockIndex::Faux(_) => todo!(),
+            }
+        }
+
+        pub fn meta(&self) -> Result<IndexMeta> {
+            match self {
+                MockIndex::Vrai(index) => index.meta(),
+                MockIndex::Faux(_) => todo!(),
+            }
+        }
+        pub fn settings(&self) -> Result<Settings<Checked>> {
+            match self {
+                MockIndex::Vrai(index) => index.settings(),
+                MockIndex::Faux(_) => todo!(),
+            }
+        }
+
+        pub fn retrieve_documents<S: AsRef<str>>(
+            &self,
+            offset: usize,
+            limit: usize,
+            attributes_to_retrieve: Option<Vec<S>>,
+        ) -> Result<Vec<Map<String, Value>>> {
+            match self {
+                MockIndex::Vrai(index) => {
+                    index.retrieve_documents(offset, limit, attributes_to_retrieve)
+                }
+                MockIndex::Faux(_) => todo!(),
+            }
+        }
+
+        pub fn retrieve_document<S: AsRef<str>>(
+            &self,
+            doc_id: String,
+            attributes_to_retrieve: Option<Vec<S>>,
+        ) -> Result<Map<String, Value>> {
+            match self {
+                MockIndex::Vrai(index) => index.retrieve_document(doc_id, attributes_to_retrieve),
+                MockIndex::Faux(_) => todo!(),
+            }
+        }
+
+        pub fn size(&self) -> u64 {
+            match self {
+                MockIndex::Vrai(index) => index.size(),
+                MockIndex::Faux(_) => todo!(),
+            }
+        }
+
+        pub fn snapshot(&self, path: impl AsRef<Path>) -> Result<()> {
+            match self {
+                MockIndex::Vrai(index) => index.snapshot(path),
+                MockIndex::Faux(faux) => faux.get("snapshot").call(path.as_ref()),
+            }
+        }
+
+        pub fn inner(&self) -> &milli::Index {
+            match self {
+                MockIndex::Vrai(index) => index.inner(),
+                MockIndex::Faux(_) => todo!(),
+            }
+        }
+
+        pub fn update_primary_key(&self, primary_key: Option<String>) -> Result<IndexMeta> {
+            match self {
+                MockIndex::Vrai(index) => index.update_primary_key(primary_key),
+                MockIndex::Faux(_) => todo!(),
+            }
+        }
+        pub fn perform_search(&self, query: SearchQuery) -> Result<SearchResult> {
+            match self {
+                MockIndex::Vrai(index) => index.perform_search(query),
+                MockIndex::Faux(faux) => faux.get("perform_search").call(query),
+            }
+        }
+
+        pub fn dump(&self, path: impl AsRef<Path>) -> Result<()> {
+            match self {
+                MockIndex::Vrai(index) => index.dump(path),
+                MockIndex::Faux(faux) => faux.get("dump").call(path.as_ref()),
+            }
+        }
+    }
+
+    #[test]
+    fn test_faux_index() {
+        let faux = Mocker::default();
+        faux.when("snapshot")
+            .times(2)
+            .then(|_: &Path| -> Result<()> { Ok(()) });
+
+        let index = MockIndex::faux(faux);
+
+        let path = PathBuf::from("hello");
+        index.snapshot(&path).unwrap();
+        index.snapshot(&path).unwrap();
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_faux_unexisting_method_stub() {
+        let faux = Mocker::default();
+
+        let index = MockIndex::faux(faux);
+
+        let path = PathBuf::from("hello");
+        index.snapshot(&path).unwrap();
+        index.snapshot(&path).unwrap();
+    }
+
+    #[test]
+    #[should_panic]
+    fn test_faux_panic() {
+        let faux = Mocker::default();
+        faux.when("snapshot")
+            .times(2)
+            .then(|_: &Path| -> Result<()> {
+                panic!();
+            });
+
+        let index = MockIndex::faux(faux);
+
+        let path = PathBuf::from("hello");
+        index.snapshot(&path).unwrap();
+        index.snapshot(&path).unwrap();
+    }
+}

meilisearch-lib/src/index/search.rs

@@ -6,20 +6,20 @@ use either::Either;
 use heed::RoTxn;
 use indexmap::IndexMap;
 use meilisearch_tokenizer::{Analyzer, AnalyzerConfig, Token};
-use milli::{AscDesc, FieldId, FieldsIdsMap, FilterCondition, MatchingWords, UserError};
+use milli::{AscDesc, FieldId, FieldsIdsMap, FilterCondition, MatchingWords, SortError};
+use regex::Regex;
 use serde::{Deserialize, Serialize};
-use serde_json::Value;
+use serde_json::{json, Value};
 
 use crate::index::error::FacetError;
-use crate::index::IndexError;
 
-use super::error::Result;
-use super::Index;
+use super::error::{IndexError, Result};
+use super::index::Index;
 
 pub type Document = IndexMap<String, Value>;
 type MatchesInfo = BTreeMap<String, Vec<MatchInfo>>;
 
-#[derive(Serialize, Debug, Clone)]
+#[derive(Serialize, Debug, Clone, PartialEq)]
 pub struct MatchInfo {
     start: usize,
     length: usize,
@@ -35,7 +35,7 @@ pub const fn default_crop_length() -> usize {
     DEFAULT_CROP_LENGTH
 }
 
-#[derive(Deserialize, Debug)]
+#[derive(Deserialize, Debug, Clone, PartialEq)]
 #[serde(rename_all = "camelCase", deny_unknown_fields)]
 pub struct SearchQuery {
     pub q: Option<String>,
@@ -55,7 +55,7 @@ pub struct SearchQuery {
     pub facets_distribution: Option<Vec<String>>,
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone, Serialize, PartialEq)]
 pub struct SearchHit {
     #[serde(flatten)]
     pub document: Document,
@@ -65,7 +65,7 @@ pub struct SearchHit {
     pub matches_info: Option<MatchesInfo>,
 }
 
-#[derive(Serialize, Debug)]
+#[derive(Serialize, Debug, Clone, PartialEq)]
 #[serde(rename_all = "camelCase")]
 pub struct SearchResult {
     pub hits: Vec<SearchHit>,
@@ -110,12 +110,9 @@ impl Index {
         if let Some(ref sort) = query.sort {
             let sort = match sort.iter().map(|s| AscDesc::from_str(s)).collect() {
                 Ok(sorts) => sorts,
-                Err(UserError::InvalidAscDescSyntax { name }) => {
-                    return Err(IndexError::Milli(
-                        UserError::InvalidSortName { name }.into(),
-                    ))
+                Err(asc_desc_error) => {
+                    return Err(IndexError::Milli(SortError::from(asc_desc_error).into()))
                 }
-                Err(err) => return Err(IndexError::Milli(err.into())),
             };
 
             search.sort_criteria(sort);
@@ -193,7 +190,7 @@ impl Index {
         let documents_iter = self.documents(&rtxn, documents_ids)?;
 
         for (_id, obkv) in documents_iter {
-            let document = make_document(&to_retrieve_ids, &fields_ids_map, obkv)?;
+            let mut document = make_document(&to_retrieve_ids, &fields_ids_map, obkv)?;
 
             let matches_info = query
                 .matches
@@ -207,6 +204,10 @@ impl Index {
                 &formatted_options,
             )?;
 
+            if let Some(sort) = query.sort.as_ref() {
+                insert_geo_distance(sort, &mut document);
+            }
+
             let hit = SearchHit {
                 document,
                 formatted,
@@ -247,6 +248,25 @@ impl Index {
     }
 }
 
+fn insert_geo_distance(sorts: &[String], document: &mut Document) {
+    lazy_static::lazy_static! {
+        static ref GEO_REGEX: Regex =
+            Regex::new(r"_geoPoint\(\s*([[:digit:].\-]+)\s*,\s*([[:digit:].\-]+)\s*\)").unwrap();
+    };
+    if let Some(capture_group) = sorts.iter().find_map(|sort| GEO_REGEX.captures(sort)) {
+        // TODO: TAMO: milli encountered an internal error, what do we want to do?
+        let base = [
+            capture_group[1].parse().unwrap(),
+            capture_group[2].parse().unwrap(),
+        ];
+        let geo_point = &document.get("_geo").unwrap_or(&json!(null));
+        if let Some((lat, lng)) = geo_point["lat"].as_f64().zip(geo_point["lng"].as_f64()) {
+            let distance = milli::distance_between_two_points(&base, &[lat, lng]);
+            document.insert("_geoDistance".to_string(), json!(distance.round() as usize));
+        }
+    }
+}
+
 fn compute_matches<A: AsRef<[u8]>>(
     matcher: &impl Matcher,
     document: &Document,
@@ -534,6 +554,11 @@ impl<'a, A: AsRef<[u8]>> Formatter<'a, A> {
                     })
                     .collect(),
             ),
+            Value::Number(number) => {
+                let number_string_value =
+                    self.format_string(number.to_string(), matcher, format_options);
+                Value::String(number_string_value)
+            }
             value => value,
         }
     }
@@ -662,7 +687,7 @@ fn parse_filter_array(
         }
     }
 
-    Ok(FilterCondition::from_array(txn, &index.0, ands)?)
+    Ok(FilterCondition::from_array(txn, index, ands)?)
 }
 
 #[cfg(test)]
@@ -768,6 +793,92 @@ mod test {
         assert_eq!(value["author"], "J. R. R. Tolkien");
     }
 
+    #[test]
+    fn formatted_with_highlight_in_number() {
+        let stop_words = fst::Set::default();
+        let mut config = AnalyzerConfig::default();
+        config.stop_words(&stop_words);
+        let analyzer = Analyzer::new(config);
+        let formatter = Formatter::new(&analyzer, (String::from("<em>"), String::from("</em>")));
+
+        let mut fields = FieldsIdsMap::new();
+        let title = fields.insert("title").unwrap();
+        let author = fields.insert("author").unwrap();
+        let publication_year = fields.insert("publication_year").unwrap();
+
+        let mut buf = Vec::new();
+        let mut obkv = obkv::KvWriter::new(&mut buf);
+
+        obkv.insert(
+            title,
+            Value::String("The Hobbit".into()).to_string().as_bytes(),
+        )
+        .unwrap();
+
+        obkv.finish().unwrap();
+        obkv = obkv::KvWriter::new(&mut buf);
+
+        obkv.insert(
+            author,
+            Value::String("J. R. R. Tolkien".into())
+                .to_string()
+                .as_bytes(),
+        )
+        .unwrap();
+        obkv.finish().unwrap();
+
+        obkv = obkv::KvWriter::new(&mut buf);
+
+        obkv.insert(
+            publication_year,
+            Value::Number(1937.into()).to_string().as_bytes(),
+        )
+        .unwrap();
+
+        obkv.finish().unwrap();
+
+        let obkv = obkv::KvReader::new(&buf);
+
+        let mut formatted_options = BTreeMap::new();
+        formatted_options.insert(
+            title,
+            FormatOptions {
+                highlight: false,
+                crop: None,
+            },
+        );
+        formatted_options.insert(
+            author,
+            FormatOptions {
+                highlight: false,
+                crop: None,
+            },
+        );
+        formatted_options.insert(
+            publication_year,
+            FormatOptions {
+                highlight: true,
+                crop: None,
+            },
+        );
+
+        let mut matching_words = BTreeMap::new();
+        matching_words.insert("1937", Some(4));
+
+        let value = format_fields(
+            &fields,
+            obkv,
+            &formatter,
+            &matching_words,
+            &formatted_options,
+        )
+        .unwrap();
+
+        assert_eq!(value["title"], "The Hobbit");
+        assert_eq!(value["author"], "J. R. R. Tolkien");
+        assert_eq!(value["publication_year"], "<em>1937</em>");
+    }
+
     /// https://github.com/meilisearch/MeiliSearch/issues/1368
     #[test]
     fn formatted_with_highlight_emoji() {
@@ -1332,4 +1443,65 @@ mod test {
             r##"{"about": [MatchInfo { start: 0, length: 6 }, MatchInfo { start: 31, length: 7 }, MatchInfo { start: 191, length: 7 }, MatchInfo { start: 225, length: 7 }, MatchInfo { start: 233, length: 6 }], "color": [MatchInfo { start: 0, length: 3 }]}"##
         );
     }
+
+    #[test]
+    fn test_insert_geo_distance() {
+        let value: Document = serde_json::from_str(
+            r#"{
+      "_geo": {
+        "lat": 50.629973371633746,
+        "lng": 3.0569447399419567
+      },
+      "city": "Lille",
+      "id": "1"
+    }"#,
+        )
+        .unwrap();
+
+        let sorters = &["_geoPoint(50.629973371633746,3.0569447399419567):desc".to_string()];
+        let mut document = value.clone();
+        insert_geo_distance(sorters, &mut document);
+        assert_eq!(document.get("_geoDistance"), Some(&json!(0)));
+
+        let sorters = &["_geoPoint(50.629973371633746, 3.0569447399419567):asc".to_string()];
+        let mut document = value.clone();
+        insert_geo_distance(sorters, &mut document);
+        assert_eq!(document.get("_geoDistance"), Some(&json!(0)));
+
+        let sorters =
+            &["_geoPoint(   50.629973371633746   ,  3.0569447399419567   ):desc".to_string()];
+        let mut document = value.clone();
+        insert_geo_distance(sorters, &mut document);
+        assert_eq!(document.get("_geoDistance"), Some(&json!(0)));
+
+        let sorters = &[
+            "prix:asc",
+            "villeneuve:desc",
+            "_geoPoint(50.629973371633746, 3.0569447399419567):asc",
+            "ubu:asc",
+        ]
+        .map(|s| s.to_string());
+        let mut document = value.clone();
+        insert_geo_distance(sorters, &mut document);
+        assert_eq!(document.get("_geoDistance"), Some(&json!(0)));
+
+        // only the first geoPoint is used to compute the distance
+        let sorters = &[
+            "chien:desc",
+            "_geoPoint(50.629973371633746, 3.0569447399419567):asc",
+            "pangolin:desc",
+            "_geoPoint(100.0, -80.0):asc",
+            "chat:asc",
+        ]
+        .map(|s| s.to_string());
+        let mut document = value.clone();
+        insert_geo_distance(sorters, &mut document);
+        assert_eq!(document.get("_geoDistance"), Some(&json!(0)));
+
+        // there was no _geoPoint so nothing is inserted in the document
+        let sorters = &["chien:asc".to_string()];
+        let mut document = value;
+        insert_geo_distance(sorters, &mut document);
+        assert_eq!(document.get("_geoDistance"), None);
+    }
 }

meilisearch-lib/src/index/update_handler.rs

@@ -1,13 +1,8 @@
-use std::fs::File;
-
-use crate::index::Index;
 use milli::update::UpdateBuilder;
 use milli::CompressionType;
 use rayon::ThreadPool;
 
-use crate::index_controller::UpdateMeta;
-use crate::index_controller::{Failed, Processed, Processing};
-use crate::option::IndexerOpts;
+use crate::options::IndexerOpts;
 
 pub struct UpdateHandler {
     max_nb_chunks: Option<usize>,
@@ -51,39 +46,4 @@ impl UpdateHandler {
         update_builder.chunk_compression_type(self.chunk_compression_type);
         update_builder
     }
-
-    pub fn handle_update(
-        &self,
-        meta: Processing,
-        content: Option<File>,
-        index: Index,
-    ) -> Result<Processed, Failed> {
-        use UpdateMeta::*;
-
-        let update_id = meta.id();
-
-        let update_builder = self.update_builder(update_id);
-
-        let result = match meta.meta() {
-            DocumentsAddition {
-                method,
-                format,
-                primary_key,
-            } => index.update_documents(
-                *format,
-                *method,
-                content,
-                update_builder,
-                primary_key.as_deref(),
-            ),
-            ClearDocuments => index.clear_documents(update_builder),
-            DeleteDocuments { ids } => index.delete_documents(ids, update_builder),
-            Settings(settings) => index.update_settings(&settings.clone().check(), update_builder),
-        };
-
-        match result {
-            Ok(result) => Ok(meta.process(result)),
-            Err(e) => Err(meta.fail(e.into())),
-        }
-    }
 }

meilisearch-lib/src/index/updates.rs

@@ -1,17 +1,18 @@
-use std::collections::{BTreeMap, BTreeSet, HashSet};
-use std::io;
+use std::collections::{BTreeMap, BTreeSet};
 use std::marker::PhantomData;
 use std::num::NonZeroUsize;
 
-use flate2::read::GzDecoder;
 use log::{debug, info, trace};
-use milli::update::{IndexDocumentsMethod, Setting, UpdateBuilder, UpdateFormat};
+use milli::documents::DocumentBatchReader;
+use milli::update::{IndexDocumentsMethod, Setting, UpdateBuilder};
 use serde::{Deserialize, Serialize, Serializer};
+use uuid::Uuid;
 
-use crate::index_controller::UpdateResult;
+use crate::index_controller::updates::status::{Failed, Processed, Processing, UpdateResult};
+use crate::Update;
 
 use super::error::Result;
-use super::Index;
+use super::index::{Index, IndexMeta};
 
 fn serialize_with_wildcard<S>(
     field: &Setting<Vec<String>>,
@@ -35,6 +36,9 @@ pub struct Checked;
 #[derive(Clone, Default, Debug, Serialize, Deserialize)]
 pub struct Unchecked;
 
+/// Holds all the settings for an index. `T` can either be `Checked` if they represents settings
+/// whose validity is guaranteed, or `Unchecked` if they need to be validated. In the later case, a
+/// call to `check` will return a `Settings<Checked>` from a `Settings<Unchecked>`.
 #[derive(Debug, Clone, Default, Serialize, Deserialize)]
 #[serde(deny_unknown_fields)]
 #[serde(rename_all = "camelCase")]
@@ -160,33 +164,83 @@ pub struct Facets {
 }
 
 impl Index {
-    pub fn update_documents(
-        &self,
-        format: UpdateFormat,
-        method: IndexDocumentsMethod,
-        content: Option<impl io::Read>,
-        update_builder: UpdateBuilder,
-        primary_key: Option<&str>,
-    ) -> Result<UpdateResult> {
-        let mut txn = self.write_txn()?;
-        let result = self.update_documents_txn(
-            &mut txn,
-            format,
-            method,
-            content,
-            update_builder,
-            primary_key,
-        )?;
-        txn.commit()?;
-        Ok(result)
+    pub fn handle_update(&self, update: Processing) -> std::result::Result<Processed, Failed> {
+        let update_id = update.id();
+        let update_builder = self.update_handler.update_builder(update_id);
+        let result = (|| {
+            let mut txn = self.write_txn()?;
+            let result = match update.meta() {
+                Update::DocumentAddition {
+                    primary_key,
+                    content_uuid,
+                    method,
+                } => self.update_documents(
+                    &mut txn,
+                    *method,
+                    *content_uuid,
+                    update_builder,
+                    primary_key.as_deref(),
+                ),
+                Update::Settings(settings) => {
+                    let settings = settings.clone().check();
+                    self.update_settings(&mut txn, &settings, update_builder)
+                }
+                Update::ClearDocuments => {
+                    let builder = update_builder.clear_documents(&mut txn, self);
+                    let _count = builder.execute()?;
+                    Ok(UpdateResult::Other)
+                }
+                Update::DeleteDocuments(ids) => {
+                    let mut builder = update_builder.delete_documents(&mut txn, self)?;
+
+                    // We ignore unexisting document ids
+                    ids.iter().for_each(|id| {
+                        builder.delete_external_id(id);
+                    });
+
+                    let deleted = builder.execute()?;
+                    Ok(UpdateResult::DocumentDeletion { deleted })
+                }
+            };
+            if result.is_ok() {
+                txn.commit()?;
+            }
+            result
+        })();
+
+        if let Update::DocumentAddition { content_uuid, .. } = update.from.meta() {
+            let _ = self.update_file_store.delete(*content_uuid);
+        }
+
+        match result {
+            Ok(result) => Ok(update.process(result)),
+            Err(e) => Err(update.fail(e)),
+        }
     }
 
-    pub fn update_documents_txn<'a, 'b>(
+    pub fn update_primary_key(&self, primary_key: Option<String>) -> Result<IndexMeta> {
+        match primary_key {
+            Some(primary_key) => {
+                let mut txn = self.write_txn()?;
+                let mut builder = UpdateBuilder::new(0).settings(&mut txn, self);
+                builder.set_primary_key(primary_key);
+                builder.execute(|_, _| ())?;
+                let meta = IndexMeta::new_txn(self, &txn)?;
+                txn.commit()?;
+                Ok(meta)
+            }
+            None => {
+                let meta = IndexMeta::new(self)?;
+                Ok(meta)
+            }
+        }
+    }
+
+    fn update_documents<'a, 'b>(
         &'a self,
         txn: &mut heed::RwTxn<'a, 'b>,
-        format: UpdateFormat,
         method: IndexDocumentsMethod,
-        content: Option<impl io::Read>,
+        content_uuid: Uuid,
         update_builder: UpdateBuilder,
         primary_key: Option<&str>,
     ) -> Result<UpdateResult> {
@@ -199,40 +253,22 @@ impl Index {
             builder.execute(|_, _| ())?;
         }
 
-        let mut builder = update_builder.index_documents(txn, self);
-        builder.update_format(format);
-        builder.index_documents_method(method);
-
         let indexing_callback =
             |indexing_step, update_id| debug!("update {}: {:?}", update_id, indexing_step);
 
-        let gzipped = false;
-        let addition = match content {
-            Some(content) if gzipped => {
-                builder.execute(GzDecoder::new(content), indexing_callback)?
-            }
-            Some(content) => builder.execute(content, indexing_callback)?,
-            None => builder.execute(std::io::empty(), indexing_callback)?,
-        };
+        let content_file = self.update_file_store.get_update(content_uuid).unwrap();
+        let reader = DocumentBatchReader::from_reader(content_file).unwrap();
+
+        let mut builder = update_builder.index_documents(txn, self);
+        builder.index_documents_method(method);
+        let addition = builder.execute(reader, indexing_callback)?;
 
         info!("document addition done: {:?}", addition);
 
         Ok(UpdateResult::DocumentsAddition(addition))
     }
 
-    pub fn clear_documents(&self, update_builder: UpdateBuilder) -> Result<UpdateResult> {
-        // We must use the write transaction of the update here.
-        let mut wtxn = self.write_txn()?;
-        let builder = update_builder.clear_documents(&mut wtxn, self);
-
-        let _count = builder.execute()?;
-
-        wtxn.commit()
-            .and(Ok(UpdateResult::Other))
-            .map_err(Into::into)
-    }
-
-    pub fn update_settings_txn<'a, 'b>(
+    fn update_settings<'a, 'b>(
         &'a self,
         txn: &mut heed::RwTxn<'a, 'b>,
         settings: &Settings<Checked>,
@@ -241,62 +277,7 @@ impl Index {
         // We must use the write transaction of the update here.
         let mut builder = update_builder.settings(txn, self);
 
-        match settings.searchable_attributes {
-            Setting::Set(ref names) => builder.set_searchable_fields(names.clone()),
-            Setting::Reset => builder.reset_searchable_fields(),
-            Setting::NotSet => (),
-        }
-
-        match settings.displayed_attributes {
-            Setting::Set(ref names) => builder.set_displayed_fields(names.clone()),
-            Setting::Reset => builder.reset_displayed_fields(),
-            Setting::NotSet => (),
-        }
-
-        match settings.filterable_attributes {
-            Setting::Set(ref facets) => {
-                builder.set_filterable_fields(facets.clone().into_iter().collect())
-            }
-            Setting::Reset => builder.reset_filterable_fields(),
-            Setting::NotSet => (),
-        }
-
-        match settings.sortable_attributes {
-            Setting::Set(ref fields) => {
-                builder.set_sortable_fields(fields.iter().cloned().collect())
-            }
-            Setting::Reset => {
-                // TODO we must use the reset_sortable_fields in a futur PR.
-                builder.set_sortable_fields(HashSet::new())
-            }
-            Setting::NotSet => (),
-        }
-
-        match settings.ranking_rules {
-            Setting::Set(ref criteria) => builder.set_criteria(criteria.clone()),
-            Setting::Reset => builder.reset_criteria(),
-            Setting::NotSet => (),
-        }
-
-        match settings.stop_words {
-            Setting::Set(ref stop_words) => builder.set_stop_words(stop_words.clone()),
-            Setting::Reset => builder.reset_stop_words(),
-            Setting::NotSet => (),
-        }
-
-        match settings.synonyms {
-            Setting::Set(ref synonyms) => {
-                builder.set_synonyms(synonyms.clone().into_iter().collect())
-            }
-            Setting::Reset => builder.reset_synonyms(),
-            Setting::NotSet => (),
-        }
-
-        match settings.distinct_attribute {
-            Setting::Set(ref attr) => builder.set_distinct_field(attr.clone()),
-            Setting::Reset => builder.reset_distinct_field(),
-            Setting::NotSet => (),
-        }
+        apply_settings_to_builder(settings, &mut builder);
 
         builder.execute(|indexing_step, update_id| {
             debug!("update {}: {:?}", update_id, indexing_step)
@@ -304,35 +285,60 @@ impl Index {
 
         Ok(UpdateResult::Other)
     }
+}
 
-    pub fn update_settings(
-        &self,
-        settings: &Settings<Checked>,
-        update_builder: UpdateBuilder,
-    ) -> Result<UpdateResult> {
-        let mut txn = self.write_txn()?;
-        let result = self.update_settings_txn(&mut txn, settings, update_builder)?;
-        txn.commit()?;
-        Ok(result)
+pub fn apply_settings_to_builder(
+    settings: &Settings<Checked>,
+    builder: &mut milli::update::Settings,
+) {
+    match settings.searchable_attributes {
+        Setting::Set(ref names) => builder.set_searchable_fields(names.clone()),
+        Setting::Reset => builder.reset_searchable_fields(),
+        Setting::NotSet => (),
     }
 
-    pub fn delete_documents(
-        &self,
-        document_ids: &[String],
-        update_builder: UpdateBuilder,
-    ) -> Result<UpdateResult> {
-        let mut txn = self.write_txn()?;
-        let mut builder = update_builder.delete_documents(&mut txn, self)?;
+    match settings.displayed_attributes {
+        Setting::Set(ref names) => builder.set_displayed_fields(names.clone()),
+        Setting::Reset => builder.reset_displayed_fields(),
+        Setting::NotSet => (),
+    }
 
-        // We ignore unexisting document ids
-        document_ids.iter().for_each(|id| {
-            builder.delete_external_id(id);
-        });
+    match settings.filterable_attributes {
+        Setting::Set(ref facets) => {
+            builder.set_filterable_fields(facets.clone().into_iter().collect())
+        }
+        Setting::Reset => builder.reset_filterable_fields(),
+        Setting::NotSet => (),
+    }
 
-        let deleted = builder.execute()?;
-        txn.commit()
-            .and(Ok(UpdateResult::DocumentDeletion { deleted }))
-            .map_err(Into::into)
+    match settings.sortable_attributes {
+        Setting::Set(ref fields) => builder.set_sortable_fields(fields.iter().cloned().collect()),
+        Setting::Reset => builder.reset_sortable_fields(),
+        Setting::NotSet => (),
+    }
+
+    match settings.ranking_rules {
+        Setting::Set(ref criteria) => builder.set_criteria(criteria.clone()),
+        Setting::Reset => builder.reset_criteria(),
+        Setting::NotSet => (),
+    }
+
+    match settings.stop_words {
+        Setting::Set(ref stop_words) => builder.set_stop_words(stop_words.clone()),
+        Setting::Reset => builder.reset_stop_words(),
+        Setting::NotSet => (),
+    }
+
+    match settings.synonyms {
+        Setting::Set(ref synonyms) => builder.set_synonyms(synonyms.clone().into_iter().collect()),
+        Setting::Reset => builder.reset_synonyms(),
+        Setting::NotSet => (),
+    }
+
+    match settings.distinct_attribute {
+        Setting::Set(ref attr) => builder.set_distinct_field(attr.clone()),
+        Setting::Reset => builder.reset_distinct_field(),
+        Setting::NotSet => (),
     }
 }
 

meilisearch-lib/src/index_controller/dump_actor/actor.rs

@@ -7,20 +7,22 @@ use chrono::Utc;
 use futures::{lock::Mutex, stream::StreamExt};
 use log::{error, trace};
 use tokio::sync::{mpsc, oneshot, RwLock};
-use update_actor::UpdateActorHandle;
-use uuid_resolver::UuidResolverHandle;
 
 use super::error::{DumpActorError, Result};
 use super::{DumpInfo, DumpMsg, DumpStatus, DumpTask};
-use crate::index_controller::{update_actor, uuid_resolver};
+use crate::index_controller::index_resolver::index_store::IndexStore;
+use crate::index_controller::index_resolver::uuid_store::UuidStore;
+use crate::index_controller::index_resolver::IndexResolver;
+use crate::index_controller::updates::UpdateSender;
 
 pub const CONCURRENT_DUMP_MSG: usize = 10;
 
-pub struct DumpActor<UuidResolver, Update> {
+pub struct DumpActor<U, I> {
     inbox: Option<mpsc::Receiver<DumpMsg>>,
-    uuid_resolver: UuidResolver,
-    update: Update,
+    index_resolver: Arc<IndexResolver<U, I>>,
+    update: UpdateSender,
     dump_path: PathBuf,
+    analytics_path: PathBuf,
     lock: Arc<Mutex<()>>,
     dump_infos: Arc<RwLock<HashMap<String, DumpInfo>>>,
     update_db_size: usize,
@@ -32,16 +34,17 @@ fn generate_uid() -> String {
     Utc::now().format("%Y%m%d-%H%M%S%3f").to_string()
 }
 
-impl<UuidResolver, Update> DumpActor<UuidResolver, Update>
+impl<U, I> DumpActor<U, I>
 where
-    UuidResolver: UuidResolverHandle + Send + Sync + Clone + 'static,
-    Update: UpdateActorHandle + Send + Sync + Clone + 'static,
+    U: UuidStore + Sync + Send + 'static,
+    I: IndexStore + Sync + Send + 'static,
 {
     pub fn new(
         inbox: mpsc::Receiver<DumpMsg>,
-        uuid_resolver: UuidResolver,
-        update: Update,
+        index_resolver: Arc<IndexResolver<U, I>>,
+        update: UpdateSender,
         dump_path: impl AsRef<Path>,
+        analytics_path: impl AsRef<Path>,
         index_db_size: usize,
         update_db_size: usize,
     ) -> Self {
@@ -49,9 +52,10 @@ where
         let lock = Arc::new(Mutex::new(()));
         Self {
             inbox: Some(inbox),
-            uuid_resolver,
+            index_resolver,
             update,
             dump_path: dump_path.as_ref().into(),
+            analytics_path: analytics_path.as_ref().into(),
             dump_infos,
             lock,
             index_db_size,
@@ -117,9 +121,10 @@ where
         ret.send(Ok(info)).expect("Dump actor is dead");
 
         let task = DumpTask {
-            path: self.dump_path.clone(),
-            uuid_resolver: self.uuid_resolver.clone(),
-            update_handle: self.update.clone(),
+            dump_path: self.dump_path.clone(),
+            db_path: self.analytics_path.clone(),
+            index_resolver: self.index_resolver.clone(),
+            update_sender: self.update.clone(),
             uid: uid.clone(),
             update_db_size: self.update_db_size,
             index_db_size: self.index_db_size,

meilisearch-lib/src/index_controller/dump_actor/error.rs

@@ -1,22 +1,22 @@
 use meilisearch_error::{Code, ErrorCode};
 
-use crate::index_controller::update_actor::error::UpdateActorError;
-use crate::index_controller::uuid_resolver::error::UuidResolverError;
+use crate::index_controller::index_resolver::error::IndexResolverError;
+use crate::index_controller::updates::error::UpdateLoopError;
 
 pub type Result<T> = std::result::Result<T, DumpActorError>;
 
 #[derive(thiserror::Error, Debug)]
 pub enum DumpActorError {
-    #[error("Another dump is already in progress")]
+    #[error("A dump is already processing. You must wait until the current process is finished before requesting another dump.")]
     DumpAlreadyRunning,
-    #[error("Dump `{0}` not found")]
+    #[error("Dump `{0}` not found.")]
     DumpDoesNotExist(String),
-    #[error("Internal error: {0}")]
+    #[error("An internal error has occurred. `{0}`.")]
     Internal(Box<dyn std::error::Error + Send + Sync + 'static>),
     #[error("{0}")]
-    UuidResolver(#[from] UuidResolverError),
+    IndexResolver(#[from] IndexResolverError),
     #[error("{0}")]
-    UpdateActor(#[from] UpdateActorError),
+    UpdateLoop(#[from] UpdateLoopError),
 }
 
 macro_rules! internal_error {
@@ -43,10 +43,10 @@ impl ErrorCode for DumpActorError {
     fn error_code(&self) -> Code {
         match self {
             DumpActorError::DumpAlreadyRunning => Code::DumpAlreadyInProgress,
-            DumpActorError::DumpDoesNotExist(_) => Code::NotFound,
+            DumpActorError::DumpDoesNotExist(_) => Code::DumpNotFound,
             DumpActorError::Internal(_) => Code::Internal,
-            DumpActorError::UuidResolver(e) => e.error_code(),
-            DumpActorError::UpdateActor(e) => e.error_code(),
+            DumpActorError::IndexResolver(e) => e.error_code(),
+            DumpActorError::UpdateLoop(e) => e.error_code(),
         }
     }
 }

meilisearch-lib/src/index_controller/dump_actor/handle_impl.rs

@@ -1,8 +1,10 @@
 use std::path::Path;
+use std::sync::Arc;
 
-use actix_web::web::Bytes;
 use tokio::sync::{mpsc, oneshot};
 
+use crate::index_controller::index_resolver::HardStateIndexResolver;
+
 use super::error::Result;
 use super::{DumpActor, DumpActorHandle, DumpInfo, DumpMsg};
 
@@ -31,17 +33,19 @@ impl DumpActorHandle for DumpActorHandleImpl {
 impl DumpActorHandleImpl {
     pub fn new(
         path: impl AsRef<Path>,
-        uuid_resolver: crate::index_controller::uuid_resolver::UuidResolverHandleImpl,
-        update: crate::index_controller::update_actor::UpdateActorHandleImpl<Bytes>,
+        analytics_path: impl AsRef<Path>,
+        index_resolver: Arc<HardStateIndexResolver>,
+        update: crate::index_controller::updates::UpdateSender,
         index_db_size: usize,
         update_db_size: usize,
     ) -> anyhow::Result<Self> {
         let (sender, receiver) = mpsc::channel(10);
         let actor = DumpActor::new(
             receiver,
-            uuid_resolver,
+            index_resolver,
             update,
             path,
+            analytics_path,
             index_db_size,
             update_db_size,
         );

meilisearch-lib/src/index_controller/dump_actor/loaders/mod.rs

@@ -0,0 +1,19 @@
+pub mod v1;
+pub mod v2;
+pub mod v3;
+
+mod compat {
+    /// Parses the v1 version of the Asc ranking rules `asc(price)`and returns the field name.
+    pub fn asc_ranking_rule(text: &str) -> Option<&str> {
+        text.split_once("asc(")
+            .and_then(|(_, tail)| tail.rsplit_once(")"))
+            .map(|(field, _)| field)
+    }
+
+    /// Parses the v1 version of the Desc ranking rules `desc(price)`and returns the field name.
+    pub fn desc_ranking_rule(text: &str) -> Option<&str> {
+        text.split_once("desc(")
+            .and_then(|(_, tail)| tail.rsplit_once(")"))
+            .map(|(field, _)| field)
+    }
+}

meilisearch-lib/src/index_controller/dump_actor/loaders/v1.rs

@@ -1,27 +1,28 @@
 use std::collections::{BTreeMap, BTreeSet};
 use std::fs::{create_dir_all, File};
-use std::io::BufRead;
+use std::io::{BufReader, Seek, SeekFrom};
 use std::marker::PhantomData;
 use std::path::Path;
-use std::sync::Arc;
 
 use heed::EnvOpenOptions;
-use log::{error, info, warn};
-use milli::update::{IndexDocumentsMethod, Setting, UpdateFormat};
+use log::{error, warn};
+use milli::documents::DocumentBatchReader;
+use milli::update::Setting;
 use serde::{Deserialize, Deserializer, Serialize};
 use uuid::Uuid;
 
-use crate::index_controller::{self, uuid_resolver::HeedUuidStore, IndexMetadata};
-use crate::index_controller::{asc_ranking_rule, desc_ranking_rule};
-use crate::{
-    index::{update_handler::UpdateHandler, Index, Unchecked},
-    option::IndexerOpts,
-};
+use crate::document_formats::read_ndjson;
+use crate::index::apply_settings_to_builder;
+use crate::index::update_handler::UpdateHandler;
+use crate::index_controller::dump_actor::loaders::compat::{asc_ranking_rule, desc_ranking_rule};
+use crate::index_controller::index_resolver::uuid_store::HeedUuidStore;
+use crate::index_controller::{self, IndexMetadata};
+use crate::{index::Unchecked, options::IndexerOpts};
 
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(rename_all = "camelCase")]
 pub struct MetadataV1 {
-    db_version: String,
+    pub db_version: String,
     indexes: Vec<IndexMetadata>,
 }
 
@@ -33,11 +34,6 @@ impl MetadataV1 {
         size: usize,
         indexer_options: &IndexerOpts,
     ) -> anyhow::Result<()> {
-        info!(
-            "Loading dump, dump database version: {}, dump version: V1",
-            self.db_version
-        );
-
         let uuid_store = HeedUuidStore::new(&dst)?;
         for index in self.indexes {
             let uuid = Uuid::new_v4();
@@ -93,46 +89,55 @@ fn load_index(
     size: usize,
     indexer_options: &IndexerOpts,
 ) -> anyhow::Result<()> {
-    let index_path = dst.as_ref().join(&format!("indexes/index-{}", uuid));
+    let index_path = dst.as_ref().join(&format!("indexes/{}", uuid));
 
     create_dir_all(&index_path)?;
     let mut options = EnvOpenOptions::new();
     options.map_size(size);
     let index = milli::Index::new(options, index_path)?;
-    let index = Index(Arc::new(index));
 
+    let update_handler = UpdateHandler::new(indexer_options)?;
+
+    let mut txn = index.write_txn()?;
     // extract `settings.json` file and import content
     let settings = import_settings(&src)?;
     let settings: index_controller::Settings<Unchecked> = settings.into();
 
-    let mut txn = index.write_txn()?;
-
     let handler = UpdateHandler::new(indexer_options)?;
 
-    index.update_settings_txn(&mut txn, &settings.check(), handler.update_builder(0))?;
+    let mut builder = handler.update_builder(0).settings(&mut txn, &index);
 
-    let file = File::open(&src.as_ref().join("documents.jsonl"))?;
-    let mut reader = std::io::BufReader::new(file);
-    reader.fill_buf()?;
-    if !reader.buffer().is_empty() {
-        index.update_documents_txn(
-            &mut txn,
-            UpdateFormat::JsonStream,
-            IndexDocumentsMethod::ReplaceDocuments,
-            Some(reader),
-            handler.update_builder(0),
-            primary_key,
-        )?;
+    if let Some(primary_key) = primary_key {
+        builder.set_primary_key(primary_key.to_string());
+    }
+
+    apply_settings_to_builder(&settings.check(), &mut builder);
+
+    builder.execute(|_, _| ())?;
+
+    let reader = BufReader::new(File::open(&src.as_ref().join("documents.jsonl"))?);
+
+    let mut tmp_doc_file = tempfile::tempfile()?;
+
+    read_ndjson(reader, &mut tmp_doc_file)?;
+
+    tmp_doc_file.seek(SeekFrom::Start(0))?;
+
+    let documents_reader = DocumentBatchReader::from_reader(tmp_doc_file)?;
+
+    //If the document file is empty, we don't perform the document addition, to prevent
+    //a primary key error to be thrown.
+    if !documents_reader.is_empty() {
+        let builder = update_handler
+            .update_builder(0)
+            .index_documents(&mut txn, &index);
+        builder.execute(documents_reader, |_, _| ())?;
     }
 
     txn.commit()?;
 
     // Finaly, we extract the original milli::Index and close it
-    Arc::try_unwrap(index.0)
-        .map_err(|_e| "Couldn't close the index properly")
-        .unwrap()
-        .prepare_for_closing()
-        .wait();
+    index.prepare_for_closing().wait();
 
     // Updates are ignored in dumps V1.
 

meilisearch-lib/src/index_controller/dump_actor/loaders/v2.rs

@@ -0,0 +1,391 @@
+use std::fs::{File, OpenOptions};
+use std::io::Write;
+use std::path::{Path, PathBuf};
+
+use serde_json::{Deserializer, Value};
+use tempfile::NamedTempFile;
+
+use crate::index_controller::dump_actor::loaders::compat::{asc_ranking_rule, desc_ranking_rule};
+use crate::index_controller::dump_actor::Metadata;
+use crate::index_controller::updates::status::{
+    Aborted, Enqueued, Failed, Processed, Processing, UpdateResult, UpdateStatus,
+};
+use crate::index_controller::updates::store::dump::UpdateEntry;
+use crate::index_controller::updates::store::Update;
+use crate::options::IndexerOpts;
+
+use super::v3;
+
+/// The dump v2 reads the dump folder and patches all the needed file to make it compatible with a
+/// dump v3, then calls the dump v3 to actually handle the dump.
+pub fn load_dump(
+    meta: Metadata,
+    src: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+    index_db_size: usize,
+    update_db_size: usize,
+    indexing_options: &IndexerOpts,
+) -> anyhow::Result<()> {
+    let indexes_path = src.as_ref().join("indexes");
+
+    let dir_entries = std::fs::read_dir(indexes_path)?;
+    for entry in dir_entries {
+        let entry = entry?;
+
+        // rename the index folder
+        let path = entry.path();
+        let new_path = patch_index_uuid_path(&path).expect("invalid index folder.");
+
+        std::fs::rename(path, &new_path)?;
+
+        let settings_path = new_path.join("meta.json");
+
+        patch_settings(settings_path)?;
+    }
+
+    let update_dir = src.as_ref().join("updates");
+    let update_path = update_dir.join("data.jsonl");
+    patch_updates(update_dir, update_path)?;
+
+    v3::load_dump(
+        meta,
+        src,
+        dst,
+        index_db_size,
+        update_db_size,
+        indexing_options,
+    )
+}
+
+fn patch_index_uuid_path(path: &Path) -> Option<PathBuf> {
+    let uuid = path.file_name()?.to_str()?.trim_start_matches("index-");
+    let new_path = path.parent()?.join(uuid);
+    Some(new_path)
+}
+
+fn patch_settings(path: impl AsRef<Path>) -> anyhow::Result<()> {
+    let mut meta_file = File::open(&path)?;
+    let mut meta: Value = serde_json::from_reader(&mut meta_file)?;
+
+    // We first deserialize the dump meta into a serde_json::Value and change
+    // the custom ranking rules settings from the old format to the new format.
+    if let Some(ranking_rules) = meta.pointer_mut("/settings/rankingRules") {
+        patch_custom_ranking_rules(ranking_rules);
+    }
+
+    let mut meta_file = OpenOptions::new().truncate(true).write(true).open(path)?;
+
+    serde_json::to_writer(&mut meta_file, &meta)?;
+
+    Ok(())
+}
+
+fn patch_updates(dir: impl AsRef<Path>, path: impl AsRef<Path>) -> anyhow::Result<()> {
+    let mut output_update_file = NamedTempFile::new_in(&dir)?;
+    let update_file = File::open(&path)?;
+
+    let stream = Deserializer::from_reader(update_file).into_iter::<compat::UpdateEntry>();
+
+    for update in stream {
+        let update_entry = update?;
+
+        let update_entry = UpdateEntry::from(update_entry);
+
+        serde_json::to_writer(&mut output_update_file, &update_entry)?;
+        output_update_file.write_all(b"\n")?;
+    }
+
+    output_update_file.flush()?;
+    output_update_file.persist(path)?;
+
+    Ok(())
+}
+
+/// Converts the ranking rules from the format `asc(_)`, `desc(_)` to the format `_:asc`, `_:desc`.
+///
+/// This is done for compatibility reasons, and to avoid a new dump version,
+/// since the new syntax was introduced soon after the new dump version.
+fn patch_custom_ranking_rules(ranking_rules: &mut Value) {
+    *ranking_rules = match ranking_rules.take() {
+        Value::Array(values) => values
+            .into_iter()
+            .filter_map(|value| match value {
+                Value::String(s) if s.starts_with("asc") => asc_ranking_rule(&s)
+                    .map(|f| format!("{}:asc", f))
+                    .map(Value::String),
+                Value::String(s) if s.starts_with("desc") => desc_ranking_rule(&s)
+                    .map(|f| format!("{}:desc", f))
+                    .map(Value::String),
+                otherwise => Some(otherwise),
+            })
+            .collect(),
+        otherwise => otherwise,
+    }
+}
+
+impl From<compat::UpdateEntry> for UpdateEntry {
+    fn from(compat::UpdateEntry { uuid, update }: compat::UpdateEntry) -> Self {
+        let update = match update {
+            compat::UpdateStatus::Processing(meta) => UpdateStatus::Processing(meta.into()),
+            compat::UpdateStatus::Enqueued(meta) => UpdateStatus::Enqueued(meta.into()),
+            compat::UpdateStatus::Processed(meta) => UpdateStatus::Processed(meta.into()),
+            compat::UpdateStatus::Aborted(meta) => UpdateStatus::Aborted(meta.into()),
+            compat::UpdateStatus::Failed(meta) => UpdateStatus::Failed(meta.into()),
+        };
+
+        Self { uuid, update }
+    }
+}
+
+impl From<compat::Failed> for Failed {
+    fn from(other: compat::Failed) -> Self {
+        let compat::Failed {
+            from,
+            error,
+            failed_at,
+        } = other;
+
+        Self {
+            from: from.into(),
+            msg: error.message,
+            code: compat::error_code_from_str(&error.error_code)
+                .expect("Invalid update: Invalid error code"),
+            failed_at,
+        }
+    }
+}
+
+impl From<compat::Aborted> for Aborted {
+    fn from(other: compat::Aborted) -> Self {
+        let compat::Aborted { from, aborted_at } = other;
+
+        Self {
+            from: from.into(),
+            aborted_at,
+        }
+    }
+}
+
+impl From<compat::Processing> for Processing {
+    fn from(other: compat::Processing) -> Self {
+        let compat::Processing {
+            from,
+            started_processing_at,
+        } = other;
+
+        Self {
+            from: from.into(),
+            started_processing_at,
+        }
+    }
+}
+
+impl From<compat::Enqueued> for Enqueued {
+    fn from(other: compat::Enqueued) -> Self {
+        let compat::Enqueued {
+            update_id,
+            meta,
+            enqueued_at,
+            content,
+        } = other;
+
+        let meta = match meta {
+            compat::UpdateMeta::DocumentsAddition {
+                method,
+                primary_key,
+                ..
+            } => {
+                Update::DocumentAddition {
+                    primary_key,
+                    method,
+                    // Just ignore if the uuid is no present. If it is needed later, an error will
+                    // be thrown.
+                    content_uuid: content.unwrap_or_default(),
+                }
+            }
+            compat::UpdateMeta::ClearDocuments => Update::ClearDocuments,
+            compat::UpdateMeta::DeleteDocuments { ids } => Update::DeleteDocuments(ids),
+            compat::UpdateMeta::Settings(settings) => Update::Settings(settings),
+        };
+
+        Self {
+            update_id,
+            meta,
+            enqueued_at,
+        }
+    }
+}
+
+impl From<compat::Processed> for Processed {
+    fn from(other: compat::Processed) -> Self {
+        let compat::Processed {
+            from,
+            success,
+            processed_at,
+        } = other;
+
+        Self {
+            success: success.into(),
+            processed_at,
+            from: from.into(),
+        }
+    }
+}
+
+impl From<compat::UpdateResult> for UpdateResult {
+    fn from(other: compat::UpdateResult) -> Self {
+        match other {
+            compat::UpdateResult::DocumentsAddition(r) => Self::DocumentsAddition(r),
+            compat::UpdateResult::DocumentDeletion { deleted } => {
+                Self::DocumentDeletion { deleted }
+            }
+            compat::UpdateResult::Other => Self::Other,
+        }
+    }
+}
+
+/// compat structure from pre-dumpv3 meilisearch
+mod compat {
+    use anyhow::bail;
+    use chrono::{DateTime, Utc};
+    use meilisearch_error::Code;
+    use milli::update::{DocumentAdditionResult, IndexDocumentsMethod};
+    use serde::{Deserialize, Serialize};
+    use uuid::Uuid;
+
+    use crate::index::{Settings, Unchecked};
+
+    #[derive(Serialize, Deserialize)]
+    pub struct UpdateEntry {
+        pub uuid: Uuid,
+        pub update: UpdateStatus,
+    }
+
+    #[derive(Debug, Clone, Serialize, Deserialize)]
+    pub enum UpdateFormat {
+        Json,
+    }
+
+    #[derive(Debug, Clone, Serialize, Deserialize)]
+    pub enum UpdateResult {
+        DocumentsAddition(DocumentAdditionResult),
+        DocumentDeletion { deleted: u64 },
+        Other,
+    }
+
+    #[allow(clippy::large_enum_variant)]
+    #[derive(Debug, Clone, Serialize, Deserialize)]
+    #[serde(tag = "type")]
+    pub enum UpdateMeta {
+        DocumentsAddition {
+            method: IndexDocumentsMethod,
+            format: UpdateFormat,
+            primary_key: Option<String>,
+        },
+        ClearDocuments,
+        DeleteDocuments {
+            ids: Vec<String>,
+        },
+        Settings(Settings<Unchecked>),
+    }
+
+    #[derive(Debug, Serialize, Deserialize, Clone)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Enqueued {
+        pub update_id: u64,
+        pub meta: UpdateMeta,
+        pub enqueued_at: DateTime<Utc>,
+        pub content: Option<Uuid>,
+    }
+
+    #[derive(Debug, Serialize, Deserialize, Clone)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Processed {
+        pub success: UpdateResult,
+        pub processed_at: DateTime<Utc>,
+        #[serde(flatten)]
+        pub from: Processing,
+    }
+
+    #[derive(Debug, Serialize, Deserialize, Clone)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Processing {
+        #[serde(flatten)]
+        pub from: Enqueued,
+        pub started_processing_at: DateTime<Utc>,
+    }
+
+    #[derive(Debug, Serialize, Deserialize, Clone)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Aborted {
+        #[serde(flatten)]
+        pub from: Enqueued,
+        pub aborted_at: DateTime<Utc>,
+    }
+
+    #[derive(Debug, Serialize, Deserialize)]
+    #[serde(rename_all = "camelCase")]
+    pub struct Failed {
+        #[serde(flatten)]
+        pub from: Processing,
+        pub error: ResponseError,
+        pub failed_at: DateTime<Utc>,
+    }
+
+    #[derive(Debug, Serialize, Deserialize)]
+    #[serde(tag = "status", rename_all = "camelCase")]
+    pub enum UpdateStatus {
+        Processing(Processing),
+        Enqueued(Enqueued),
+        Processed(Processed),
+        Aborted(Aborted),
+        Failed(Failed),
+    }
+
+    type StatusCode = ();
+
+    #[derive(Debug, Serialize, Deserialize, Clone)]
+    #[serde(rename_all = "camelCase")]
+    pub struct ResponseError {
+        #[serde(skip)]
+        pub code: StatusCode,
+        pub message: String,
+        pub error_code: String,
+        pub error_type: String,
+        pub error_link: String,
+    }
+
+    pub fn error_code_from_str(s: &str) -> anyhow::Result<Code> {
+        let code = match s {
+            "index_creation_failed" => Code::CreateIndex,
+            "index_already_exists" => Code::IndexAlreadyExists,
+            "index_not_found" => Code::IndexNotFound,
+            "invalid_index_uid" => Code::InvalidIndexUid,
+            "invalid_state" => Code::InvalidState,
+            "missing_primary_key" => Code::MissingPrimaryKey,
+            "primary_key_already_present" => Code::PrimaryKeyAlreadyPresent,
+            "invalid_request" => Code::InvalidRankingRule,
+            "max_fields_limit_exceeded" => Code::MaxFieldsLimitExceeded,
+            "missing_document_id" => Code::MissingDocumentId,
+            "invalid_facet" => Code::Filter,
+            "invalid_filter" => Code::Filter,
+            "invalid_sort" => Code::Sort,
+            "bad_parameter" => Code::BadParameter,
+            "bad_request" => Code::BadRequest,
+            "document_not_found" => Code::DocumentNotFound,
+            "internal" => Code::Internal,
+            "invalid_geo_field" => Code::InvalidGeoField,
+            "invalid_token" => Code::InvalidToken,
+            "missing_authorization_header" => Code::MissingAuthorizationHeader,
+            "payload_too_large" => Code::PayloadTooLarge,
+            "unretrievable_document" => Code::RetrieveDocument,
+            "search_error" => Code::SearchDocuments,
+            "unsupported_media_type" => Code::UnsupportedMediaType,
+            "dump_already_in_progress" => Code::DumpAlreadyInProgress,
+            "dump_process_failed" => Code::DumpProcessFailed,
+            _ => bail!("unknow error code."),
+        };
+
+        Ok(code)
+    }
+}

meilisearch-lib/src/index_controller/dump_actor/loaders/v3.rs

@@ -0,0 +1,33 @@
+use std::path::Path;
+
+use log::info;
+
+use crate::analytics;
+use crate::index_controller::dump_actor::Metadata;
+use crate::index_controller::index_resolver::IndexResolver;
+use crate::index_controller::update_file_store::UpdateFileStore;
+use crate::index_controller::updates::store::UpdateStore;
+use crate::options::IndexerOpts;
+
+pub fn load_dump(
+    meta: Metadata,
+    src: impl AsRef<Path>,
+    dst: impl AsRef<Path>,
+    index_db_size: usize,
+    update_db_size: usize,
+    indexing_options: &IndexerOpts,
+) -> anyhow::Result<()> {
+    info!(
+        "Loading dump from {}, dump database version: {}, dump version: V3",
+        meta.dump_date, meta.db_version
+    );
+
+    IndexResolver::load_dump(src.as_ref(), &dst, index_db_size, indexing_options)?;
+    UpdateFileStore::load_dump(src.as_ref(), &dst)?;
+    UpdateStore::load_dump(&src, &dst, update_db_size)?;
+    analytics::copy_user_id(src.as_ref(), dst.as_ref());
+
+    info!("Loading indexes.");
+
+    Ok(())
+}

meilisearch-lib/src/index_controller/dump_actor/mod.rs

@@ -0,0 +1,386 @@
+use std::fs::File;
+use std::path::{Path, PathBuf};
+use std::sync::Arc;
+
+use chrono::{DateTime, Utc};
+use log::{info, trace, warn};
+use serde::{Deserialize, Serialize};
+use tokio::fs::create_dir_all;
+
+use loaders::v1::MetadataV1;
+
+pub use actor::DumpActor;
+pub use handle_impl::*;
+pub use message::DumpMsg;
+
+use super::index_resolver::index_store::IndexStore;
+use super::index_resolver::uuid_store::UuidStore;
+use super::index_resolver::IndexResolver;
+use super::updates::UpdateSender;
+use crate::analytics;
+use crate::compression::{from_tar_gz, to_tar_gz};
+use crate::index_controller::dump_actor::error::DumpActorError;
+use crate::index_controller::dump_actor::loaders::{v2, v3};
+use crate::index_controller::updates::UpdateMsg;
+use crate::options::IndexerOpts;
+use error::Result;
+
+mod actor;
+pub mod error;
+mod handle_impl;
+mod loaders;
+mod message;
+
+const META_FILE_NAME: &str = "metadata.json";
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct Metadata {
+    db_version: String,
+    index_db_size: usize,
+    update_db_size: usize,
+    dump_date: DateTime<Utc>,
+}
+
+impl Metadata {
+    pub fn new(index_db_size: usize, update_db_size: usize) -> Self {
+        Self {
+            db_version: env!("CARGO_PKG_VERSION").to_string(),
+            index_db_size,
+            update_db_size,
+            dump_date: Utc::now(),
+        }
+    }
+}
+
+#[async_trait::async_trait]
+#[cfg_attr(test, mockall::automock)]
+pub trait DumpActorHandle {
+    /// Start the creation of a dump
+    /// Implementation: [handle_impl::DumpActorHandleImpl::create_dump]
+    async fn create_dump(&self) -> Result<DumpInfo>;
+
+    /// Return the status of an already created dump
+    /// Implementation: [handle_impl::DumpActorHandleImpl::dump_info]
+    async fn dump_info(&self, uid: String) -> Result<DumpInfo>;
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+#[serde(tag = "dumpVersion")]
+pub enum MetadataVersion {
+    V1(MetadataV1),
+    V2(Metadata),
+    V3(Metadata),
+}
+
+impl MetadataVersion {
+    pub fn new_v3(index_db_size: usize, update_db_size: usize) -> Self {
+        let meta = Metadata::new(index_db_size, update_db_size);
+        Self::V3(meta)
+    }
+
+    pub fn db_version(&self) -> &str {
+        match self {
+            Self::V1(meta) => &meta.db_version,
+            Self::V2(meta) | Self::V3(meta) => &meta.db_version,
+        }
+    }
+
+    pub fn version(&self) -> &str {
+        match self {
+            MetadataVersion::V1(_) => "V1",
+            MetadataVersion::V2(_) => "V2",
+            MetadataVersion::V3(_) => "V3",
+        }
+    }
+
+    pub fn dump_date(&self) -> Option<&DateTime<Utc>> {
+        match self {
+            MetadataVersion::V1(_) => None,
+            MetadataVersion::V2(meta) | MetadataVersion::V3(meta) => Some(&meta.dump_date),
+        }
+    }
+}
+
+#[derive(Debug, Serialize, Deserialize, PartialEq, Clone)]
+#[serde(rename_all = "snake_case")]
+pub enum DumpStatus {
+    Done,
+    InProgress,
+    Failed,
+}
+
+#[derive(Debug, Serialize, Clone)]
+#[serde(rename_all = "camelCase")]
+pub struct DumpInfo {
+    pub uid: String,
+    pub status: DumpStatus,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    pub error: Option<String>,
+    started_at: DateTime<Utc>,
+    #[serde(skip_serializing_if = "Option::is_none")]
+    finished_at: Option<DateTime<Utc>>,
+}
+
+impl DumpInfo {
+    pub fn new(uid: String, status: DumpStatus) -> Self {
+        Self {
+            uid,
+            status,
+            error: None,
+            started_at: Utc::now(),
+            finished_at: None,
+        }
+    }
+
+    pub fn with_error(&mut self, error: String) {
+        self.status = DumpStatus::Failed;
+        self.finished_at = Some(Utc::now());
+        self.error = Some(error);
+    }
+
+    pub fn done(&mut self) {
+        self.finished_at = Some(Utc::now());
+        self.status = DumpStatus::Done;
+    }
+
+    pub fn dump_already_in_progress(&self) -> bool {
+        self.status == DumpStatus::InProgress
+    }
+}
+
+pub fn load_dump(
+    dst_path: impl AsRef<Path>,
+    src_path: impl AsRef<Path>,
+    index_db_size: usize,
+    update_db_size: usize,
+    indexer_opts: &IndexerOpts,
+) -> anyhow::Result<()> {
+    // Setup a temp directory path in the same path as the database, to prevent cross devices
+    // references.
+    let temp_path = dst_path
+        .as_ref()
+        .parent()
+        .map(ToOwned::to_owned)
+        .unwrap_or_else(|| ".".into());
+    if cfg!(windows) {
+        std::env::set_var("TMP", temp_path);
+    } else {
+        std::env::set_var("TMPDIR", temp_path);
+    }
+
+    let tmp_src = tempfile::tempdir()?;
+    let tmp_src_path = tmp_src.path();
+
+    from_tar_gz(&src_path, tmp_src_path)?;
+
+    let meta_path = tmp_src_path.join(META_FILE_NAME);
+    let mut meta_file = File::open(&meta_path)?;
+    let meta: MetadataVersion = serde_json::from_reader(&mut meta_file)?;
+
+    let tmp_dst = tempfile::tempdir()?;
+
+    info!(
+        "Loading dump {}, dump database version: {}, dump version: {}",
+        meta.dump_date()
+            .map(|t| format!("from {}", t))
+            .unwrap_or_else(String::new),
+        meta.db_version(),
+        meta.version()
+    );
+
+    match meta {
+        MetadataVersion::V1(meta) => {
+            meta.load_dump(&tmp_src_path, tmp_dst.path(), index_db_size, indexer_opts)?
+        }
+        MetadataVersion::V2(meta) => v2::load_dump(
+            meta,
+            &tmp_src_path,
+            tmp_dst.path(),
+            index_db_size,
+            update_db_size,
+            indexer_opts,
+        )?,
+        MetadataVersion::V3(meta) => v3::load_dump(
+            meta,
+            &tmp_src_path,
+            tmp_dst.path(),
+            index_db_size,
+            update_db_size,
+            indexer_opts,
+        )?,
+    }
+    // Persist and atomically rename the db
+    let persisted_dump = tmp_dst.into_path();
+    if dst_path.as_ref().exists() {
+        warn!("Overwriting database at {}", dst_path.as_ref().display());
+        std::fs::remove_dir_all(&dst_path)?;
+    }
+
+    std::fs::rename(&persisted_dump, &dst_path)?;
+
+    Ok(())
+}
+
+struct DumpTask<U, I> {
+    dump_path: PathBuf,
+    db_path: PathBuf,
+    index_resolver: Arc<IndexResolver<U, I>>,
+    update_sender: UpdateSender,
+    uid: String,
+    update_db_size: usize,
+    index_db_size: usize,
+}
+
+impl<U, I> DumpTask<U, I>
+where
+    U: UuidStore + Sync + Send + 'static,
+    I: IndexStore + Sync + Send + 'static,
+{
+    async fn run(self) -> Result<()> {
+        trace!("Performing dump.");
+
+        create_dir_all(&self.dump_path).await?;
+
+        let temp_dump_dir = tokio::task::spawn_blocking(tempfile::TempDir::new).await??;
+        let temp_dump_path = temp_dump_dir.path().to_owned();
+
+        let meta = MetadataVersion::new_v3(self.index_db_size, self.update_db_size);
+        let meta_path = temp_dump_path.join(META_FILE_NAME);
+        let mut meta_file = File::create(&meta_path)?;
+        serde_json::to_writer(&mut meta_file, &meta)?;
+        analytics::copy_user_id(&self.db_path, &temp_dump_path);
+
+        create_dir_all(&temp_dump_path.join("indexes")).await?;
+        let uuids = self.index_resolver.dump(temp_dump_path.clone()).await?;
+
+        UpdateMsg::dump(&self.update_sender, uuids, temp_dump_path.clone()).await?;
+
+        let dump_path = tokio::task::spawn_blocking(move || -> Result<PathBuf> {
+            let temp_dump_file = tempfile::NamedTempFile::new_in(&self.dump_path)?;
+            to_tar_gz(temp_dump_path, temp_dump_file.path())
+                .map_err(|e| DumpActorError::Internal(e.into()))?;
+
+            let dump_path = self.dump_path.join(self.uid).with_extension("dump");
+            temp_dump_file.persist(&dump_path)?;
+
+            Ok(dump_path)
+        })
+        .await??;
+
+        info!("Created dump in {:?}.", dump_path);
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use std::collections::HashSet;
+
+    use futures::future::{err, ok};
+    use once_cell::sync::Lazy;
+    use uuid::Uuid;
+
+    use super::*;
+    use crate::index::error::Result as IndexResult;
+    use crate::index::test::Mocker;
+    use crate::index::Index;
+    use crate::index_controller::index_resolver::error::IndexResolverError;
+    use crate::index_controller::index_resolver::index_store::MockIndexStore;
+    use crate::index_controller::index_resolver::uuid_store::MockUuidStore;
+    use crate::index_controller::updates::create_update_handler;
+
+    fn setup() {
+        static SETUP: Lazy<()> = Lazy::new(|| {
+            if cfg!(windows) {
+                std::env::set_var("TMP", ".");
+            } else {
+                std::env::set_var("TMPDIR", ".");
+            }
+        });
+
+        // just deref to make sure the env is setup
+        *SETUP
+    }
+
+    #[actix_rt::test]
+    async fn test_dump_normal() {
+        setup();
+
+        let tmp = tempfile::tempdir().unwrap();
+
+        let uuids = std::iter::repeat_with(Uuid::new_v4)
+            .take(4)
+            .collect::<HashSet<_>>();
+        let mut uuid_store = MockUuidStore::new();
+        let uuids_cloned = uuids.clone();
+        uuid_store
+            .expect_dump()
+            .once()
+            .returning(move |_| Box::pin(ok(uuids_cloned.clone())));
+
+        let mut index_store = MockIndexStore::new();
+        index_store.expect_get().times(4).returning(move |uuid| {
+            let mocker = Mocker::default();
+            let uuids_clone = uuids.clone();
+            mocker.when::<(), Uuid>("uuid").once().then(move |_| {
+                assert!(uuids_clone.contains(&uuid));
+                uuid
+            });
+            mocker
+                .when::<&Path, IndexResult<()>>("dump")
+                .once()
+                .then(move |_| Ok(()));
+            Box::pin(ok(Some(Index::faux(mocker))))
+        });
+
+        let index_resolver = Arc::new(IndexResolver::new(uuid_store, index_store));
+
+        let update_sender =
+            create_update_handler(index_resolver.clone(), tmp.path(), 4096 * 100).unwrap();
+
+        let task = DumpTask {
+            dump_path: tmp.path().into(),
+            // this should do nothing
+            db_path: tmp.path().into(),
+            index_resolver,
+            update_sender,
+            uid: String::from("test"),
+            update_db_size: 4096 * 10,
+            index_db_size: 4096 * 10,
+        };
+
+        task.run().await.unwrap();
+    }
+
+    #[actix_rt::test]
+    async fn error_performing_dump() {
+        let tmp = tempfile::tempdir().unwrap();
+
+        let mut uuid_store = MockUuidStore::new();
+        uuid_store
+            .expect_dump()
+            .once()
+            .returning(move |_| Box::pin(err(IndexResolverError::ExistingPrimaryKey)));
+
+        let index_store = MockIndexStore::new();
+        let index_resolver = Arc::new(IndexResolver::new(uuid_store, index_store));
+
+        let update_sender =
+            create_update_handler(index_resolver.clone(), tmp.path(), 4096 * 100).unwrap();
+
+        let task = DumpTask {
+            dump_path: tmp.path().into(),
+            // this should do nothing
+            db_path: tmp.path().into(),
+            index_resolver,
+            update_sender,
+            uid: String::from("test"),
+            update_db_size: 4096 * 10,
+            index_db_size: 4096 * 10,
+        };
+
+        assert!(task.run().await.is_err());
+    }
+}